From b4cd036705b1472a4ed3874fadb942ceb43d53f9 Mon Sep 17 00:00:00 2001 From: ducdetronquito Date: Mon, 22 Sep 2025 12:15:03 +0200 Subject: [PATCH] chore: Setup pyright and expose type annotations --- .github/workflows/pullrequest-push.yml | 1 + .pre-commit-config.yaml | 5 +++ magicparse/__init__.py | 15 ++++---- magicparse/builders.py | 46 +++++++++++------------ magicparse/fields.py | 38 +++++++++---------- magicparse/post_processors.py | 13 ++++--- magicparse/pre_processors.py | 13 ++++--- magicparse/py.typed | 0 magicparse/schema.py | 34 +++++++++-------- magicparse/transform.py | 23 ++++++------ magicparse/type_converters.py | 15 +++++--- magicparse/validators.py | 13 +++++-- poetry.lock | 52 +++++++++++++++++++++++++- pyproject.toml | 9 +++++ tests/test_builders.py | 8 ++-- tests/test_fields.py | 5 ++- tests/test_post_processors.py | 3 +- tests/test_pre_processors.py | 5 ++- tests/test_schema.py | 8 ++-- tests/test_type_converters.py | 3 +- tests/test_validators.py | 3 +- 21 files changed, 202 insertions(+), 110 deletions(-) create mode 100644 magicparse/py.typed diff --git a/.github/workflows/pullrequest-push.yml b/.github/workflows/pullrequest-push.yml index 166f438..a7b7fb1 100644 --- a/.github/workflows/pullrequest-push.yml +++ b/.github/workflows/pullrequest-push.yml @@ -50,6 +50,7 @@ jobs: poetry install --no-interaction --no-ansi --no-root --only dev ruff check --diff ./ ruff format --check --diff ./ + pyright secrets: inherit unit-tests: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6e3d70c..1ec6cba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,3 +13,8 @@ repos: entry: poetry run ruff format language: system types: [file, python] + - id: pyright + name: Pyright + entry: poetry run pyright + language: system + types: [file, python] diff --git a/magicparse/__init__.py b/magicparse/__init__.py index 19b067a..974dccf 100644 --- a/magicparse/__init__.py +++ b/magicparse/__init__.py @@ -1,3 +1,4 @@ +from collections.abc import Sequence from io import BytesIO from .schema import ( @@ -15,7 +16,7 @@ ) from .transform import Transform from .type_converters import TypeConverter, builtins as builtins_type_converters -from typing import Any, Dict, Iterable, List, Union +from typing import Any, Iterable from .validators import Validator, builtins as builtins_validators @@ -33,23 +34,21 @@ ] -def parse(data: Union[bytes, BytesIO], schema_options: Dict[str, Any]) -> List[RowParsed | RowSkipped | RowFailed]: +def parse(data: bytes | BytesIO, schema_options: dict[str, Any]) -> list[RowParsed | RowSkipped | RowFailed]: schema_definition = Schema.build(schema_options) return schema_definition.parse(data) -def stream_parse( - data: Union[bytes, BytesIO], schema_options: Dict[str, Any] -) -> Iterable[RowParsed | RowSkipped | RowFailed]: +def stream_parse(data: bytes | BytesIO, schema_options: dict[str, Any]) -> Iterable[RowParsed | RowSkipped | RowFailed]: schema_definition = Schema.build(schema_options) return schema_definition.stream_parse(data) -Registrable = Union[Schema, Transform] +Registrable = type[Schema] | type[Transform] -def register(items: Union[Registrable, List[Registrable]]) -> None: - if not isinstance(items, list): +def register(items: Registrable | Sequence[Registrable]) -> None: + if not isinstance(items, Sequence): items = [items] for item in items: diff --git a/magicparse/builders.py b/magicparse/builders.py index 6e60a30..562b8c4 100644 --- a/magicparse/builders.py +++ b/magicparse/builders.py @@ -1,12 +1,15 @@ from abc import ABC from decimal import Decimal +from typing import Any, cast from .transform import Transform, OnError class Builder(Transform, ABC): + registry = dict[str, type["Builder"]]() + @classmethod - def build(cls, options: dict) -> "Builder": + def build(cls, options: dict[str, Any]) -> "Builder": try: name = options["name"] except: @@ -25,23 +28,20 @@ def build(cls, options: dict) -> "Builder": class Concat(Builder): - def __init__(self, on_error: OnError, fields: list[str]) -> None: + def __init__(self, on_error: OnError, fields: Any) -> None: super().__init__(on_error) if ( - not fields - or isinstance(fields, str) - or not isinstance(fields, list) - or not all(isinstance(field, str) for field in fields) - or len(fields) < 2 + not isinstance(fields, list) + or not all(isinstance(field, str) for field in fields) # type: ignore[reportUnknownVariableType] + or len(fields) < 2 # type: ignore[reportUnknownVariableType] ): raise ValueError( "composite-processor 'concat': 'fields' parameter must be a list[str] with at least two elements" ) + self.fields = cast(list[str], fields) - self.fields = fields - - def apply(self, row: dict) -> str: - return "".join(row[field] for field in self.fields) + def apply(self, value: dict[str, Any]) -> str: + return "".join(value[field] for field in self.fields) @staticmethod def key() -> str: @@ -49,7 +49,7 @@ def key() -> str: class Divide(Builder): - def __init__(self, on_error: OnError, numerator: str, denominator: str) -> None: + def __init__(self, on_error: OnError, numerator: Any, denominator: Any) -> None: super().__init__(on_error) if not numerator or not isinstance(numerator, str): raise ValueError("builder 'divide': 'numerator' parameter must be a non null str") @@ -58,8 +58,8 @@ def __init__(self, on_error: OnError, numerator: str, denominator: str) -> None: self.numerator = numerator self.denominator = denominator - def apply(self, row: dict) -> Decimal: - return row[self.numerator] / row[self.denominator] + def apply(self, value: dict[str, Any]) -> Decimal: + return value[self.numerator] / value[self.denominator] @staticmethod def key() -> str: @@ -67,7 +67,7 @@ def key() -> str: class Multiply(Builder): - def __init__(self, on_error: OnError, x_factor: str, y_factor: str) -> None: + def __init__(self, on_error: OnError, x_factor: Any, y_factor: Any) -> None: super().__init__(on_error) if not x_factor or not isinstance(x_factor, str): raise ValueError("builder 'multiply': 'x_factor' parameter must be a non null str") @@ -76,8 +76,8 @@ def __init__(self, on_error: OnError, x_factor: str, y_factor: str) -> None: self.x_factor = x_factor self.y_factor = y_factor - def apply(self, row: dict): - return row[self.x_factor] * row[self.y_factor] + def apply(self, value: dict[str, Any]): + return value[self.x_factor] * value[self.y_factor] @staticmethod def key() -> str: @@ -85,19 +85,19 @@ def key() -> str: class Coalesce(Builder): - def __init__(self, on_error: OnError, fields: list[str]) -> None: + def __init__(self, on_error: OnError, fields: Any) -> None: super().__init__(on_error) if not fields: raise ValueError("parameters should defined fields to coalesce") - if not isinstance(fields, list) or not all(isinstance(field, str) for field in fields) or len(fields) < 2: + if not isinstance(fields, list) or not all(isinstance(field, str) for field in fields) or len(fields) < 2: # type: ignore[reportUnknownVariableType] raise ValueError("parameters should have two fields at least") - self.fields = fields + self.fields = cast(list[str], fields) - def apply(self, row: dict) -> str: + def apply(self, value: dict[str, Any]) -> str | None: for field in self.fields: - if row[field]: - return row[field] + if value[field]: + return value[field] return None @staticmethod diff --git a/magicparse/fields.py b/magicparse/fields.py index 56b9d80..44613f2 100644 --- a/magicparse/fields.py +++ b/magicparse/fields.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import List +from typing import Any from .builders import Builder from .type_converters import TypeConverter @@ -10,7 +10,7 @@ class Field(ABC): - def __init__(self, key: str, options: dict) -> None: + def __init__(self, key: str, options: dict[str, Any]) -> None: self.key = key pre_processors = [PreProcessor.build(item) for item in options.get("pre-processors", [])] type_converter = TypeConverter.build(options) @@ -37,19 +37,19 @@ def _process_raw_value(self, raw_value: str) -> Result: return Ok(value=raw_value) @abstractmethod - def _read_raw_value(self, row: List[str] | dict) -> str: + def _read_raw_value(self, row: Any) -> str: pass - def parse(self, row: List[str] | dict) -> Result: + def parse(self, row: str | list[str] | dict[str, Any]) -> Result: raw_value = self._read_raw_value(row) return self._process_raw_value(raw_value) @abstractmethod - def error(self, exception: Exception): + def error(self, exception: Exception) -> dict[str, Any]: pass @classmethod - def build(cls, options: dict) -> "Field": + def build(cls, options: dict[str, Any]) -> "Field": options = options.copy() key = options.pop("key", None) if not key: @@ -68,14 +68,14 @@ def build(cls, options: dict) -> "Field": class CsvField(Field): - def __init__(self, key: str, options: dict) -> None: + def __init__(self, key: str, options: dict[str, Any]) -> None: super().__init__(key, options) - self.column_number = options["column-number"] + self.column_number = int(options["column-number"]) - def _read_raw_value(self, row: List[str] | dict) -> str: + def _read_raw_value(self, row: list[str]) -> str: return row[self.column_number - 1] - def error(self, exception: Exception) -> dict: + def error(self, exception: Exception) -> dict[str, Any]: return { "column-number": self.column_number, "field-key": self.key, @@ -84,16 +84,16 @@ def error(self, exception: Exception) -> dict: class ColumnarField(Field): - def __init__(self, key: str, options: dict) -> None: + def __init__(self, key: str, options: dict[str, Any]) -> None: super().__init__(key, options) - self.column_start = options["column-start"] - self.column_length = options["column-length"] + self.column_start = int(options["column-start"]) + self.column_length = int(options["column-length"]) self.column_end = self.column_start + self.column_length - def _read_raw_value(self, row: str | dict) -> str: + def _read_raw_value(self, row: str) -> str: return row[self.column_start : self.column_end] - def error(self, exception: Exception) -> dict: + def error(self, exception: Exception) -> dict[str, Any]: return { "column-start": self.column_start, "column-length": self.column_length, @@ -103,21 +103,21 @@ def error(self, exception: Exception) -> dict: class ComputedField(Field): - def __init__(self, key: str, options: dict) -> None: + def __init__(self, key: str, options: dict[str, Any]) -> None: super().__init__(key, options) self.builder = Builder.build(options["builder"]) - def _read_raw_value(self, row: List[str] | dict) -> str: + def _read_raw_value(self, row: dict[str, Any]) -> str: return self.builder.apply(row) - def error(self, exception: Exception) -> dict: + def error(self, exception: Exception) -> dict[str, Any]: return { "field-key": self.key, "error": exception.args[0], } @classmethod - def build(cls, options: dict) -> "ComputedField": + def build(cls, options: dict[str, Any]) -> "ComputedField": key = options.pop("key", None) if not key: raise ValueError("key is required in computed field definition") diff --git a/magicparse/post_processors.py b/magicparse/post_processors.py index 9a475f7..7124099 100644 --- a/magicparse/post_processors.py +++ b/magicparse/post_processors.py @@ -1,11 +1,13 @@ from .transform import Transform, OnError from decimal import Decimal -from typing import TypeVar +from typing import Any class PostProcessor(Transform): + registry = dict[str, type["PostProcessor"]]() + @classmethod - def build(cls, options: dict) -> "PostProcessor": + def build(cls, options: dict[str, Any]) -> "PostProcessor": try: name = options["name"] except: @@ -23,9 +25,10 @@ def build(cls, options: dict) -> "PostProcessor": return post_processor(on_error=on_error) -class Divide(PostProcessor): - Number = TypeVar("Number", int, float, Decimal) +type Number = int | float | Decimal + +class Divide(PostProcessor): def __init__(self, on_error: OnError, denominator: int) -> None: super().__init__(on_error) if denominator <= 0: @@ -42,8 +45,6 @@ def key() -> str: class Round(PostProcessor): - Number = TypeVar("Number", int, float, Decimal) - def __init__(self, on_error: OnError, precision: int) -> None: super().__init__(on_error) if precision < 0: diff --git a/magicparse/pre_processors.py b/magicparse/pre_processors.py index f87238d..9aebec8 100644 --- a/magicparse/pre_processors.py +++ b/magicparse/pre_processors.py @@ -1,10 +1,13 @@ import re +from typing import Any from .transform import Transform, OnError class PreProcessor(Transform): + registry = dict[str, type["PreProcessor"]]() + @classmethod - def build(cls, options: dict) -> "PreProcessor": + def build(cls, options: dict[str, Any]) -> "PreProcessor": try: name = options["name"] except: @@ -36,7 +39,7 @@ def key() -> str: class Map(PreProcessor): - def __init__(self, on_error: OnError, values: dict) -> None: + def __init__(self, on_error: OnError, values: dict[str, Any]) -> None: super().__init__(on_error) self.values = values self._keys = ", ".join(f"'{key}'" for key in self.values.keys()) @@ -91,11 +94,11 @@ def key() -> str: class RegexExtract(PreProcessor): def __init__(self, on_error: OnError, pattern: str) -> None: super().__init__(on_error) - pattern = re.compile(pattern) - if "value" not in pattern.groupindex: + _pattern = re.compile(pattern) + if "value" not in _pattern.groupindex: raise ValueError("regex-extract's pattern must contain a group named 'value'") - self.pattern = pattern + self.pattern = _pattern def apply(self, value: str) -> str: match = re.match(self.pattern, value) diff --git a/magicparse/py.typed b/magicparse/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/magicparse/schema.py b/magicparse/schema.py index e8d1fe9..7f209d2 100644 --- a/magicparse/schema.py +++ b/magicparse/schema.py @@ -6,25 +6,25 @@ from magicparse.transform import SkipRow from .fields import Field, ComputedField from io import BytesIO -from typing import Any, Dict, List, Union, Iterable +from typing import Any, Dict, Iterator, List, Union, Iterable @dataclass(frozen=True, slots=True) class RowParsed: row_number: int - values: dict + values: dict[str, Any] @dataclass(frozen=True, slots=True) class RowSkipped: row_number: int - errors: list[dict] + errors: list[dict[str, Any]] @dataclass(frozen=True, slots=True) class RowFailed: row_number: int - errors: list[dict] + errors: list[dict[str, Any]] class Schema(ABC): @@ -40,15 +40,16 @@ def __init__(self, options: Dict[str, Any]) -> None: self.encoding = options.get("encoding", "utf-8") @abstractmethod - def get_reader(self, stream: BytesIO) -> Iterable: + def get_reader(self, stream: BytesIO) -> Iterator[list[str] | str]: pass @staticmethod + @abstractmethod def key() -> str: pass @classmethod - def build(cls, options: Dict[str, Any]) -> "Schema": + def build(cls, options: dict[str, Any]) -> "Schema": file_type = options["file_type"] schema = cls.registry.get(file_type) if schema: @@ -57,13 +58,13 @@ def build(cls, options: Dict[str, Any]) -> "Schema": raise ValueError("unknown file type") @classmethod - def register(cls, schema: "Schema") -> None: + def register(cls, schema: type["Schema"]) -> None: if not hasattr(cls, "registry"): - cls.registry = {} + cls.registry = dict[str, type["Schema"]]() cls.registry[schema.key()] = schema - def parse(self, data: Union[bytes, BytesIO]) -> List[RowParsed] | List[RowSkipped] | List[RowFailed]: + def parse(self, data: Union[bytes, BytesIO]) -> list[RowParsed | RowSkipped | RowFailed]: return list(self.stream_parse(data)) def stream_parse(self, data: Union[bytes, BytesIO]) -> Iterable[RowParsed | RowSkipped | RowFailed]: @@ -97,14 +98,17 @@ def stream_parse(self, data: Union[bytes, BytesIO]) -> Iterable[RowParsed | RowS yield RowParsed(row_number, {**fields.values, **computed_fields.values}) def process_fields( - self, fields: List[Field], row: List[str], row_number: int + self, fields: list[Field] | list[ComputedField], row: str | list[str] | dict[str, Any], row_number: int ) -> RowParsed | RowSkipped | RowFailed: - item = {} - errors = [] + item = dict[str, Any]() + errors = list[dict[str, Any]]() skip_row = False for field in fields: try: - source = row | item if isinstance(row, dict) else row + if isinstance(row, dict): + source = row | item + else: + source = row parsed_value = field.parse(source) except Exception as exc: errors.append(field.error(exc)) @@ -129,7 +133,7 @@ def __init__(self, options: Dict[str, Any]) -> None: self.delimiter = options.get("delimiter", ",") self.quotechar = options.get("quotechar", None) - def get_reader(self, stream: BytesIO) -> Iterable[List[str]]: + def get_reader(self, stream: BytesIO) -> Iterator[list[str]]: stream_reader = codecs.getreader(self.encoding) stream_content = stream_reader(stream) csv_quoting = csv.QUOTE_NONE @@ -148,7 +152,7 @@ def key() -> str: class ColumnarSchema(Schema): - def get_reader(self, stream: BytesIO) -> Iterable[str]: + def get_reader(self, stream: BytesIO) -> Iterator[str]: stream_reader_factory = codecs.getreader(self.encoding) stream_reader = stream_reader_factory(stream) while True: diff --git a/magicparse/transform.py b/magicparse/transform.py index dcc68c6..de6343e 100644 --- a/magicparse/transform.py +++ b/magicparse/transform.py @@ -1,7 +1,7 @@ -from abc import ABC, abstractclassmethod, abstractmethod, abstractstaticmethod +from abc import ABC, abstractmethod from dataclasses import dataclass -from enum import Enum -from typing import Any +from enum import StrEnum +from typing import Any, Self @dataclass(frozen=True, slots=True) @@ -17,30 +17,31 @@ class SkipRow: type Result = Ok | SkipRow -class OnError(Enum): +class OnError(StrEnum): RAISE = "raise" SKIP_ROW = "skip-row" class Transform(ABC): + registry: dict[str, type[Self]] + def __init__(self, on_error: OnError) -> None: self.on_error = on_error - @abstractclassmethod - def build(cls, options: dict) -> "Transform": + @classmethod + @abstractmethod + def build(cls, options: dict[str, Any]) -> "Transform": pass @abstractmethod def apply(self, value: Any) -> Any: pass - @abstractstaticmethod + @staticmethod + @abstractmethod def key() -> str: pass @classmethod - def register(cls, transform: "Transform") -> None: - if not hasattr(cls, "registry"): - cls.registry = {} - + def register(cls, transform: type[Self]) -> None: cls.registry[transform.key()] = transform diff --git a/magicparse/type_converters.py b/magicparse/type_converters.py index 945276d..cf33930 100644 --- a/magicparse/type_converters.py +++ b/magicparse/type_converters.py @@ -1,20 +1,25 @@ from abc import abstractmethod from datetime import datetime, time from decimal import Decimal -from typing import Any +from typing import Any, cast from .transform import Transform from .transform import OnError class TypeConverter(Transform): + registry = dict[str, type["TypeConverter"]]() + def __init__(self, nullable: bool, on_error: OnError) -> None: super().__init__(on_error) self.nullable = nullable def apply(self, value: str | None) -> Any | None: - if value is None and self.nullable: - return None + if value is None: + if self.nullable: + return None + else: + raise ValueError("type is non nullable") return self.convert(value) @@ -23,9 +28,9 @@ def convert(self, value: str) -> Any: pass @classmethod - def build(cls, options) -> "TypeConverter": + def build(cls, options: dict[str, Any]) -> "Transform": try: - type = options["type"] + type = cast(str | dict[str, Any], options["type"]) if isinstance(type, str): key = type type = {} diff --git a/magicparse/validators.py b/magicparse/validators.py index a777998..78e4ca8 100644 --- a/magicparse/validators.py +++ b/magicparse/validators.py @@ -1,11 +1,14 @@ from decimal import Decimal +from typing import Any from .transform import Transform, OnError import re class Validator(Transform): + registry = dict[str, type["Validator"]]() + @classmethod - def build(cls, options: dict) -> "Validator": + def build(cls, options: dict[str, Any]) -> "Validator": try: name = options["name"] except: @@ -16,6 +19,8 @@ def build(cls, options: dict) -> "Validator": except: raise ValueError(f"invalid validator '{name}'") + assert issubclass(validator, Validator) + on_error = options.setdefault("on-error", OnError.RAISE) if "parameters" in options: return validator(on_error=on_error, **options["parameters"]) @@ -24,11 +29,11 @@ def build(cls, options: dict) -> "Validator": class RegexMatches(Validator): - def __init__(self, on_error: str, pattern: str) -> None: + def __init__(self, on_error: OnError, pattern: str) -> None: super().__init__(on_error) self.pattern = re.compile(pattern) - def apply(self, value: str | None) -> str: + def apply(self, value: str) -> str | None: if re.match(self.pattern, value): return value @@ -40,7 +45,7 @@ def key() -> str: class GreaterThan(Validator): - def __init__(self, on_error: str, threshold: float) -> None: + def __init__(self, on_error: OnError, threshold: float) -> None: super().__init__(on_error) self.threshold = Decimal(threshold) diff --git a/poetry.lock b/poetry.lock index 46aca23..2665a96 100644 --- a/poetry.lock +++ b/poetry.lock @@ -183,6 +183,24 @@ files = [ {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, ] +[[package]] +name = "nodejs-wheel-binaries" +version = "22.19.0" +description = "unoffical Node.js package" +optional = false +python-versions = ">=3.7" +files = [ + {file = "nodejs_wheel_binaries-22.19.0-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:43eca1526455a1fb4cb777095198f7ebe5111a4444749c87f5c2b84645aaa72a"}, + {file = "nodejs_wheel_binaries-22.19.0-py2.py3-none-macosx_11_0_x86_64.whl", hash = "sha256:feb06709e1320790d34babdf71d841ec7f28e4c73217d733e7f5023060a86bfc"}, + {file = "nodejs_wheel_binaries-22.19.0-py2.py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db9f5777292491430457c99228d3a267decf12a09d31246f0692391e3513285e"}, + {file = "nodejs_wheel_binaries-22.19.0-py2.py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1392896f1a05a88a8a89b26e182d90fdf3020b4598a047807b91b65731e24c00"}, + {file = "nodejs_wheel_binaries-22.19.0-py2.py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:9164c876644f949cad665e3ada00f75023e18f381e78a1d7b60ccbbfb4086e73"}, + {file = "nodejs_wheel_binaries-22.19.0-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:6b4b75166134010bc9cfebd30dc57047796a27049fef3fc22316216d76bc0af7"}, + {file = "nodejs_wheel_binaries-22.19.0-py2.py3-none-win_amd64.whl", hash = "sha256:3f271f5abfc71b052a6b074225eca8c1223a0f7216863439b86feaca814f6e5a"}, + {file = "nodejs_wheel_binaries-22.19.0-py2.py3-none-win_arm64.whl", hash = "sha256:666a355fe0c9bde44a9221cd543599b029045643c8196b8eedb44f28dc192e06"}, + {file = "nodejs_wheel_binaries-22.19.0.tar.gz", hash = "sha256:e69b97ef443d36a72602f7ed356c6a36323873230f894799f4270a853932fdb3"}, +] + [[package]] name = "packaging" version = "24.1" @@ -276,6 +294,27 @@ files = [ {file = "pyflakes-3.4.0.tar.gz", hash = "sha256:b24f96fafb7d2ab0ec5075b7350b3d2d2218eab42003821c06344973d3ea2f58"}, ] +[[package]] +name = "pyright" +version = "1.1.405" +description = "Command line wrapper for pyright" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyright-1.1.405-py3-none-any.whl", hash = "sha256:a2cb13700b5508ce8e5d4546034cb7ea4aedb60215c6c33f56cec7f53996035a"}, + {file = "pyright-1.1.405.tar.gz", hash = "sha256:5c2a30e1037af27eb463a1cc0b9f6d65fec48478ccf092c1ac28385a15c55763"}, +] + +[package.dependencies] +nodeenv = ">=1.6.0" +nodejs-wheel-binaries = {version = "*", optional = true, markers = "extra == \"nodejs\""} +typing-extensions = ">=4.1" + +[package.extras] +all = ["nodejs-wheel-binaries", "twine (>=3.4.1)"] +dev = ["twine (>=3.4.1)"] +nodejs = ["nodejs-wheel-binaries"] + [[package]] name = "pytest" version = "8.3.2" @@ -442,6 +481,17 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "typing-extensions" +version = "4.15.0" +description = "Backported and Experimental Type Hints for Python 3.9+" +optional = false +python-versions = ">=3.9" +files = [ + {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, + {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, +] + [[package]] name = "urllib3" version = "2.2.2" @@ -482,4 +532,4 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "fe8198ebec8153b02401f4c89ae4a2b26e49d6d07a463ff3003611527084c482" +content-hash = "fabe335ba81c7bae7c6401aa39ce93fb2ffb3d3e81e2ee5e1fb58ae8ab5ddba7" diff --git a/pyproject.toml b/pyproject.toml index 0987855..ba6760f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ awscli = "~1" flake8-pyproject = "~1.2.3" ruff = "^0.13.1" pre-commit = "^4.3.0" +pyright = {version = "^1.1.405", extras = ["nodejs"]} [build-system] requires = ["poetry-core>=1.2.0"] @@ -33,3 +34,11 @@ ignore = [ [tool.pytest.ini_options] python_files = ["tests/*"] + +[tool.pyright] +venvPath = "." +venv = ".venv" +typeCheckingMode = "strict" +include = ["magicparse/**", "tests/**"] + +executionEnvironments = [{ root = "magicparse" }] diff --git a/tests/test_builders.py b/tests/test_builders.py index 2eff7a1..9ae8b5a 100644 --- a/tests/test_builders.py +++ b/tests/test_builders.py @@ -1,9 +1,11 @@ from decimal import Decimal +from typing import Any import pytest from unittest import TestCase from magicparse import Builder +from magicparse.transform import OnError class TestBuild(TestCase): @@ -12,11 +14,11 @@ class WithoutParamBuilder(Builder): def key() -> str: return "without-param" - def apply(self, value): + def apply(self, value: Any): pass class WithParamBuilder(Builder): - def __init__(self, on_error: str, setting: str) -> None: + def __init__(self, on_error: OnError, setting: str) -> None: super().__init__(on_error) self.setting = setting @@ -24,7 +26,7 @@ def __init__(self, on_error: str, setting: str) -> None: def key() -> str: return "with-param" - def apply(self, value): + def apply(self, value: Any): pass def test_without_parameter(self): diff --git a/tests/test_fields.py b/tests/test_fields.py index a534334..d10b760 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1,4 +1,5 @@ from decimal import Decimal +from typing import Any import pytest from magicparse.transform import Ok @@ -10,10 +11,10 @@ class DummyField(Field): - def _read_raw_value(self, row: str | dict) -> str: + def _read_raw_value(self, row: Any) -> str: return row - def error(self, exception: Exception): + def error(self, exception: Exception) -> dict[str, Any]: return {} diff --git a/tests/test_post_processors.py b/tests/test_post_processors.py index 286f472..4c76769 100644 --- a/tests/test_post_processors.py +++ b/tests/test_post_processors.py @@ -1,4 +1,5 @@ from decimal import Decimal +from typing import Any from magicparse.post_processors import Divide, PostProcessor import pytest from unittest import TestCase @@ -55,7 +56,7 @@ class NoThanksPostProcessor(PostProcessor): def key() -> str: return "no-thanks" - def apply(self, value): + def apply(self, value: Any): return f"{value} ? No thanks" def test_register(self): diff --git a/tests/test_pre_processors.py b/tests/test_pre_processors.py index a633129..a4413e2 100644 --- a/tests/test_pre_processors.py +++ b/tests/test_pre_processors.py @@ -1,4 +1,5 @@ import re +from typing import Any from magicparse.pre_processors import ( LeftPadZeroes, Map, @@ -138,7 +139,7 @@ def test_pattern_found(self): "parameters": {"pattern": "^xxx(?P\\d{13})xxx$"}, } ) - pre_processor.apply("xxx9780201379624xxx") == "9780201379624" + assert pre_processor.apply("xxx9780201379624xxx") == "9780201379624" class TestRegister(TestCase): @@ -147,7 +148,7 @@ class YesPreProcessor(PreProcessor): def key() -> str: return "yes" - def apply(self, value): + def apply(self, value: Any): return f"YES {value}" def test_register(self): diff --git a/tests/test_schema.py b/tests/test_schema.py index 9f3ebc7..887d065 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1,5 +1,6 @@ from decimal import Decimal -from typing import Any +from io import BytesIO +from typing import Any, Iterator from magicparse import Schema from magicparse.post_processors import PostProcessor @@ -385,8 +386,8 @@ class PipedSchema(Schema): def key() -> str: return "piped" - def get_reader(self, stream): - for item in stream.read().split("|"): + def get_reader(self, stream: BytesIO) -> Iterator[list[str] | str]: + for item in stream.read().decode("utf-8").split("|"): yield [item] def test_register(self): @@ -477,6 +478,7 @@ def test_concat(self): rows = list(schema.stream_parse(b"A;B")) print(rows) assert len(rows) == 1 + assert isinstance(rows[0], RowParsed) assert rows[0].row_number == 1 assert rows[0].values == { "field_1": "A", diff --git a/tests/test_type_converters.py b/tests/test_type_converters.py index 63edab6..1d6e266 100644 --- a/tests/test_type_converters.py +++ b/tests/test_type_converters.py @@ -1,5 +1,6 @@ from datetime import datetime, time, timedelta, timezone from decimal import Decimal +from typing import Any from unittest import TestCase from uuid import UUID @@ -125,7 +126,7 @@ class GuidConverter(TypeConverter): def key() -> str: return "guid" - def convert(self, value): + def convert(self, value: Any): return UUID(value) def test_register(self): diff --git a/tests/test_validators.py b/tests/test_validators.py index 4cc90f0..42247ea 100644 --- a/tests/test_validators.py +++ b/tests/test_validators.py @@ -1,4 +1,5 @@ from decimal import Decimal +from typing import Any from magicparse.validators import GreaterThan, NotNullOrEmpty, RegexMatches, Validator import pytest import re @@ -70,7 +71,7 @@ class IsTheAnswerValidator(Validator): def key() -> str: return "is-the-answer" - def apply(self, value): + def apply(self, value: Any): if value == 42: return value raise ValueError(f"{value} is not the answer !")