style: upgrade pylint and dependencies and fix linting following upgrade

Author: georgeRobertson2

poetry.lock

@@ -78,9 +78,9 @@ optional = true
 
 [tool.poetry.group.lint.dependencies]
 black = "24.3.0"
-astroid = "2.14.2"
+astroid = "3.3.9"
 isort = "5.11.5"
-pylint = "2.16.4"
+pylint = "3.3.9"
 mypy = "0.991"
 boto3-stubs = {extras = ["essential"], version = "1.26.72"}
 botocore-stubs = "1.29.72"
@@ -142,10 +142,6 @@ source_pkgs = [
 [tool.coverage.report]
 show_missing = true
 
-[tool.pylint]
-init-hook = "import sys; sys.path.append('./pylint_checkers')"
-load-plugins = "check_typing_imports"
-
 [tool.pylint.main]
 extension-pkg-allow-list = ["pyspark", "lxml", "pydantic"]
 fail-under = 10.0
@@ -194,7 +190,7 @@ max-statements = 50
 min-public-methods = 2
 
 [tool.pylint.exceptions]
-overgeneral-exceptions = ["BaseException", "Exception"]
+overgeneral-exceptions = ["builtins.BaseException", "builtins.Exception"]
 
 [tool.pylint.format]
 ignore-long-lines = "^\\s*(# )?<?https?://\\S+>?$"
@@ -229,6 +225,8 @@ disable = [
     "use-symbolic-message-instead",
     "logging-fstring-interpolation",
     "fixme",
+    "too-many-positional-arguments",
+    "too-many-arguments",
 ]
 enable = ["c-extension-no-member"]
 

pylint_checkers/check_typing_imports.py

@@ -152,7 +152,7 @@ def _add_cnst_field(rel: DuckDBPyRelation) -> tuple[str, DuckDBPyRelation]:
             group_pl = entity.pl().pivot(
                 columns=[config.pivot_column],
                 values=agg_cols,
-                index=(group_cols or [const_fld]),
+                index=(group_cols or [const_fld]),  # pylint: disable=E0606
                 aggregate_function=config.agg_function,
             )
             if const_fld in group_pl.columns:

pyproject.toml

@@ -44,8 +44,8 @@ def schemas(self) -> dict[EntityName, type[BaseModel]]:
         """The per-entity schemas, as pydantic models."""
         if not self._schemas:
             for entity_name, validator in self.validators.items():
-                self._schemas[entity_name] = validator.model  # type: ignore
-        return self._schemas.copy()
+                self._schemas[entity_name] = validator.model # type: ignore # pylint: disable=E1137
+        return self._schemas.copy()  # pylint: disable=E1101
 
     @root_validator(allow_reuse=True)
     @classmethod

src/dve/core_engine/backends/implementations/duckdb/rules.py

@@ -38,7 +38,7 @@
 
 FieldName = str
 """The name of a field within a model/schema."""
-TypeOrDef = Union[
+TypeOrDef = Union[  # pylint: disable=C0103
     TypeName, "_CallableTypeDefinition", "_ModelTypeDefinition", "_TypeAliasDefinition"
 ]
 """The name or definition of a type."""
@@ -181,7 +181,7 @@ class V1EngineConfig(BaseEngineConfig):
     @validate_arguments
     def _update_rule_store(self, rule_store: dict[RuleName, BusinessComponentSpecConfigUnion]):
         """Update the rule store rules to add/override the rules from the new store."""
-        self._rule_store_rules.update(rule_store)
+        self._rule_store_rules.update(rule_store)  # pylint: disable=E1101
 
     def _load_rule_store(self, uri: URI):
         """Load a JSON rule store from the provided URI and update the stored
@@ -198,7 +198,7 @@ def _load_rule_store(self, uri: URI):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         uri_prefix = self.location.rsplit("/", 1)[0]
-        for rule_store_config in self.transformations.rule_stores:
+        for rule_store_config in self.transformations.rule_stores:  # pylint: disable=E1101
             uri = joinuri(uri_prefix, rule_store_config.filename)
             self._load_rule_store(uri)
 
@@ -281,7 +281,7 @@ def _load_rules_and_vars(self) -> tuple[list[Rule], list[TemplateVariables]]:
         rules, local_variable_list = [], []
         added_rules: set[RuleName] = set()
 
-        for index, complex_rule_config in enumerate(self.transformations.complex_rules):
+        for index, complex_rule_config in enumerate(self.transformations.complex_rules):  # pylint: disable=E1101
             rule, local_params, deps = self._resolve_business_rule(complex_rule_config)
             missing_rules = deps - added_rules
             if missing_rules:
@@ -295,9 +295,9 @@ def _load_rules_and_vars(self) -> tuple[list[Rule], list[TemplateVariables]]:
 
         rule, local_params = self._create_rule(
             name="root",
-            rules=self.transformations.rules,
-            filters=self.transformations.filters,
-            post_filter_rules=self.transformations.post_filter_rules,
+            rules=self.transformations.rules,  # pylint: disable=E1101
+            filters=self.transformations.filters,  # pylint: disable=E1101
+            post_filter_rules=self.transformations.post_filter_rules,  # pylint: disable=E1101
         )
         rules.append(rule)
         local_variable_list.append(local_params)
@@ -338,14 +338,14 @@ def load_error_message_info(self, uri):
 
     def get_reference_data_config(self) -> dict[EntityName, ReferenceConfig]:  # type: ignore
         """Gets the reference data configuration from the transformations"""
-        return self.transformations.reference_data
+        return self.transformations.reference_data  # pylint: disable=E1101
 
     def get_rule_metadata(self) -> RuleMetadata:
         """Gets the rule metadata from the Engine configuration"""
         rules, local_variables = self._load_rules_and_vars()
         return RuleMetadata(
             rules=rules,
             local_variables=local_variables,
-            global_variables=self.transformations.parameters,
+            global_variables=self.transformations.parameters,  # pylint: disable=E1101
             reference_data_config=self.get_reference_data_config(),
         )

src/dve/core_engine/backends/metadata/contract.py

@@ -137,7 +137,7 @@ def build(
             debug=debug,
             **kwargs,
         )
-        self.main_log.info(f"Output path: {self.output_prefix_uri!r}")
+        self.main_log.info(f"Output path: {self.output_prefix_uri!r}")  # pylint: disable=E1101
         return self
 
     @classmethod
@@ -155,13 +155,13 @@ def build_from_model(cls, model_str: JSONstring):
         return cls.build(**EngineRunValidation(**json.loads(model_str)).dict())
 
     def __enter__(self) -> "CoreEngine":
-        self.main_log.info("Entering pipeline context.")
+        self.main_log.info("Entering pipeline context.")  # pylint: disable=E1101
         if self._cache_dir is not None:
             raise ValueError("Pipeline already within context")
 
         self._cache_dir = TemporaryPrefix(self.cache_prefix_uri)
         self._cache_dir.__enter__()
-        self.main_log.info(f"Pipeline will cache to {self.cache_prefix!r}")
+        self.main_log.info(f"Pipeline will cache to {self.cache_prefix!r}")  # pylint: disable=E1101
         return self
 
     def __exit__(
@@ -170,14 +170,14 @@ def __exit__(
         exc_value: Optional[Exception],
         traceback: Optional[TracebackType],
     ) -> None:
-        self.main_log.info(f"Exiting pipeline context, clearing {self.cache_prefix!r}")
+        self.main_log.info(f"Exiting pipeline context, clearing {self.cache_prefix!r}")  # pylint: disable=E1101
         cache_dir = self._cache_dir
         self._cache_dir = None
 
         if cache_dir is not None:
             cache_dir.__exit__(exc_type, exc_value, traceback)
 
-        self.main_log.info("Cleared cache.")
+        self.main_log.info("Cleared cache.")  # pylint: disable=E1101
 
     @property
     def cache_prefix(self) -> URI:
@@ -198,17 +198,17 @@ def _write_entity_outputs(self, entities: SparkEntities) -> SparkEntities:
         """
         output_entities = {}
 
-        self.main_log.info(f"Writing entities to the output location: {self.output_prefix_uri}")
+        self.main_log.info(f"Writing entities to the output location: {self.output_prefix_uri}")  # pylint: disable=E1101
         for entity_name, entity in entities.items():
             entity = entity.drop(RECORD_INDEX_COLUMN_NAME)
 
-            self.main_log.info(f"Entity: {entity_name} {type(entity)}")
+            self.main_log.info(f"Entity: {entity_name} {type(entity)}")  # pylint: disable=E1101
 
             output_uri = joinuri(self.output_prefix_uri, entity_name)
             if get_resource_exists(output_uri):
-                self.main_log.info(f"{output_uri} already exists - will be overwritten")
+                self.main_log.info(f"{output_uri} already exists - will be overwritten")  # pylint: disable=E1101
 
-            self.main_log.info(f"+ Writing parquet output to {output_uri!r}")
+            self.main_log.info(f"+ Writing parquet output to {output_uri!r}")  # pylint: disable=E1101
             entity.write.mode("overwrite").parquet(output_uri)
             spark_session = SparkSession.builder.getOrCreate()
             output_entities[entity_name] = spark_session.read.format("parquet").load(
@@ -228,7 +228,7 @@ def _write_outputs(self, entities: SparkEntities) -> SparkEntities:
 
     def _show_available_entities(self, entities: SparkEntities, *, verbose: bool = False) -> None:
         """Print current entities."""
-        self.main_log.info("Displaying available dataframes in this run:")
+        self.main_log.info("Displaying available dataframes in this run:")  # pylint: disable=E1101
 
         for entity_name, entity in entities.items():
             # FIXME: Currently a print statement because log messages

src/dve/core_engine/configuration/v1/__init__.py

@@ -5,7 +5,7 @@
 from multiprocessing import Queue as ProcessQueue
 from pathlib import Path
 from queue import Queue as ThreadQueue
-from typing import TYPE_CHECKING, Any, List, Optional, TypeVar, Union  # pylint: disable=W1901
+from typing import TYPE_CHECKING, Any, List, Optional, TypeVar, Union
 
 from pyspark.sql import DataFrame
 from pyspark.sql.types import StructType

src/dve/core_engine/engine.py

@@ -152,7 +152,7 @@ def check_validates(cls, value: Optional[str]) -> bool:
         return is_valid
 
     @classmethod
-    def validate(cls, value: Optional[str], field: fields.ModelField) -> str:  # type: ignore
+    def validate(cls, value: Optional[str], field: fields.ModelField) -> str:  # type: ignore  # pylint: disable=W0221
         """Validates the given postcode"""
         nhs_number = cls.ensure_format(value)
 

src/dve/core_engine/type_hints.py

@@ -81,15 +81,15 @@ def validate_error_message(cls, value: str, values: dict[str, Any]) -> str:
     def get_field_validator(self, field_name: str, **extra_kwargs: Any) -> classmethod:
         """Get a validator a given field."""
         func = getattr(function_library, self.name)
+        _kwargs = self.kwargs_ | extra_kwargs
         return create_validator(
             func,
             field_name,
             exc.ERRORS[self.error_type.replace("_", "")],
             self.error_message,
             return_result=True,
             fields=self.fields,
-            **self.kwargs_,
-            **extra_kwargs,
+            **_kwargs,
         )
 
 
@@ -257,7 +257,7 @@ def get_type_and_validators(
             python_type_callable = chain_get(self.callable, *type_mappings, pyd, dt, __builtins__)
             if not callable(python_type_callable):
                 raise ValueError("Fetched callable is not callable")
-            python_type = python_type_callable(**self.constraints)
+            python_type = python_type_callable(**self.constraints)  # pylint: disable=E1134
         else:
             raise ValueError("No field type set")
 
@@ -360,7 +360,7 @@ def as_model(
                 field_name,
                 *type_mappings,
                 schemas=schemas,
-                is_mandatory=field_name in self.mandatory_fields,
+                is_mandatory=field_name in self.mandatory_fields,  # pylint: disable=E1135
             )
             pyd_fields[field_name] = (python_type, default)
             validators.update(field_validators)
@@ -398,7 +398,7 @@ def load_models(
     ) -> dict[EntityName, pyd.main.ModelMetaclass]:
         """Load the models from the dataset definition."""
         loaded_schemas: dict[EntityName, pyd.main.ModelMetaclass] = {}
-        for model_name, specification in self.schemas.items():
+        for model_name, specification in self.schemas.items():  # pylint: disable=E1101
             loaded_schemas[model_name] = specification.as_model(
                 model_name, self.types, *type_mappings, schemas=loaded_schemas
             )