style: sort formating, linting and static typing

Author: stevenhsd

src/dve/core_engine/backends/base/contract.py

@@ -337,9 +337,9 @@ def read_raw_entities(
         successful = True
         for entity_name, resource in entity_locations.items():
             reader_metadata = contract_metadata.reader_metadata[entity_name]
-            extension = "." + (
-                get_file_suffix(resource) or ""
-            ).lower()  # Already checked that extension supported.
+            extension = (
+                "." + (get_file_suffix(resource) or "").lower()
+            )  # Already checked that extension supported.
 
             reader_config = reader_metadata[extension]
             reader_type = get_reader(reader_config.reader)
@@ -368,11 +368,11 @@ def read_raw_entities(
                 messages.extend(new_messages)
 
         return entities, dedup_messages(messages), successful
-    
+
     def add_record_index(self, entity: EntityType, **kwargs) -> EntityType:
         """Add a record index to the entity"""
         raise NotImplementedError(f"add_record_index not implemented in {self.__class__}")
-    
+
     def drop_record_index(self, entity: EntityType, **kwargs) -> EntityType:
         """Drop a record index from the entity"""
         raise NotImplementedError(f"drop_record_index not implemented in {self.__class__}")

src/dve/core_engine/backends/base/reader.py

@@ -126,11 +126,11 @@ def read_to_entity_type(
             raise ReaderLacksEntityTypeSupport(entity_type=entity_type) from err
 
         return reader_func(self, resource, entity_name, schema)
-    
+
     def add_record_index(self, entity: EntityType, **kwargs) -> EntityType:
         """Add a record index to the entity"""
         raise NotImplementedError(f"add_record_index not implemented in {self.__class__}")
-    
+
     def drop_record_index(self, entity: EntityType, **kwargs) -> EntityType:
         """Drop a record index to the entity"""
         raise NotImplementedError(f"drop_record_index not implemented in {self.__class__}")

src/dve/core_engine/backends/base/utilities.py

@@ -12,10 +12,6 @@
 from dve.core_engine.type_hints import ExpressionArray, MultiExpression
 from dve.parser.type_hints import URI
 
-import polars as pl
-
-from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
-
 BRACKETS = {"(": ")", "{": "}", "[": "]", "<": ">"}
 """A mapping of opening brackets to their closing counterpart."""
 STRING_START_CHARS = {'"', "'"}

src/dve/core_engine/backends/implementations/duckdb/contract.py

@@ -16,7 +16,6 @@
 from pydantic import BaseModel
 from pydantic.fields import ModelField
 
-from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
 import dve.parser.file_handling as fh
 from dve.common.error_utils import (
     BackgroundMessageWriter,
@@ -39,6 +38,7 @@
 from dve.core_engine.backends.metadata.contract import DataContractMetadata
 from dve.core_engine.backends.types import StageSuccessful
 from dve.core_engine.backends.utilities import get_polars_type_from_annotation, stringify_model
+from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
 from dve.core_engine.message import FeedbackMessage
 from dve.core_engine.type_hints import URI, EntityLocations
 from dve.core_engine.validation import RowValidator, apply_row_validator_helper
@@ -55,6 +55,7 @@ def __call__(self, row: pd.Series):
         self.errors.extend(self.row_validator(row.to_dict())[1])  # type: ignore
         return row  # no op
 
+
 @duckdb_record_index
 @duckdb_write_parquet
 @duckdb_read_parquet
@@ -173,7 +174,7 @@ def apply_data_contract(
                         msg_count += len(msgs)
 
                 self.logger.info(f"Data contract found {msg_count} issues in {entity_name}")
-                
+
                 if not RECORD_INDEX_COLUMN_NAME in relation.columns:
                     relation = self.add_record_index(relation)
 

src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py

@@ -288,18 +288,28 @@ def duckdb_rel_to_dictionaries(
     while rows := entity.fetchmany(batch_size):
         yield from (dict(zip(cols, rw)) for rw in rows)
 
-def _add_duckdb_record_index(self, entity: DuckDBPyRelation) -> DuckDBPyRelation:
+
+def _add_duckdb_record_index(
+    self, entity: DuckDBPyRelation  # pylint: disable=W0613
+) -> DuckDBPyRelation:
+    """Add record index to duckdb relation"""
     if RECORD_INDEX_COLUMN_NAME in entity.columns:
         return entity
 
     return entity.select(f"*, row_number() OVER () as {RECORD_INDEX_COLUMN_NAME}")
 
-def _drop_duckdb_record_index(self, entity: DuckDBPyRelation) -> DuckDBPyRelation:
+
+def _drop_duckdb_record_index(
+    self, entity: DuckDBPyRelation  # pylint: disable=W0613
+) -> DuckDBPyRelation:
+    """Drop record index from duckdb relation"""
     if RECORD_INDEX_COLUMN_NAME not in entity.columns:
         return entity
     return entity.select(StarExpression(exclude=[RECORD_INDEX_COLUMN_NAME]))
 
+
 def duckdb_record_index(cls):
+    """Class decorator to add record index methods for duckdb implementations"""
     setattr(cls, "add_record_index", _add_duckdb_record_index)
     setattr(cls, "drop_record_index", _drop_duckdb_record_index)
     return cls

src/dve/core_engine/backends/implementations/duckdb/readers/csv.py

@@ -6,7 +6,13 @@
 
 import duckdb as ddb
 import polars as pl
-from duckdb import DuckDBPyConnection, DuckDBPyRelation, StarExpression, default_connection, read_csv
+from duckdb import (
+    DuckDBPyConnection,
+    DuckDBPyRelation,
+    StarExpression,
+    default_connection,
+    read_csv,
+)
 from pydantic import BaseModel
 
 from dve.core_engine.backends.base.reader import BaseFileReader, read_function
@@ -24,6 +30,7 @@
 from dve.core_engine.type_hints import URI, EntityName
 from dve.parser.file_handling import get_content_length
 
+
 @duckdb_record_index
 @duckdb_write_parquet
 class DuckDBCSVReader(BaseFileReader):
@@ -113,6 +120,7 @@ def read_to_relation(  # pylint: disable=unused-argument
         reader_options["columns"] = ddb_schema
         return self.add_record_index(read_csv(resource, **reader_options, parallel=False))
 
+
 @polars_record_index
 class PolarsToDuckDBCSVReader(DuckDBCSVReader):
     """
@@ -144,11 +152,14 @@ def read_to_relation(  # pylint: disable=unused-argument
             for fld in schema.__fields__.values()
         }
         reader_options["dtypes"] = polars_types
-        
 
         # there is a raise_if_empty arg for 0.18+. Future reference when upgrading. Makes L85
         # redundant
-        df = self.add_record_index(pl.scan_csv(resource, **reader_options).select(list(polars_types.keys())))  # type: ignore  # pylint: disable=W0612
+        df = self.add_record_index(  # pylint: disable=W0612
+            pl.scan_csv(resource, **reader_options).select(  # type: ignore
+                list(polars_types.keys())
+            )
+        )
 
         return ddb.sql("SELECT * FROM df")
 
@@ -192,7 +203,9 @@ def __init__(
     def read_to_relation(  # pylint: disable=unused-argument
         self, resource: URI, entity_name: EntityName, schema: type[BaseModel]
     ) -> DuckDBPyRelation:
-        entity: DuckDBPyRelation = super().read_to_relation(resource=resource, entity_name=entity_name, schema=schema)
+        entity: DuckDBPyRelation = super().read_to_relation(
+            resource=resource, entity_name=entity_name, schema=schema
+        )
         entity = entity.select(StarExpression(exclude=[RECORD_INDEX_COLUMN_NAME])).distinct()
         no_records = entity.shape[0]
 

src/dve/core_engine/backends/implementations/duckdb/readers/json.py

@@ -9,13 +9,14 @@
 
 from dve.core_engine.backends.base.reader import BaseFileReader, read_function
 from dve.core_engine.backends.implementations.duckdb.duckdb_helpers import (
+    duckdb_record_index,
     duckdb_write_parquet,
     get_duckdb_type_from_annotation,
 )
 from dve.core_engine.backends.implementations.duckdb.types import SQLType
-from dve.core_engine.backends.implementations.duckdb.duckdb_helpers import duckdb_record_index
 from dve.core_engine.type_hints import URI, EntityName
 
+
 @duckdb_record_index
 @duckdb_write_parquet
 class DuckDBJSONReader(BaseFileReader):
@@ -48,4 +49,6 @@ def read_to_relation(  # pylint: disable=unused-argument
             for fld in schema.__fields__.values()
         }
 
-        return self.add_record_index(read_json(resource, columns=ddb_schema, format=self._json_format))  # type: ignore
+        return self.add_record_index(
+            read_json(resource, columns=ddb_schema, format=self._json_format)  # type: ignore
+        )

src/dve/core_engine/backends/implementations/duckdb/readers/xml.py

@@ -11,9 +11,14 @@
 from dve.core_engine.backends.exceptions import MessageBearingError
 from dve.core_engine.backends.implementations.duckdb.duckdb_helpers import duckdb_write_parquet
 from dve.core_engine.backends.readers.xml import XMLStreamReader
-from dve.core_engine.backends.utilities import get_polars_type_from_annotation, polars_record_index, stringify_model
+from dve.core_engine.backends.utilities import (
+    get_polars_type_from_annotation,
+    polars_record_index,
+    stringify_model,
+)
 from dve.core_engine.type_hints import URI
 
+
 @polars_record_index
 @duckdb_write_parquet
 class DuckDBXMLStreamReader(XMLStreamReader):
@@ -39,7 +44,9 @@ def read_to_relation(self, resource: URI, entity_name: str, schema: type[BaseMod
             for fld in stringify_model(schema).__fields__.values()
         }
 
-        _lazy_frame = self.add_record_index(pl.LazyFrame(
-            data=self.read_to_py_iterator(resource, entity_name, schema), schema=polars_schema
-        ))
+        _lazy_frame = self.add_record_index(
+            pl.LazyFrame(
+                data=self.read_to_py_iterator(resource, entity_name, schema), schema=polars_schema
+            )
+        )
         return self.ddb_connection.sql("select * from _lazy_frame")

src/dve/core_engine/backends/implementations/duckdb/rules.py

@@ -57,6 +57,7 @@
 from dve.core_engine.templating import template_object
 from dve.core_engine.type_hints import Messages
 
+
 @duckdb_record_index
 @duckdb_write_parquet
 @duckdb_read_parquet

src/dve/core_engine/backends/implementations/spark/contract.py

@@ -17,7 +17,6 @@
     dump_processing_errors,
     get_feedback_errors_uri,
 )
-
 from dve.core_engine.backends.base.contract import BaseDataContract, reader_override
 from dve.core_engine.backends.base.utilities import generate_error_casting_entity_message
 from dve.core_engine.backends.exceptions import (
@@ -42,6 +41,7 @@
 COMPLEX_TYPES: set[type[DataType]] = {StructType, ArrayType, MapType}
 """Spark types indicating complex types."""
 
+
 @spark_record_index
 @spark_write_parquet
 @spark_read_parquet
@@ -86,6 +86,7 @@ def create_entity_from_py_iterator(
             schema=get_type_from_annotation(schema),
         )
 
+    # pylint: disable=R0915
     def apply_data_contract(
         self,
         working_dir: URI,