style: address review comments and linting issues

Author: stevenhsd

src/dve/core_engine/backends/implementations/duckdb/reference_data.py

@@ -15,9 +15,7 @@
 
 # pylint: disable=too-few-public-methods
 class DuckDBRefDataLoader(BaseRefDataLoader[DuckDBPyRelation]):
-    """A reference data loader using already existing DuckDB tables.
-    reference_entity_config and dataset_config_uri (if config uses relative paths)
-    should be supplied using setter methods for the dataset being processed before running."""
+    """A reference data loader using already existing DuckDB tables."""
 
     def __init__(
         self,

src/dve/core_engine/backends/implementations/spark/backend.py

@@ -12,7 +12,6 @@
 from dve.core_engine.backends.implementations.spark.rules import SparkStepImplementations
 from dve.core_engine.backends.implementations.spark.spark_helpers import get_type_from_annotation
 from dve.core_engine.backends.implementations.spark.types import SparkEntities
-from dve.core_engine.backends.types import EntityType
 from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
 from dve.core_engine.loggers import get_child_logger, get_logger
 from dve.core_engine.models import SubmissionInfo
@@ -64,7 +63,7 @@ def load_reference_data(
         reference_data_loader = SparkRefDataLoader(
             spark=self.spark_session,
             reference_data_config=reference_entity_config,
-            dataset_config_uri=self.dataset_config_uri, # type: ignore
+            dataset_config_uri=self.dataset_config_uri,  # type: ignore
         )
         if sub_info_entity is not None:
             reference_data_loader.entity_cache["dve_submission_info"] = sub_info_entity

src/dve/core_engine/backends/implementations/spark/reference_data.py

@@ -15,9 +15,7 @@
 
 # pylint: disable=too-few-public-methods
 class SparkRefDataLoader(BaseRefDataLoader[DataFrame]):
-    """A reference data loader using already existing Apache Spark Tables.
-    reference_entity_config and dataset_config_uri (if config uses relative paths)
-    should be supplied using setter methods for the dataset being processed before running."""
+    """A reference data loader using already existing Apache Spark Tables."""
 
     def __init__(
         self,

src/dve/pipeline/duckdb_pipeline.py

@@ -6,7 +6,7 @@
 from duckdb import DuckDBPyConnection, DuckDBPyRelation
 
 import dve.parser.file_handling as fh
-from dve.core_engine.backends.base.reference_data import BaseRefDataLoader, ReferenceConfig
+from dve.core_engine.backends.base.reference_data import ReferenceConfig
 from dve.core_engine.backends.implementations.duckdb.auditing import DDBAuditingManager
 from dve.core_engine.backends.implementations.duckdb.contract import DuckDBDataContract
 from dve.core_engine.backends.implementations.duckdb.duckdb_helpers import duckdb_get_entity_count
@@ -47,13 +47,13 @@ def __init__(
             logger,
         )
 
-    def get_reference_data_loader(
+    def init_reference_data_loader(
         self, reference_data_config: dict[str, ReferenceConfig], **kwargs
     ) -> DuckDBRefDataLoader:
         return DuckDBRefDataLoader(
             connection=self._connection,
             reference_data_config=reference_data_config,
-            dataset_config_uri=fh.get_parent(self._rules_path), # type: ignore
+            dataset_config_uri=fh.get_parent(self._rules_path),  # type: ignore
             **kwargs
         )
 

src/dve/pipeline/pipeline.py

@@ -68,7 +68,6 @@ def __init__(
         self._submitted_files_path = submitted_files_path
         self._processed_files_path = processed_files_path
         self._rules_path = rules_path
-        self._reference_data_loader = None
         self._job_run_id = job_run_id
         self._audit_tables = audit_tables
         self._data_contract = data_contract
@@ -113,7 +112,7 @@ def get_entity_count(entity: EntityType) -> int:
         """Get a row count of an entity stored as parquet"""
         raise NotImplementedError()
 
-    def get_reference_data_loader(
+    def init_reference_data_loader(
         self, reference_data_config: dict[EntityName, ReferenceConfig], **kwargs
     ) -> BaseRefDataLoader:
         """Get reference data loader if required for business rules"""
@@ -558,7 +557,9 @@ def apply_business_rules(  # pylint: disable=R0914
             self._processed_files_path, submission_info.submission_id
         )
         ref_data = config.get_reference_data_config()
-        reference_data: BaseRefDataLoader = self.get_reference_data_loader(reference_data_config=ref_data)
+        reference_data: BaseRefDataLoader = self.init_reference_data_loader(
+            reference_data_config=ref_data
+        )
         rules = config.get_rule_metadata()
         entities = {}
         contract = fh.joinuri(

src/dve/pipeline/spark_pipeline.py

@@ -7,7 +7,7 @@
 from pyspark.sql import DataFrame, SparkSession
 
 import dve.parser.file_handling as fh
-from dve.core_engine.backends.base.reference_data import BaseRefDataLoader, ReferenceConfig
+from dve.core_engine.backends.base.reference_data import ReferenceConfig
 from dve.core_engine.backends.implementations.spark.auditing import SparkAuditingManager
 from dve.core_engine.backends.implementations.spark.contract import SparkDataContract
 from dve.core_engine.backends.implementations.spark.reference_data import SparkRefDataLoader
@@ -49,13 +49,13 @@ def __init__(
             logger,
         )
 
-    def get_reference_data_loader(
+    def init_reference_data_loader(
         self, reference_data_config: dict[str, ReferenceConfig], **kwargs
     ) -> SparkRefDataLoader:
         return SparkRefDataLoader(
             spark=self._spark,
             reference_data_config=reference_data_config,
-            dataset_config_uri=fh.get_parent(self._rules_path), # type: ignore
+            dataset_config_uri=fh.get_parent(self._rules_path),  # type: ignore
             **kwargs
         )