fix: ddb xml reader connection args consistent with other ddb readers

Author: stevenhsd

src/dve/core_engine/backends/implementations/duckdb/readers/xml.py

@@ -3,6 +3,7 @@
 
 from typing import Optional
 
+import duckdb
 import polars as pl
 from duckdb import DuckDBPyConnection, DuckDBPyRelation, default_connection
 from pydantic import BaseModel
@@ -24,8 +25,8 @@
 class DuckDBXMLStreamReader(XMLStreamReader):
     """A reader for XML files"""
 
-    def __init__(self, *, ddb_connection: Optional[DuckDBPyConnection] = None, **kwargs):
-        self.ddb_connection = ddb_connection if ddb_connection else default_connection
+    def __init__(self, *, connection: Optional[DuckDBPyConnection] = None, **kwargs):
+        self._connection = connection if connection else duckdb.connect(":memory:")
         super().__init__(**kwargs)
 
     @read_function(DuckDBPyRelation)
@@ -49,4 +50,4 @@ def read_to_relation(self, resource: URI, entity_name: str, schema: type[BaseMod
                 data=self.read_to_py_iterator(resource, entity_name, schema), schema=polars_schema
             )
         )
-        return self.ddb_connection.sql("select * from _lazy_frame")
+        return self._connection.sql("select * from _lazy_frame")

tests/test_core_engine/test_backends/test_readers/test_ddb_xml.py

@@ -67,10 +67,10 @@ def test_ddb_xml_reader_all_str(temp_xml_file):
     uri, header_model, header_data, class_data_model, class_data = temp_xml_file
     ddb_conn = default_connection
     header_reader = DuckDBXMLStreamReader(
-        ddb_connection=ddb_conn, root_tag="root", record_tag="Header"
+        connection=ddb_conn, root_tag="root", record_tag="Header"
     )
     class_reader = DuckDBXMLStreamReader(
-        ddb_connection=ddb_conn, root_tag="root", record_tag="ClassData"
+        connection=ddb_conn, root_tag="root", record_tag="ClassData"
     )
     header_rel: DuckDBPyRelation = header_reader.read_to_relation(
         uri.as_uri(), "header", header_model
@@ -90,10 +90,10 @@ def test_ddb_xml_reader_write_parquet(temp_xml_file):
     uri, header_model, header_data, class_data_model, class_data = temp_xml_file
     ddb_conn = default_connection
     header_reader = DuckDBXMLStreamReader(
-        ddb_connection=ddb_conn, root_tag="root", record_tag="Header"
+        connection=ddb_conn, root_tag="root", record_tag="Header"
     )
     class_reader = DuckDBXMLStreamReader(
-        ddb_connection=ddb_conn, root_tag="root", record_tag="ClassData"
+        connection=ddb_conn, root_tag="root", record_tag="ClassData"
     )
     header_rel: DuckDBPyRelation = header_reader.read_to_relation(
         uri.as_uri(), "header", header_model
@@ -105,10 +105,10 @@ def test_ddb_xml_reader_write_parquet(temp_xml_file):
     target_class_loc: Path = uri.parent.joinpath("class_parquet.parquet").as_posix()
     header_reader.write_parquet(entity=header_rel, target_location=target_header_loc)
     class_reader.write_parquet(entity=class_rel, target_location=target_class_loc)
-    header_parquet_rel: DuckDBPyRelation = header_reader.ddb_connection.read_parquet(
+    header_parquet_rel: DuckDBPyRelation = header_reader._connection.read_parquet(
         target_header_loc
     )
-    class_parquet_rel: DuckDBPyRelation = class_reader.ddb_connection.read_parquet(target_class_loc)
+    class_parquet_rel: DuckDBPyRelation = class_reader._connection.read_parquet(target_class_loc)
     assert header_parquet_rel.df().to_dict(orient="records") == header_rel.df().to_dict(
         orient="records"
     )

tests/test_pipeline/test_foundry_ddb_pipeline.py

@@ -31,7 +31,7 @@
 @pytest.fixture(scope="function")
 def prep_multithreading_test():
     sub_details: dict[str, tuple[DuckDBPyConnection, str, DDBAuditingManager]] = {}
-    for idx in range(1, 4):
+    for idx in range(1, 10):
         db = f"dve_{uuid4().hex}"
         tmp_dir = tempfile.mkdtemp(prefix="ddb_foundry_testing")
         db_file = Path(tmp_dir, db + ".duckdb")