test: fix tests and linting and remove all default_connections

Author: georgeRobertson

src/dve/core_engine/backends/implementations/duckdb/readers/xml.py

@@ -5,7 +5,7 @@
 
 import duckdb
 import polars as pl
-from duckdb import DuckDBPyConnection, DuckDBPyRelation, default_connection
+from duckdb import DuckDBPyConnection, DuckDBPyRelation
 from pydantic import BaseModel
 
 from dve.core_engine.backends.base.reader import read_function

tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py

@@ -2,8 +2,9 @@
 from pathlib import Path
 from typing import Any, Dict, List, Tuple
 
+import duckdb
 import pytest
-from duckdb import DuckDBPyRelation, default_connection
+from duckdb import DuckDBPyRelation
 from duckdb.typing import DuckDBPyType
 
 from dve.core_engine.backends.implementations.duckdb.contract import DuckDBDataContract
@@ -34,7 +35,7 @@
 
 def test_duckdb_data_contract_csv(temp_csv_file):
     uri, _, _, mdl = temp_csv_file
-    connection = default_connection
+    connection = duckdb.connect(":memory:")
 
     contract_meta = json.dumps(
         {
@@ -106,7 +107,7 @@ def test_duckdb_data_contract_csv(temp_csv_file):
 
 def test_duckdb_data_contract_xml(temp_xml_file):
     uri, header_model, header_data, class_model, class_data = temp_xml_file
-    connection = default_connection
+    connection = duckdb.connect()
     contract_meta = json.dumps(
         {
             "contract": {
@@ -153,10 +154,10 @@ def test_duckdb_data_contract_xml(temp_xml_file):
     contract_dict = json.loads(contract_meta).get("contract")
     entities: Dict[str, DuckDBPyRelation] = {
         "test_header": DuckDBXMLStreamReader(
-            ddb_connection=connection, root_tag="root", record_tag="Header"
+            connection=connection, root_tag="root", record_tag="Header"
         ).read_to_relation(str(uri), "header", header_model),
         "test_class_info": DuckDBXMLStreamReader(
-            ddb_connection=connection, root_tag="root", record_tag="ClassData"
+            connection=connection, root_tag="root", record_tag="ClassData"
         ).read_to_relation(str(uri), "class_info", class_model),
     }
     entity_locations: dict[str, URI] = {}
@@ -218,7 +219,7 @@ def test_ddb_data_contract_read_and_write_basic_parquet(
 ):
     # can we read in a stringified parquet and run the data contract on it?
     # basic file - simple data structures
-    connection = default_connection
+    connection = duckdb.connect(":memory:")
     parquet_uri, contract_meta, _ = simple_all_string_parquet
     data_contract = DuckDBDataContract(connection)
     # check can read
@@ -279,7 +280,7 @@ def test_ddb_data_contract_read_nested_parquet(nested_all_string_parquet):
     # can we read in a stringified parquet and run the data contract on it?
     # more complex file - nested, arrays of structs
     parquet_uri, contract_meta, _ = nested_all_string_parquet
-    connection = default_connection
+    connection = duckdb.connect()
     data_contract = DuckDBDataContract(connection)
     # check can read
     entity = data_contract.read_parquet(path=parquet_uri)
@@ -337,7 +338,7 @@ def test_ddb_data_contract_read_nested_parquet(nested_all_string_parquet):
 def test_duckdb_data_contract_custom_error_details(nested_all_string_parquet_w_errors,
                                                   nested_parquet_custom_dc_err_details):
     parquet_uri, contract_meta, _ = nested_all_string_parquet_w_errors
-    connection = default_connection
+    connection = duckdb.connect(":memory:")
     data_contract = DuckDBDataContract(connection)
 
     entity = data_contract.read_parquet(path=parquet_uri)

tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_rules.py

@@ -4,6 +4,7 @@
 from pathlib import Path
 from typing import Iterator, List, Optional, Set, Tuple, Type
 
+import duckdb
 import numpy as np
 import polars as pl
 import pytest
@@ -12,7 +13,6 @@
     ConstantExpression,
     DuckDBPyRelation,
     StarExpression,
-    default_connection,
 )
 
 from dve.core_engine.backends.base.core import EntityManager
@@ -49,7 +49,7 @@
     simple_typecast_parquet,
 )
 
-DUCKDB_STEP_BACKEND = DuckDBStepImplementations(default_connection)
+DUCKDB_STEP_BACKEND = DuckDBStepImplementations(duckdb.connect())
 """The backend for the duckdb steps."""
 
 

tests/test_core_engine/test_backends/test_readers/test_ddb_csv.py

@@ -2,9 +2,10 @@
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
+import duckdb
 import polars as pl
 import pytest
-from duckdb import DuckDBPyRelation, default_connection
+from duckdb import DuckDBPyRelation
 from pydantic import BaseModel
 
 from dve.core_engine.backends.exceptions import EmptyFileError, MessageBearingError
@@ -71,7 +72,7 @@ def temp_empty_csv_file(temp_dir: Path):
 
 def test_ddb_csv_reader_all_str(temp_csv_file):
     uri, header, data, mdl = temp_csv_file
-    reader = DuckDBCSVReader(header=True, delim=",", connection=default_connection)
+    reader = DuckDBCSVReader(header=True, delim=",", connection=duckdb.connect())
     rel: DuckDBPyRelation = reader.read_to_entity_type(
         DuckDBPyRelation, str(uri), "test", stringify_model(mdl)
     )
@@ -84,7 +85,7 @@ def test_ddb_csv_reader_all_str(temp_csv_file):
 
 def test_ddb_csv_reader_cast(temp_csv_file):
     uri, header, data, mdl = temp_csv_file
-    reader = DuckDBCSVReader(header=True, delim=",", connection=default_connection)
+    reader = DuckDBCSVReader(header=True, delim=",", connection=duckdb.connect())
     rel: DuckDBPyRelation = reader.read_to_entity_type(DuckDBPyRelation, str(uri), "test", mdl)
     expected_dtypes = {**{
         fld.name: str(get_duckdb_type_from_annotation(fld.annotation))
@@ -98,7 +99,7 @@ def test_ddb_csv_reader_cast(temp_csv_file):
 
 def test_ddb_csv_write_parquet(temp_csv_file):
     uri, header, data, mdl = temp_csv_file
-    reader = DuckDBCSVReader(header=True, delim=",", connection=default_connection)
+    reader = DuckDBCSVReader(header=True, delim=",", connection=duckdb.connect())
     rel: DuckDBPyRelation = reader.read_to_entity_type(
         DuckDBPyRelation, str(uri), "test", stringify_model(mdl)
     )
@@ -110,7 +111,7 @@ def test_ddb_csv_write_parquet(temp_csv_file):
 
 def test_ddb_csv_read_empty_file(temp_empty_csv_file):
     uri, mdl = temp_empty_csv_file
-    reader = DuckDBCSVReader(header=True, delim=",", connection=default_connection)
+    reader = DuckDBCSVReader(header=True, delim=",", connection=duckdb.connect())
 
     with pytest.raises(EmptyFileError):
         reader.read_to_relation(str(uri), "test", mdl)
@@ -119,7 +120,7 @@ def test_ddb_csv_read_empty_file(temp_empty_csv_file):
 def test_polars_to_ddb_csv_reader(temp_csv_file):
     uri, header, data, mdl = temp_csv_file
     reader = PolarsToDuckDBCSVReader(
-        header=True, delim=",", quotechar='"', connection=default_connection
+        header=True, delim=",", quotechar='"', connection=duckdb.connect()
     )
     entity = reader.read_to_relation(str(uri), "test", mdl)
 
@@ -141,7 +142,7 @@ def test_ddb_csv_repeating_header_reader_non_duplicate(temp_dir):
     file_uri = temp_dir.joinpath("test_header.csv")
 
     reader = DuckDBCSVRepeatingHeaderReader(
-        header=True, delim=",", quotechar='"', connection=default_connection
+        header=True, delim=",", quotechar='"', connection=duckdb.connect()
     )
     entity = reader.read_to_relation(str(file_uri), "test", SimpleHeaderModel)
 
@@ -162,7 +163,7 @@ def test_ddb_csv_repeating_header_reader_with_more_than_one_set_of_distinct_valu
 
     file_uri = temp_dir.joinpath("test_header.csv")
     reader = DuckDBCSVRepeatingHeaderReader(
-        header=True, delim=",", quotechar='"', connection=default_connection
+        header=True, delim=",", quotechar='"', connection=duckdb.connect()
     )
 
     with pytest.raises(MessageBearingError):
@@ -182,7 +183,7 @@ def test_DuckDBCSVReader_with_null_empty_strings(temp_dir):
         header=True,
         delim=",",
         quotechar='"',
-        connection=default_connection,
+        connection=duckdb.connect(),
         null_empty_strings=True,
     )
 
@@ -207,7 +208,7 @@ def test_DuckDBCSVRepeatingHeaderReader_with_null_empty_strings(temp_dir):
         header=True,
         delim=",",
         quotechar='"',
-        connection=default_connection,
+        connection=duckdb.connect(),
         null_empty_strings=True,
     )
 
@@ -230,7 +231,7 @@ def test_PolarsToDuckDBCSVReader_with_null_empty_strings(temp_dir):
         header=True,
         delim=",",
         quotechar='"',
-        connection=default_connection,
+        connection=duckdb.connect(),
         null_empty_strings=True,
     )
 

tests/test_core_engine/test_backends/test_readers/test_ddb_json.py

@@ -4,8 +4,9 @@
 from tempfile import TemporaryDirectory
 from typing import List
 
+import duckdb
 import pytest
-from duckdb import DuckDBPyRelation, default_connection
+from duckdb import DuckDBPyRelation
 from pydantic import BaseModel
 
 from dve.core_engine.backends.implementations.duckdb.duckdb_helpers import (
@@ -87,17 +88,19 @@ def test_ddb_json_write_parquet(temp_json_file):
     )
     target_loc: Path = uri.parent.joinpath("test_parquet.parquet").as_posix()
     reader.write_parquet(rel, target_loc)
-    parquet_rel = default_connection.read_parquet(target_loc)
-    assert parquet_rel.df().to_dict(orient="records") == rel.df().to_dict(orient="records")
+    with duckdb.connect() as cnn:
+        parquet_rel = cnn.read_parquet(target_loc)
+        assert parquet_rel.df().to_dict(orient="records") == rel.df().to_dict(orient="records")
 
 def test_ddb_json_write_parquet_py_iterator(temp_json_file):
     uri, _, mdl = temp_json_file
     reader = DuckDBJSONReader()
+    conn = duckdb.connect()
     data = list(reader.read_to_py_iterator(uri.as_posix(), "test", stringify_model(mdl)))
     target_loc: Path = uri.parent.joinpath("test_parquet.parquet").as_posix()
-    reader.write_parquet(default_connection.query("select dta.* from (select unnest($data) as dta)",
+    reader.write_parquet(conn.query("select dta.* from (select unnest($data) as dta)",
                                                   params={"data": data}),
                          target_loc)
-    parquet_data = sorted(default_connection.read_parquet(target_loc).pl().iter_rows(named=True),
+    parquet_data = sorted(conn.read_parquet(target_loc).pl().iter_rows(named=True),
                           key= lambda x: x.get("bigint_field"))
     assert parquet_data == list(data)

tests/test_core_engine/test_backends/test_readers/test_ddb_xml.py

@@ -3,8 +3,9 @@
 from tempfile import TemporaryDirectory
 from typing import Dict, List
 
+import duckdb
 import pytest
-from duckdb import DuckDBPyRelation, default_connection
+from duckdb import DuckDBPyRelation
 from lxml import etree as ET
 from pydantic import BaseModel
 
@@ -65,7 +66,7 @@ class ClassDataModel(BaseModel):
 
 def test_ddb_xml_reader_all_str(temp_xml_file):
     uri, header_model, header_data, class_data_model, class_data = temp_xml_file
-    ddb_conn = default_connection
+    ddb_conn = duckdb.connect()
     header_reader = DuckDBXMLStreamReader(
         connection=ddb_conn, root_tag="root", record_tag="Header"
     )
@@ -88,7 +89,7 @@ def test_ddb_xml_reader_all_str(temp_xml_file):
 
 def test_ddb_xml_reader_write_parquet(temp_xml_file):
     uri, header_model, header_data, class_data_model, class_data = temp_xml_file
-    ddb_conn = default_connection
+    ddb_conn = duckdb.connect()
     header_reader = DuckDBXMLStreamReader(
         connection=ddb_conn, root_tag="root", record_tag="Header"
     )