File tree Expand file tree Collapse file tree
src/dve/core_engine/backends/implementations/duckdb/readers Expand file tree Collapse file tree Original file line number Diff line number Diff line change 44from collections .abc import Iterator
55from typing import Any , Optional
66
7- from duckdb import DuckDBPyRelation , read_json
7+ import duckdb
8+ from duckdb import DuckDBPyConnection , DuckDBPyRelation
89from pydantic import BaseModel
910
1011from dve .core_engine .backends .base .reader import BaseFileReader , read_function
@@ -26,9 +27,11 @@ def __init__(
2627 self ,
2728 * ,
2829 json_format : Optional [str ] = "array" ,
30+ connection : Optional [DuckDBPyConnection ] = None ,
2931 ** _ ,
3032 ):
3133 self ._json_format = json_format
34+ self ._connection = duckdb .connect (":memory:" ) if not connection else connection
3235
3336 super ().__init__ ()
3437
@@ -50,5 +53,5 @@ def read_to_relation( # pylint: disable=unused-argument
5053 }
5154
5255 return self .add_record_index (
53- read_json (resource , columns = ddb_schema , format = self ._json_format ) # type: ignore
56+ self . _connection . read_json (resource , columns = ddb_schema , format = self ._json_format ) # type: ignore
5457 )
You can’t perform that action at this time.
0 commit comments