From 3d25db0a0888ae24260b7f12c22912efff691058 Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Thu, 29 Oct 2026 11:09:44 +0000 Subject: [PATCH] Read the Arrow layouts the engine now converts, and check run ends and list views --- src/arrow/arrow_importer.cpp | 122 +++++++------ .../arrow/scan/test_arrow_scan_row_ranges.py | 80 ++++----- .../arrow/scan/test_python_arrow_scan_core.py | 169 +++++++++++++----- 3 files changed, 230 insertions(+), 141 deletions(-) diff --git a/src/arrow/arrow_importer.cpp b/src/arrow/arrow_importer.cpp index 70976bb2..7067bc6f 100644 --- a/src/arrow/arrow_importer.cpp +++ b/src/arrow/arrow_importer.cpp @@ -6,6 +6,8 @@ #include "duckdb/common/vector/flat_vector.hpp" #include "duckdb/function/table/arrow/arrow_type_info.hpp" +#include + namespace duckdb { namespace { @@ -33,12 +35,19 @@ bool ValidRange(int64_t offset, int64_t length) { return offset >= 0 && length >= 0 && offset <= NumericLimits::Maximum() - length; } -//! Checks the arrays below a requested column as the batch is checked, and refuses an offset the engine would drop: its -//! struct conversion hands its children only the struct's own offset and its list conversion none, so an offset met -//! further up would be lost there and the rows below read from the wrong places. -//! `received` is the offset the engine passes the array as its parent's, and `in_list` whether a list above it makes -//! the engine ignore parent offsets. -void CheckNested(const ArrowArray &array, const ArrowType &type, idx_t column, int64_t received, bool in_list) { +// Arrow buffers need not be aligned to their value type, so a scalar is copied out instead of read through a pointer. +template +T LoadAt(const void *buffer, int64_t index) { + if (index < 0 || index > NumericLimits::Maximum() / static_cast(sizeof(T))) { + throw InvalidInputException("an Arrow batch addresses row %d of a buffer, beyond what a buffer can hold", + index); + } + T value; + memcpy(&value, static_cast(buffer) + index * static_cast(sizeof(T)), sizeof(T)); + return value; +} + +void CheckNested(const ArrowArray &array, const ArrowType &type, idx_t column) { if (!ValidRange(array.offset, array.length)) { throw InvalidInputException("an Arrow batch carried column %d with a nested array of offset %d and length %d, " "which do not describe a range of rows", @@ -89,8 +98,7 @@ void CheckNested(const ArrowArray &array, const ArrowType &type, idx_t column, i "an Arrow batch carried column %d with a dictionary-encoded array whose dictionary is released already", column); } - // The dictionary is converted on its own, from its start. - CheckNested(*array.dictionary, type.GetDictionary(), column, 0, false); + CheckNested(*array.dictionary, type.GetDictionary(), column); return; case ArrowArrayPhysicalType::RUN_END_ENCODED: { auto &info = type.GetTypeInfo(); @@ -101,9 +109,39 @@ void CheckNested(const ArrowArray &array, const ArrowType &type, idx_t column, i "and %d values", column, run_ends.length, values.length); } - // The run ends and the values are converted on their own, from their start. - CheckNested(run_ends, info.GetChild(0), column, 0, false); - CheckNested(values, info.GetChild(1), column, 0, false); + CheckNested(run_ends, info.GetChild(0), column); + CheckNested(values, info.GetChild(1), column); + // The engine decodes runs until they are exhausted and leaves the rows after the last run end unwritten. + int64_t last_end = 0; + if (run_ends.length > 0) { + if (run_ends.n_buffers < 2 || !run_ends.buffers || !run_ends.buffers[1]) { + throw InvalidInputException( + "an Arrow batch carried column %d with a run end encoded array whose run ends have no values", + column); + } + const auto index = run_ends.offset + run_ends.length - 1; + switch (info.GetChild(0).GetDuckType().id()) { + case LogicalTypeId::SMALLINT: + last_end = LoadAt(run_ends.buffers[1], index); + break; + case LogicalTypeId::INTEGER: + last_end = LoadAt(run_ends.buffers[1], index); + break; + case LogicalTypeId::BIGINT: + last_end = LoadAt(run_ends.buffers[1], index); + break; + default: + throw InvalidInputException( + "an Arrow batch carried column %d with a run end encoded array whose run ends " + "are %s, not SMALLINT, INTEGER or BIGINT", + column, info.GetChild(0).GetDuckType().ToString()); + } + } + if (array.length > 0 && last_end < array.offset + array.length) { + throw InvalidInputException("an Arrow batch carried column %d with a run end encoded array whose runs end " + "at %d before its %d rows", + column, last_end, array.offset + array.length); + } return; } default: @@ -115,15 +153,6 @@ void CheckNested(const ArrowArray &array, const ArrowType &type, idx_t column, i case LogicalTypeId::STRUCT: case LogicalTypeId::TUPLE: case LogicalTypeId::UNION: { - const bool is_union = type.GetDuckType().id() == LogicalTypeId::UNION; - // A union adds its own offset to the one it received, a struct passes on only its own. - if ((in_list && array.offset != 0) || (!is_union && !in_list && received != 0)) { - throw InvalidInputException( - "an Arrow batch carried column %d with a struct or union below an offset the conversion would drop, " - "which would read the wrong rows", - column); - } - const auto passed = is_union ? received + array.offset : array.offset; auto &info = type.GetTypeInfo(); // The engine converts every child the array has into vectors sized by the type. if (array.n_children != NumericCast(info.ChildCount())) { @@ -134,7 +163,7 @@ void CheckNested(const ArrowArray &array, const ArrowType &type, idx_t column, i for (idx_t i = 0; i < info.ChildCount(); i++) { auto &child = child_of(i); check_covers(child, 1); - CheckNested(child, info.GetChild(i), column, passed, in_list); + CheckNested(child, info.GetChild(i), column); } return; } @@ -144,14 +173,12 @@ void CheckNested(const ArrowArray &array, const ArrowType &type, idx_t column, i auto &child = child_of(0); auto value_at = [&](idx_t buffer, int64_t index) -> int64_t { if (info.GetSizeType() == ArrowVariableSizeType::SUPER_SIZE) { - return static_cast(array.buffers[buffer])[index]; + return LoadAt(array.buffers[buffer], index); } - return static_cast(array.buffers[buffer])[index]; + return LoadAt(array.buffers[buffer], index); }; - // When the lists address no child values and start at offset 0, the engine reads the child as plain values, - // which a dictionary-encoded child is not; a list view counts as starting at 0 whenever its lists are empty. - const bool dictionary_child = info.GetChild().GetPhysicalType() == ArrowArrayPhysicalType::DICTIONARY_ENCODED; - bool only_empty_from_zero = false; + // The child's own range is checked first, so the offset checks below compare against a valid length. + CheckNested(child, info.GetChild(), column); if (!info.IsView() && array.length > 0) { if (array.n_buffers < 2 || !array.buffers || !array.buffers[1]) { throw InvalidInputException("an Arrow batch carried column %d with a list that has no offsets", column); @@ -165,36 +192,30 @@ void CheckNested(const ArrowArray &array, const ArrowType &type, idx_t column, i "fit its child of %d rows", column, first, last, child.length); } - only_empty_from_zero = last == 0; - } else if (info.IsView() && array.length > 0 && dictionary_child) { - if (array.n_buffers < 3 || !array.buffers || !array.buffers[2]) { - throw InvalidInputException("an Arrow batch carried column %d with a list view that has no sizes", - column); + } else if (info.IsView() && array.length > 0) { + if (array.n_buffers < 3 || !array.buffers || !array.buffers[1] || !array.buffers[2]) { + throw InvalidInputException( + "an Arrow batch carried column %d with a list view that has no offsets or sizes", column); } - only_empty_from_zero = true; - for (int64_t row = array.offset; row < array.offset + array.length && only_empty_from_zero; row++) { - only_empty_from_zero = value_at(2, row) == 0; + // The engine reads the child from the lowest offset to the highest end of every list with a size. + for (int64_t row = array.offset; row < array.offset + array.length; row++) { + const auto size = value_at(2, row); + const auto start = value_at(1, row); + if (size != 0 && (size < 0 || start < 0 || start > child.length - size)) { + throw InvalidInputException( + "an Arrow batch carried column %d with a list view whose list at row %d " + "reaches past its child of %d rows", + column, row - array.offset, child.length); + } } } - if (dictionary_child && only_empty_from_zero) { - throw InvalidInputException("an Arrow batch carried column %d with only empty lists over a " - "dictionary-encoded child, which the conversion cannot read", - column); - } - CheckNested(child, info.GetChild(), column, 0, true); return; } case LogicalTypeId::ARRAY: { auto &child = child_of(0); auto &info = type.GetTypeInfo(); - // The engine's fixed size list conversion reads a run end encoded child as plain values. - if (array.length > 0 && info.GetChild().GetPhysicalType() == ArrowArrayPhysicalType::RUN_END_ENCODED) { - throw InvalidInputException("an Arrow batch carried column %d with a run end encoded array inside a fixed " - "size list, which the conversion cannot read", - column); - } + CheckNested(child, info.GetChild(), column); check_covers(child, info.FixedSize()); - CheckNested(child, info.GetChild(), column, 0, true); return; } default: @@ -287,7 +308,7 @@ shared_ptr ArrowBatchImporter::Adopt(ArrowArray &incoming) co throw InvalidInputException("an Arrow batch has a non-zero null count but no validity bitmap"); } for (auto column : columns) { - CheckNested(*incoming.children[column], *arrow_columns.at(column), column, 0, false); + CheckNested(*incoming.children[column], *arrow_columns.at(column), column); } auto owned = make_shared_ptr(); owned->arrow_array = incoming; @@ -336,8 +357,7 @@ bool ArrowBatchImporter::Next(DataChunk &output) { D_ASSERT(output.ColumnCount() == columns.size()); auto &array = batch->arrow_array; const auto rows = MinValue(STANDARD_VECTOR_SIZE, end - offset); - // The batch offset is folded into the row position, which the engine hands on to nested children, rather than - // passed as the parent offset, which its struct conversion does not hand on. + // A batch offset shifts the rows the way the start of a row range does, so both go into the row position. const auto position = NumericCast(array.offset) + offset; output.SetChildCardinality(rows); auto batch_validity = BatchValidity(array, offset, rows); diff --git a/tests/fast/arrow/scan/test_arrow_scan_row_ranges.py b/tests/fast/arrow/scan/test_arrow_scan_row_ranges.py index 736b9449..fbfd75a8 100644 --- a/tests/fast/arrow/scan/test_arrow_scan_row_ranges.py +++ b/tests/fast/arrow/scan/test_arrow_scan_row_ranges.py @@ -469,6 +469,31 @@ def _sliced_table(): return table, [(k,) for k in range(3, SPLIT_ROWS)] +def _struct_of_struct_table(rows): + k = arange(rows) + inner = pa.StructArray.from_arrays([pa.array(k), pa.array(k * 2)], names=["x", "y"]) + return pa.table({"k": k, "s": pa.StructArray.from_arrays([inner, pa.array(k)], names=["a", "z"])}) + + +def _differently_chunked_columns(): + table = _struct_of_struct_table(10) + return pa.table( + { + "k": pa.chunked_array([table["k"].chunks[0].slice(0, 4), table["k"].chunks[0].slice(4)]), + "s": table["s"], + } + ) + + +def _struct_of_struct_source(make_table, wrap=None): + def build(): + table = make_table() + expected = [(row["k"], row["s"]) for row in table.to_pylist()] + return (wrap(table) if wrap else table), expected + + return build + + LAYOUTS = { "struct_of_struct_with_null_rows": _column_layout(_struct_of_struct), "sparse_union_of_struct": _column_layout(_sparse_union_of_struct), @@ -487,37 +512,15 @@ def _sliced_table(): "plain_array_with_null_rows": _plain_array_source(0), "plain_array_with_null_rows_and_an_offset": _plain_array_source(5), "one_chunk_table_slice": _sliced_table, -} - -STRUCT_OFFSET_REFUSAL = ( - "an Arrow batch carried column 1 with a struct or union below an offset the conversion would drop, which would " - "read the wrong rows" -) - - -def _struct_of_struct_table(rows): - k = arange(rows) - inner = pa.StructArray.from_arrays([pa.array(k), pa.array(k * 2)], names=["x", "y"]) - return pa.table({"k": k, "s": pa.StructArray.from_arrays([inner, pa.array(k)], names=["a", "z"])}) - - -def _differently_chunked_columns(): - table = _struct_of_struct_table(10) - return pa.table( - { - "k": pa.chunked_array([table["k"].chunks[0].slice(0, 4), table["k"].chunks[0].slice(4)]), - "s": table["s"], - } - ) - - -REFUSED = { - "table_slice": lambda: _struct_of_struct_table(LARGE_BATCH).slice(3), - "table_slice_in_batches": lambda: pa.Table.from_batches( - _struct_of_struct_table(LARGE_BATCH).slice(3).to_batches(max_chunksize=65_536) + "struct_of_struct_table_slice": _struct_of_struct_source(lambda: _struct_of_struct_table(LARGE_BATCH).slice(3)), + "struct_of_struct_table_slice_in_batches": _struct_of_struct_source( + lambda: pa.Table.from_batches(_struct_of_struct_table(LARGE_BATCH).slice(3).to_batches(max_chunksize=65_536)) ), - "dataset": lambda: pytest.importorskip("pyarrow.dataset").dataset(_struct_of_struct_table(LARGE_BATCH)), - "differently_chunked_columns": _differently_chunked_columns, + "struct_of_struct_dataset": _struct_of_struct_source( + lambda: _struct_of_struct_table(LARGE_BATCH), + lambda table: pytest.importorskip("pyarrow.dataset").dataset(table), + ), + "struct_of_struct_differently_chunked_columns": _struct_of_struct_source(_differently_chunked_columns), } @@ -531,16 +534,6 @@ def test_nested_layouts_read_the_same_whole_and_split(self, layout, threads): con.register("src", obj) assert con.execute("SELECT * FROM src").fetchall() == expected - @pytest.mark.timeout(60) - @pytest.mark.parametrize("threads", [1, THREADS]) - @pytest.mark.parametrize("layout", list(REFUSED)) - def test_a_refused_layout_is_refused_the_same_when_split(self, layout, threads): - obj = REFUSED[layout]() - with duckdb.connect(config={"threads": threads}) as con: - con.register("src", obj) - with pytest.raises(duckdb.InvalidInputException, match=re.escape(STRUCT_OFFSET_REFUSAL)): - con.execute("SELECT * FROM src").fetchall() - RELEASE_TABLE_ROWS = 3 * LARGE_BATCH @@ -899,7 +892,8 @@ def hold_gil(x): def _list_of_dictionary(empty, valued): - child = pa.DictionaryArray.from_arrays(pa.array([0, 1] * valued, pa.int32()), pa.array(["x", "y"])) + indices = pa.array(np.arange(2 * valued, dtype=np.int32) % 3) + child = pa.DictionaryArray.from_arrays(indices, pa.array(["x", "y", "z"])) offsets = [0] * (empty + 1) + [2 * (i + 1) for i in range(valued)] return pa.ListArray.from_arrays(pa.array(offsets, pa.int32()), child) @@ -925,6 +919,10 @@ def test_a_struct_dictionary_without_a_null_value_reads_its_values(self, indices def test_a_list_of_dictionaries_after_a_vector_of_empty_lists_reads_its_lists(self, empty, valued, threads): column = _list_of_dictionary(empty, valued) expected = [(len(column), pc.sum(pc.list_value_length(column)).as_py())] + first_valued = [(value,) for value in column.slice(empty - 2, 5).to_pylist()] + last = [(value,) for value in column.slice(len(column) - 3).to_pylist()] with duckdb.connect(config={"threads": threads}) as con: con.register("src", pa.table({"l": column})) assert con.execute("SELECT count(*), sum(len(l)) FROM src").fetchall() == expected + assert con.execute(f"SELECT l FROM src LIMIT 5 OFFSET {empty - 2}").fetchall() == first_valued + assert con.execute(f"SELECT l FROM src OFFSET {len(column) - 3}").fetchall() == last diff --git a/tests/fast/arrow/scan/test_python_arrow_scan_core.py b/tests/fast/arrow/scan/test_python_arrow_scan_core.py index e97dc22c..fc580a61 100644 --- a/tests/fast/arrow/scan/test_python_arrow_scan_core.py +++ b/tests/fast/arrow/scan/test_python_arrow_scan_core.py @@ -1698,25 +1698,51 @@ def _field(struct, name): return ctypes.addressof(struct) + getattr(type(struct), name).offset +def _first_field(schema_address): + return _ArrowSchemaHead.from_address(_ArrowSchemaHead.from_address(schema_address).children[0]) + + class PokingStream(CountingStream): - def __init__(self, schema, batches, poke) -> None: + def __init__(self, schema, batches, poke, poke_schema=None) -> None: self.pokes = _Pokes() + self.poke_schema = poke_schema super().__init__(schema, batches, poke and (lambda array: poke(array.children[0].contents, self.pokes))) + def _get_schema(self, stream, out): + code = super()._get_schema(stream, out) + if self.poke_schema: + self.poke_schema(_first_field(out)) + return code + def _release_one(self, pointer): self.pokes.undo() super()._release_one(pointer) class PokingSource(PythonStreamSource): - """One batch, whose first column `poke(column, pokes)` changes after the export.""" + """One batch, whose first column `poke(column, pokes)` changes after the export. + + `poke_schema(field)` changes the first field of every schema exported, for a type pyarrow cannot build. + """ - def __init__(self, table, poke) -> None: + def __init__(self, table, poke, poke_schema=None) -> None: super().__init__(table, max_chunksize=table.num_rows) self.poke = poke + self.poke_schema = poke_schema + + def __arrow_c_schema__(self) -> object: + capsule = super().__arrow_c_schema__() + if self.poke_schema: + self.poke_schema(_first_field(_capsule_pointer(id(capsule), b"arrow_schema"))) + return capsule def stream(self, columns, filters): - stream = PokingStream(self.table.schema, self.table.to_batches(max_chunksize=self.max_chunksize), self.poke) + stream = PokingStream( + self.table.schema, + self.table.to_batches(max_chunksize=self.max_chunksize), + self.poke, + self.poke_schema, + ) self.streams.append(stream) return stream.capsule(), False @@ -1743,8 +1769,8 @@ def _drop_the_validity_bitmap(column, pokes): pokes.set(ctypes.c_void_p, column.buffers, None) -# One malformed or unreadable batch each: the table, the change to its first column after the export, and the text -# the refusal must carry. +# One malformed batch each: the table, the change to its first column after the export, the text the refusal must +# carry, and for a type pyarrow cannot build, the change to the first field of each exported schema. MALFORMED_NESTED = { "short_struct_child": ( _struct_table, @@ -1791,16 +1817,43 @@ def _drop_the_validity_bitmap(column, pokes): lambda column, pokes: pokes.set(ctypes.c_int64, _field(_child(column, 0), "length"), 1_999), "column 0 with a list whose offsets 0 to 2000 do not fit its child of 1999 rows", ), + "list_view_without_sizes": ( + lambda: pa.table({"l": pa.array([[1], [2, 3]] * 1_000, type=pa.list_view(pa.int64()))}), + lambda column, pokes: pokes.set(ctypes.c_void_p, column.buffers + 2 * ctypes.sizeof(ctypes.c_void_p), None), + "column 0 with a list view that has no offsets or sizes", + ), + "run_ends_stop_before_the_rows": ( + lambda: pa.table({"l": pa.FixedSizeListArray.from_arrays(_run_end_array(6), 2).slice(1)}), + lambda column, pokes: [ + pokes.set(ctypes.c_int64, _field(_child(_child(column, 0), index), "length"), 1) for index in (0, 1) + ], + "column 0 with a run end encoded array whose runs end at 3 before its 6 rows", + ), + "list_view_past_its_child": ( + lambda: pa.table( + { + "l": pa.ListViewArray.from_arrays( + pa.array([1], pa.int32()), pa.array([1], pa.int32()), pa.array([10, 99]) + ) + } + ), + lambda column, pokes: pokes.set(ctypes.c_int64, _field(_child(column, 0), "length"), 1), + "column 0 with a list view whose list at row 0 reaches past its child of 1 rows", + ), + "run_ends_of_tinyint": ( + lambda: pa.table( + {"r": pa.RunEndEncodedArray.from_arrays(pa.array([4], pa.int16()), pa.array([7], pa.int64()))} + ), + None, + "column 0 with a run end encoded array whose run ends are TINYINT, not SMALLINT, INTEGER or BIGINT", + # The one int16 run end of 4 reads as the int8 run end 4 on a little endian machine. + lambda field: ctypes.memmove(_ArrowSchemaHead.from_address(field.children[0]).format, b"c", 1), + ), "null_count_without_bitmap": ( lambda: pa.table({"i": pa.array([1, None] * 1_000)}), _drop_the_validity_bitmap, "column 0 with a non-zero null count but no validity bitmap", ), - "run_end_encoded_in_a_fixed_size_list": ( - lambda: pa.table({"l": pa.FixedSizeListArray.from_arrays(_run_end_array(4), 2)}), - None, - "column 0 with a run end encoded array inside a fixed size list", - ), "struct_child_null_count_without_bitmap": ( lambda: pa.table({"s": pa.StructArray.from_arrays([pa.array([1, None] * 1_000)], ["x"])}), lambda column, pokes: _drop_the_validity_bitmap(_child(column, 0), pokes), @@ -1811,29 +1864,6 @@ def _drop_the_validity_bitmap(column, pokes): lambda column, pokes: _drop_the_validity_bitmap(_child(column, 0), pokes), "column 0 with a non-zero null count but no validity bitmap", ), - "only_empty_lists_over_a_dictionary": ( - lambda: pa.table({"l": pa.array([[], None, []], type=pa.list_(pa.dictionary(pa.int32(), pa.string())))}), - None, - "column 0 with only empty lists over a dictionary-encoded child", - ), - "only_empty_large_lists_over_a_dictionary": ( - lambda: pa.table({"l": pa.array([[], None, []], type=pa.large_list(pa.dictionary(pa.int32(), pa.string())))}), - None, - "column 0 with only empty lists over a dictionary-encoded child", - ), - "only_empty_list_views_over_a_dictionary": ( - lambda: pa.table( - { - "l": pa.ListViewArray.from_arrays( - pa.array([5, 5], pa.int32()), - pa.array([0, 0], pa.int32()), - pa.array(list("abcde")).dictionary_encode(), - ) - } - ), - None, - "column 0 with only empty lists over a dictionary-encoded child", - ), } @@ -1912,17 +1942,20 @@ def _x() -> object: ), id="struct_in_a_sliced_union", ), - ], - ) - def test_an_offset_the_conversion_would_drop_is_refused(self, layout): - with duckdb.connect() as con: - con._register_arrow_source("s", ArraySource(layout(self._x()))) - with pytest.raises(duckdb.InvalidInputException, match="an offset the conversion would drop"): - con.execute("SELECT * FROM s").fetchall() - - @pytest.mark.parametrize( - "layout", - [ + pytest.param( + lambda x: pa.StructArray.from_arrays( + [ + pa.ListArray.from_arrays( + pa.array([0, 2, 3], pa.int32()), + pa.UnionArray.from_sparse( + pa.array([0, 1, 0, 1, 0], pa.int8()), [x, pa.array(list("abcde"))], ["i", "s"] + ).slice(1, 3), + ) + ], + ["l"], + ), + id="sliced_union_in_a_list", + ), pytest.param( lambda x: pa.StructArray.from_arrays( [pa.StructArray.from_arrays([pa.StructArray.from_arrays([x], ["x"]).slice(1, 4)], ["b"])], ["a"] @@ -1973,8 +2006,7 @@ def test_a_variant_is_checked_as_the_struct_it_is_stored_as(self): table = con.sql("SELECT {'v': x::VARIANT} AS s, x::VARIANT AS v, x FROM range(10) t(x)").to_arrow_table() con._register_arrow_source("t", TableSource(table.slice(2))) assert con.execute("SELECT v::BIGINT, x FROM t").fetchall() == [(x, x) for x in range(2, 10)] - with pytest.raises(duckdb.InvalidInputException, match="column 0 with a struct or union below an offset"): - con.execute("SELECT s FROM t").fetchall() + assert con.execute("SELECT s FROM t").fetchall() == [({"v": x},) for x in range(2, 10)] @pytest.mark.parametrize( ("make", "expected"), @@ -1996,6 +2028,30 @@ def test_a_variant_is_checked_as_the_struct_it_is_stored_as(self): [({},), (None,), ({},)], id="empty_maps", ), + pytest.param( + lambda: pa.array([[], None, []], type=pa.list_(pa.dictionary(pa.int32(), pa.string()))), + [([],), (None,), ([],)], + id="only_empty_lists", + ), + pytest.param( + lambda: pa.array([[], None, []], type=pa.large_list(pa.dictionary(pa.int32(), pa.string()))), + [([],), (None,), ([],)], + id="only_empty_large_lists", + ), + pytest.param( + lambda: pa.ListViewArray.from_arrays( + pa.array([5, 5], pa.int32()), + pa.array([0, 0], pa.int32()), + pa.array(list("abcde")).dictionary_encode(), + ), + [([],), ([],)], + id="only_empty_list_views_from_offset_five", + ), + pytest.param( + lambda: pa.array([[], None, []], type=pa.list_view(pa.dictionary(pa.int32(), pa.string()))), + [([],), (None,), ([],)], + id="only_empty_list_views", + ), ], ) def test_empty_lists_over_a_dictionary_the_conversion_reads_are_read(self, make, expected): @@ -2036,6 +2092,21 @@ def test_empty_lists_over_a_dictionary_the_conversion_reads_are_read(self, make, [(({"r": 7}, {"r": 7}),), (({"r": 8}, {"r": 8}),)], id="struct_in_a_fixed_size_list", ), + pytest.param( + lambda: pa.FixedSizeListArray.from_arrays(_run_end_array(4), 2), + [((7, 7),), ((8, 8),)], + id="fixed_size_list", + ), + pytest.param( + lambda: pa.FixedSizeListArray.from_arrays(_run_end_array(6), 2, mask=pa.array([False, True, False])), + [((7, 7),), (None,), ((8, 8),)], + id="fixed_size_list_with_a_null_row", + ), + pytest.param( + lambda: pa.FixedSizeListArray.from_arrays(_run_end_array(6), 2).slice(1), + [((7, 8),), ((8, 8),)], + id="sliced_fixed_size_list", + ), ], ) def test_a_run_end_encoded_array_below_a_list_the_conversion_reads_is_read(self, make, expected): @@ -2053,8 +2124,8 @@ def test_a_malformed_nested_array_is_refused_and_released(self, case): module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) import duckdb -make, poke, message = module.MALFORMED_NESTED[{case!r}] -source = module.PokingSource(make(), poke) +make, poke, message, *poke_schema = module.MALFORMED_NESTED[{case!r}] +source = module.PokingSource(make(), poke, *poke_schema) con = duckdb.connect(config={{"threads": 4}}) con._register_arrow_source("t", source) try: