diff --git a/docs/architecture.md b/docs/architecture.md index 0c7b7bec..abbc4f77 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -159,6 +159,9 @@ Pandas frames of at least 100,000 rows whose columns are all NumPy `float64`, with an explicit `StructType` of `DoubleType` fields, are encoded to Arrow IPC from their columns, with pandas' missing values as nulls, and sent inline as the row path sends them, without the importer's configuration RPC. +Pandas object columns containing exact `datetime.date` values or `None` use the +Arrow importer at 10,000 rows or more with an explicit `StructType` of `DateType` +fields. Unsupported values and customized inputs retain the row conversion path. Pandas frames of at least 10,000 rows whose columns are all kept in pandas' Arrow string storage (`string[pyarrow]`, and the `str` dtype of pandas 3), with an explicit `StructType` of `StringType` fields, pass that storage to this importer diff --git a/python/pyspark/sql/_pandas_input.py b/python/pyspark/sql/_pandas_input.py index f4891c23..cf828275 100644 --- a/python/pyspark/sql/_pandas_input.py +++ b/python/pyspark/sql/_pandas_input.py @@ -796,19 +796,31 @@ def prepare_date_table(frame, schema): except ImportError: return None columns = [frame.iloc[:, i] for i in range(width)] + arrays = [] for column, field in zip(columns, schema.fields): - if type(column) is not Series or type(column.array) is not NumpyExtensionArray: + if type(column) is not Series: + return None + backing = column.array + if type(backing) is not NumpyExtensionArray: + return None + array = backing._ndarray + if (type(array) is not np.ndarray or array.ndim != 1 + or type(array.dtype) is not type(np.dtype("object")) + or array.dtype != np.dtype("object") or len(array) != len(frame)): return None # Arrow rejects nulls in non-nullable fields; let the row path retain # its existing behavior for that input instead. if not all(type(value) is date or (value is None and field.nullable) - for value in column.to_numpy(copy=False)): + for value in array): return None + arrays.append(array) import pyarrow as pa return pa.Table.from_arrays( - [pa.array(column, type=pa.date32(), from_pandas=False) for column in columns], + # Series accessors can be customized independently of the DataFrame row + # path. Emit the same concrete storage whose cells were checked above. + [pa.array(array, type=pa.date32(), from_pandas=False) for array in arrays], names=schema.names, ) diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index af993c41..ce0ec28a 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -4763,6 +4763,38 @@ def test_pandas_inferred_int_beyond_int64_raises_as_pyspark(protocol, schema): # Exact built-in date columns reuse Arrow without materializing pandas rows. +@pytest.mark.parametrize("accessor", ["values", "to_numpy"]) +def test_date_series_accessors_do_not_replace_validated_cells(monkeypatch, accessor): + from pyspark.sql._pandas_input import prepare_date_table + + dates = [date.min, date.max, date(1969, 12, 31), date(2000, 2, 29)] * 2500 + nullable = [None if i % 7 == 0 else value for i, value in enumerate(dates)] + expected = {"renamed": dates, "nullable": nullable} + frame = pd.DataFrame({"original": dates, "nullable": nullable}, dtype=object) + schema = T.StructType([ + T.StructField("renamed", T.DateType(), False), + T.StructField("nullable", T.DateType()), + ]) + row_values = frame.astype(object).where(frame.notna(), None).values.tolist() + calls = [] + + def replacement(series, *args, **kwargs): + calls.append(accessor) + return np.full(len(series), date(2021, 2, 3), dtype=object) + + monkeypatch.setattr(pd.Series, accessor, + property(replacement) if accessor == "values" else replacement) + table = prepare_date_table(frame, schema) + assert table is not None + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, table.schema) as writer: + writer.write_table(table) + actual = pa.ipc.open_stream(sink.getvalue()).read_all() + assert actual.to_pydict() == expected + assert frame.astype(object).where(frame.notna(), None).values.tolist() == row_values + assert calls == [] + + @pytest.mark.parametrize("cached", [False, True]) @pytest.mark.parametrize("verify", [False, True]) def test_date_columns_preserve_every_ipc_cell(protocol, monkeypatch, cached, verify):