diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs index a6b297d3..add8b6e6 100644 --- a/crates/pyspark-rs/src/session.rs +++ b/crates/pyspark-rs/src/session.rs @@ -1011,6 +1011,22 @@ impl PySparkSession { .import("pyspark.sql._pandas_input")? .getattr("prepare_string_input")? .call1((data, schema))?; + // Columns kept in pandas' Arrow string storage come as an Arrow table, which the + // importer sends, or uploads at the relation cache threshold. + if !ipc.is_none() && !ipc.is_instance_of::() { + let frame = + crate::pyarrow_import::create_dataframe(py, &self.session, &ipc, schema)?; + return Ok(match &spec { + Spec::Struct(fields) if !fields.is_empty() && ipc.len()? > 0 => { + PyDataFrame::new( + frame + .dataframe + .to_df(fields.iter().map(|field| field.name.as_str()).collect()), + ) + } + _ => frame, + }); + } if !ipc.is_none() { let fields = match &spec { Spec::Struct(fields) => fields, diff --git a/docs/architecture.md b/docs/architecture.md index ee734f0d..0c7b7bec 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -159,6 +159,10 @@ Pandas frames of at least 100,000 rows whose columns are all NumPy `float64`, with an explicit `StructType` of `DoubleType` fields, are encoded to Arrow IPC from their columns, with pandas' missing values as nulls, and sent inline as the row path sends them, without the importer's configuration RPC. +Pandas frames of at least 10,000 rows whose columns are all kept in pandas' Arrow +string storage (`string[pyarrow]`, and the `str` dtype of pandas 3), with an +explicit `StructType` of `StringType` fields, pass that storage to this importer +as Arrow string columns, without Python rows. Inputs at or above `spark.sql.session.localRelationCacheThreshold` are serialized into independent IPC streams and uploaded as session-scoped cache artifacts. The plan carries `ChunkedCachedLocalRelation` hashes instead of embedding the diff --git a/python/pyspark/sql/_pandas_input.py b/python/pyspark/sql/_pandas_input.py index 8cf6d98a..1077dd4d 100644 --- a/python/pyspark/sql/_pandas_input.py +++ b/python/pyspark/sql/_pandas_input.py @@ -533,7 +533,8 @@ def prepare_float64_input(frame, schema): def prepare_string_input(frame, schema): - """Prepare inline IPC for explicit columns of exact strings or None. + """Prepare inline IPC for explicit columns of exact strings or None, or an Arrow table for + columns that pandas keeps in Arrow string storage (see _arrow_string_table). Inspect every object before Arrow conversion: str subclasses can override conversion hooks, and pandas treats several other objects as missing values. @@ -598,9 +599,10 @@ def prepare_string_input(frame, schema): for field in schema.fields ) or len(set(schema.names)) != width: return None + dtypes = frame.dtypes if not all(type(dtype) is type(np.dtype("object")) and dtype == np.dtype("object") - for dtype in frame.dtypes): - return None + for dtype in dtypes): + return _arrow_string_table(frame, schema, dtypes) # A custom ExtensionArray may report a genuine NumPy dtype while its # Arrow hook emits different values from to_numpy(). Trust only pandas' # concrete NumPy backing array, including its name in older pandas releases. @@ -631,6 +633,46 @@ def prepare_string_input(frame, schema): return _pandas_string_ipc(arrays, schema) +def _arrow_string_table(frame, schema, dtypes): + """The columns of a frame kept in pandas' Arrow string storage (string[pyarrow], and the + str dtype of pandas 3), as an Arrow table of the schema, or None. + + For the row path, pandas converts such a column from that storage, with its missing + values as None, so the storage holds the same values. Only pandas' own array classes are + trusted to keep them there: a subclass could convert differently. The caller's guards of + prepare_string_input (frame, schema, names, size) have passed. + """ + import pandas as pd + import pyarrow as pa + from pandas.core.arrays import string_arrow + + classes = {pd.arrays.ArrowStringArray, + getattr(string_arrow, "ArrowStringArrayNumpySemantics", None)} - {None} + if not all(type(dtype) is pd.StringDtype and dtype.storage in ("pyarrow", "pyarrow_numpy") + for dtype in dtypes): + return None + arrays = [] + for (_, column), field in zip(frame.items(), schema.fields): + array = column.array + if type(array) not in classes: + return None + values = pa.chunked_array(array.__arrow_array__()) + if values.type not in (pa.string(), pa.large_string()): + return None + try: + # Malformed offsets or UTF-8 keep the row path and its error. + values.validate(full=True) + values = values.cast(pa.string()) + except (pa.ArrowException, OverflowError): + return None + if not field.nullable and values.null_count: + return None + arrays.append(values) + # The importer applies the nullability and metadata of the schema, and uploads the table + # when it is at least spark.sql.session.localRelationCacheThreshold. + return pa.Table.from_arrays(arrays, names=schema.names) + + def prepare_date_table(frame, schema): """Accept only explicit date columns whose cells are exact dates or None. diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index 9424ac5e..738c3fb4 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -3968,8 +3968,113 @@ def __str__(self): frame.iat[-1, -1] = value if reason in ("late_unsupported", "late_surrogate"): assert frame.iat[-1, -1] is value - assert prepare_string_input(frame, schema) is None + result = prepare_string_input(frame, schema) + if reason == "arrow_string_dtype": + # Arrow string storage has a path of its own, which returns an Arrow table. + assert type(result) is pa.Table + else: + assert result is None + + + +# Columns kept in pandas' Arrow string storage reach the importer as Arrow, without Python rows. + +def _arrow_string_column(texts, storage): + import pyarrow as pa + if storage == "numpy_semantics": + try: + dtype = pd.StringDtype("pyarrow", na_value=float("nan")) + except TypeError: # pandas 2.2 + dtype = pd.StringDtype("pyarrow_numpy") + return pd.array(texts, dtype=dtype) + if storage == "large_string": + return pd.arrays.ArrowStringArray(pa.chunked_array( + [pa.array(texts[:7], pa.large_string()), pa.array(texts[7:], pa.large_string())])) + return pd.array(texts, dtype="string[pyarrow]") + + +@pytest.mark.parametrize("cached", [False, True]) +@pytest.mark.parametrize("storage", ["pyarrow", "numpy_semantics", "large_string", "sliced"]) +def test_arrow_string_columns_preserve_every_ipc_cell(protocol, monkeypatch, cached, storage): + from pyspark.sql.pandas.types import from_arrow_metadata + + texts = ["", "a", "\x00", "café", "中文", "\U0001f600", "é", None] * 2000 + other = [None if i % 3 == 0 else str(i) for i in range(len(texts))] + frame = pd.DataFrame({ + "original": _arrow_string_column(texts, storage), + "b": _arrow_string_column(other, storage), + "c": _arrow_string_column([None] * len(texts), storage), + }) + expected = [texts, other, [None] * len(texts)] + if storage == "sliced": + frame = frame.iloc[3:-5] + expected = [values[3:-5] for values in expected] + schema = T.StructType([ + T.StructField("renamed.`text", T.StringType(), True, {"source": "string"}), + T.StructField("other", T.StringType()), + T.StructField("all_null", T.StringType()), + ]) + + def no_rows(*args, **kwargs): + raise AssertionError("Arrow string storage must not materialize Python rows") + + for method in ("to_numpy", "iterrows", "itertuples"): + monkeypatch.setattr(pd.DataFrame, method, no_rows) + monkeypatch.setattr(pd.DataFrame, "values", property(no_rows)) + monkeypatch.setattr(pd.Series, "tolist", no_rows) + overrides = {"spark.sql.session.localRelationCacheThreshold": "1" if cached else "999999999"} + with _arrow_input_service(protocol, **overrides) as (spark, state): + df = spark.createDataFrame(frame, schema) + df.toArrow() + assert state["column_names"] == schema.names + table = state["input_tables"][0] + assert table.schema.names == schema.names + for column, values, field in zip(table.columns, expected, schema.fields): + assert column.type == pa.string() + assert column.to_pylist() == values + actual_field = table.schema.field(field.name) + assert actual_field.nullable == field.nullable + assert from_arrow_metadata(actual_field.metadata) == (field.metadata or None) + assert bool(state["uploads"]) == cached + + +@pytest.mark.parametrize("reason", [ + "python_storage", "mixed", "nonnullable_null", "subclass", "small", "atomic_schema", +]) +def test_arrow_string_columns_keep_the_row_path(reason): + from pyspark.sql._pandas_input import prepare_string_input + + texts = ["a", None, "b"] * 4000 + columns = {"a": _arrow_string_column(texts, "pyarrow"), "b": _arrow_string_column(texts, "pyarrow")} + fields = [T.StructField("a", T.StringType()), T.StructField("b", T.StringType())] + if reason == "python_storage": + columns["b"] = pd.array(texts, dtype="string[python]") + elif reason == "mixed": + columns["b"] = pd.Series(texts, dtype=object) + elif reason == "nonnullable_null": + fields[1] = T.StructField("b", T.StringType(), False) + elif reason == "subclass": + class Custom(pd.arrays.ArrowStringArray): + pass + columns["b"] = Custom(pa.chunked_array([pa.array(texts)])) + elif reason == "small": + columns = {name: values[:9999] for name, values in columns.items()} + elif reason == "atomic_schema": + fields[1] = T.StructField("b", T.IntegerType()) + frame = pd.DataFrame(columns) + assert prepare_string_input(frame, T.StructType(fields)) is None + + +def test_arrow_string_columns_match_the_row_path(protocol): + texts = ["x" * (i % 5) if i % 4 else None for i in range(12000)] + schema = T.StructType([T.StructField("v", T.StringType())]) + tables = [] + for dtype in ("string[pyarrow]", object): + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(pd.DataFrame({"v": pd.array(texts, dtype=dtype)}), schema).toArrow() + tables.append(state["input_tables"][0]) + assert tables[0].equals(tables[1]) @pytest.mark.parametrize("reason", [