Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
dc29a19
[SPARK-60676][CONNECT] Use Arrow columns for signed int64 pandas inputs
dbtsai Oct 29, 2026
3e79f91
[SPARK-60676][CONNECT] Validate integer upload scales against mock RPC
dbtsai Oct 29, 2026
8c705c0
[SPARK-60676][CONNECT] Cover leading null inference and non-nullable …
dbtsai Oct 29, 2026
fb996d9
[SPARK-60676][CONNECT] Exercise named schemas through mock rename rel…
dbtsai Oct 29, 2026
79d3faf
[SPARK-60676][CONNECT] Avoid Arrow configuration overhead for small N…
dbtsai Oct 29, 2026
e266e8e
[SPARK-60676][CONNECT] Retain IPC row fallback for NUL field names
dbtsai Oct 29, 2026
9942f21
[SPARK-60676][CONNECT] Include pandas regressions in the existing Arr…
dbtsai Oct 29, 2026
5edd700
[SPARK-60676][CONNECT] Preserve integer pandas schema and fallback se…
dbtsai Oct 29, 2026
080aa16
[SPARK-60676][TESTS] Preserve subclass column labels on pandas 3
dbtsai Oct 30, 2026
abbc23a
[SPARK-60676][CONNECT] Preserve customized pandas columns hooks
dbtsai Oct 30, 2026
7961bde
[SPARK-60676][CONNECT] Preserve rebound pandas index storage ordering
dbtsai Oct 30, 2026
3e3a8f5
Preserve integer pandas fallback with missing Arrow aliases
dbtsai Oct 31, 2026
f058ac9
[SPARK-60676][TESTS] Retain current master coverage after rebase
dbtsai Nov 5, 2026
121a141
Preserve current pandas label iteration and fallback order after rebase
dbtsai Nov 5, 2026
aca1d02
Keep canonical string storage guard module identity
dbtsai Nov 5, 2026
ecdd67a
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
e7ae71a
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
40e5266
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
26be6ab
[SPARK-60720][CONNECT] Encode guarded pandas String input inline
dbtsai Oct 30, 2026
09d0b13
[SPARK-60720][CONNECT] Remove extra EOF blank lines
dbtsai Oct 30, 2026
4304062
Reconcile current master interfaces and formatting after rebase
dbtsai Nov 5, 2026
b2274e4
Align string label regression tests with current native iteration
dbtsai Nov 5, 2026
e31aad6
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
34dd699
[SPARK-60745][CONNECT] Keep canonical pandas Arrow strings columnar
dbtsai Oct 30, 2026
e805121
[SPARK-60745][CONNECT] Decline customized pandas manager index caches
dbtsai Oct 30, 2026
6b1d281
[SPARK-60745][CONNECT] Cover Arrow base decoding errors in row control
dbtsai Oct 30, 2026
16fc5a7
[CONNECT] Match current schema-width fallback errors
dbtsai Nov 5, 2026
52d466d
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
587d0d6
Merge remote-tracking branch 'apache/master' into the replacement of …
HyukjinKwon Nov 6, 2026
962620e
[SPARK-60745][CONNECT] Pass pandas Arrow string storage to the Arrow …
HyukjinKwon Nov 7, 2026
52d0549
Merge remote-tracking branch 'apache/master' into the replacement of …
HyukjinKwon Nov 7, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions crates/pyspark-rs/src/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1011,6 +1011,22 @@ impl PySparkSession {
.import("pyspark.sql._pandas_input")?
.getattr("prepare_string_input")?
.call1((data, schema))?;
// Columns kept in pandas' Arrow string storage come as an Arrow table, which the
// importer sends, or uploads at the relation cache threshold.
if !ipc.is_none() && !ipc.is_instance_of::<pyo3::types::PyBytes>() {
let frame =
crate::pyarrow_import::create_dataframe(py, &self.session, &ipc, schema)?;
return Ok(match &spec {
Spec::Struct(fields) if !fields.is_empty() && ipc.len()? > 0 => {
PyDataFrame::new(
frame
.dataframe
.to_df(fields.iter().map(|field| field.name.as_str()).collect()),
)
}
_ => frame,
});
}
if !ipc.is_none() {
let fields = match &spec {
Spec::Struct(fields) => fields,
Expand Down
4 changes: 4 additions & 0 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,10 @@ Pandas frames of at least 100,000 rows whose columns are all NumPy `float64`,
with an explicit `StructType` of `DoubleType` fields, are encoded to Arrow IPC
from their columns, with pandas' missing values as nulls, and sent inline as the
row path sends them, without the importer's configuration RPC.
Pandas frames of at least 10,000 rows whose columns are all kept in pandas' Arrow
string storage (`string[pyarrow]`, and the `str` dtype of pandas 3), with an
explicit `StructType` of `StringType` fields, pass that storage to this importer
as Arrow string columns, without Python rows.
Inputs at or above `spark.sql.session.localRelationCacheThreshold` are serialized
into independent IPC streams and uploaded as session-scoped cache artifacts.
The plan carries `ChunkedCachedLocalRelation` hashes instead of embedding the
Expand Down
48 changes: 45 additions & 3 deletions python/pyspark/sql/_pandas_input.py
Original file line number Diff line number Diff line change
Expand Up @@ -533,7 +533,8 @@ def prepare_float64_input(frame, schema):


def prepare_string_input(frame, schema):
"""Prepare inline IPC for explicit columns of exact strings or None.
"""Prepare inline IPC for explicit columns of exact strings or None, or an Arrow table for
columns that pandas keeps in Arrow string storage (see _arrow_string_table).

Inspect every object before Arrow conversion: str subclasses can override
conversion hooks, and pandas treats several other objects as missing values.
Expand Down Expand Up @@ -598,9 +599,10 @@ def prepare_string_input(frame, schema):
for field in schema.fields
) or len(set(schema.names)) != width:
return None
dtypes = frame.dtypes
if not all(type(dtype) is type(np.dtype("object")) and dtype == np.dtype("object")
for dtype in frame.dtypes):
return None
for dtype in dtypes):
return _arrow_string_table(frame, schema, dtypes)
# A custom ExtensionArray may report a genuine NumPy dtype while its
# Arrow hook emits different values from to_numpy(). Trust only pandas'
# concrete NumPy backing array, including its name in older pandas releases.
Expand Down Expand Up @@ -631,6 +633,46 @@ def prepare_string_input(frame, schema):
return _pandas_string_ipc(arrays, schema)


def _arrow_string_table(frame, schema, dtypes):
"""The columns of a frame kept in pandas' Arrow string storage (string[pyarrow], and the
str dtype of pandas 3), as an Arrow table of the schema, or None.

For the row path, pandas converts such a column from that storage, with its missing
values as None, so the storage holds the same values. Only pandas' own array classes are
trusted to keep them there: a subclass could convert differently. The caller's guards of
prepare_string_input (frame, schema, names, size) have passed.
"""
import pandas as pd
import pyarrow as pa
from pandas.core.arrays import string_arrow

classes = {pd.arrays.ArrowStringArray,
getattr(string_arrow, "ArrowStringArrayNumpySemantics", None)} - {None}
if not all(type(dtype) is pd.StringDtype and dtype.storage in ("pyarrow", "pyarrow_numpy")
for dtype in dtypes):
return None
arrays = []
for (_, column), field in zip(frame.items(), schema.fields):
array = column.array
if type(array) not in classes:
return None
values = pa.chunked_array(array.__arrow_array__())
if values.type not in (pa.string(), pa.large_string()):
return None
try:
# Malformed offsets or UTF-8 keep the row path and its error.
values.validate(full=True)
values = values.cast(pa.string())
except (pa.ArrowException, OverflowError):
return None
if not field.nullable and values.null_count:
return None
arrays.append(values)
# The importer applies the nullability and metadata of the schema, and uploads the table
# when it is at least spark.sql.session.localRelationCacheThreshold.
return pa.Table.from_arrays(arrays, names=schema.names)


def prepare_date_table(frame, schema):
"""Accept only explicit date columns whose cells are exact dates or None.

Expand Down
107 changes: 106 additions & 1 deletion python/tests/test_arrow_handoff.py
Original file line number Diff line number Diff line change
Expand Up @@ -3968,8 +3968,113 @@ def __str__(self):
frame.iat[-1, -1] = value
if reason in ("late_unsupported", "late_surrogate"):
assert frame.iat[-1, -1] is value
assert prepare_string_input(frame, schema) is None
result = prepare_string_input(frame, schema)
if reason == "arrow_string_dtype":
# Arrow string storage has a path of its own, which returns an Arrow table.
assert type(result) is pa.Table
else:
assert result is None



# Columns kept in pandas' Arrow string storage reach the importer as Arrow, without Python rows.

def _arrow_string_column(texts, storage):
import pyarrow as pa
if storage == "numpy_semantics":
try:
dtype = pd.StringDtype("pyarrow", na_value=float("nan"))
except TypeError: # pandas 2.2
dtype = pd.StringDtype("pyarrow_numpy")
return pd.array(texts, dtype=dtype)
if storage == "large_string":
return pd.arrays.ArrowStringArray(pa.chunked_array(
[pa.array(texts[:7], pa.large_string()), pa.array(texts[7:], pa.large_string())]))
return pd.array(texts, dtype="string[pyarrow]")


@pytest.mark.parametrize("cached", [False, True])
@pytest.mark.parametrize("storage", ["pyarrow", "numpy_semantics", "large_string", "sliced"])
def test_arrow_string_columns_preserve_every_ipc_cell(protocol, monkeypatch, cached, storage):
from pyspark.sql.pandas.types import from_arrow_metadata

texts = ["", "a", "\x00", "café", "中文", "\U0001f600", "é", None] * 2000
other = [None if i % 3 == 0 else str(i) for i in range(len(texts))]
frame = pd.DataFrame({
"original": _arrow_string_column(texts, storage),
"b": _arrow_string_column(other, storage),
"c": _arrow_string_column([None] * len(texts), storage),
})
expected = [texts, other, [None] * len(texts)]
if storage == "sliced":
frame = frame.iloc[3:-5]
expected = [values[3:-5] for values in expected]
schema = T.StructType([
T.StructField("renamed.`text", T.StringType(), True, {"source": "string"}),
T.StructField("other", T.StringType()),
T.StructField("all_null", T.StringType()),
])

def no_rows(*args, **kwargs):
raise AssertionError("Arrow string storage must not materialize Python rows")

for method in ("to_numpy", "iterrows", "itertuples"):
monkeypatch.setattr(pd.DataFrame, method, no_rows)
monkeypatch.setattr(pd.DataFrame, "values", property(no_rows))
monkeypatch.setattr(pd.Series, "tolist", no_rows)
overrides = {"spark.sql.session.localRelationCacheThreshold": "1" if cached else "999999999"}
with _arrow_input_service(protocol, **overrides) as (spark, state):
df = spark.createDataFrame(frame, schema)
df.toArrow()
assert state["column_names"] == schema.names
table = state["input_tables"][0]
assert table.schema.names == schema.names
for column, values, field in zip(table.columns, expected, schema.fields):
assert column.type == pa.string()
assert column.to_pylist() == values
actual_field = table.schema.field(field.name)
assert actual_field.nullable == field.nullable
assert from_arrow_metadata(actual_field.metadata) == (field.metadata or None)
assert bool(state["uploads"]) == cached


@pytest.mark.parametrize("reason", [
"python_storage", "mixed", "nonnullable_null", "subclass", "small", "atomic_schema",
])
def test_arrow_string_columns_keep_the_row_path(reason):
from pyspark.sql._pandas_input import prepare_string_input

texts = ["a", None, "b"] * 4000
columns = {"a": _arrow_string_column(texts, "pyarrow"), "b": _arrow_string_column(texts, "pyarrow")}
fields = [T.StructField("a", T.StringType()), T.StructField("b", T.StringType())]
if reason == "python_storage":
columns["b"] = pd.array(texts, dtype="string[python]")
elif reason == "mixed":
columns["b"] = pd.Series(texts, dtype=object)
elif reason == "nonnullable_null":
fields[1] = T.StructField("b", T.StringType(), False)
elif reason == "subclass":
class Custom(pd.arrays.ArrowStringArray):
pass

columns["b"] = Custom(pa.chunked_array([pa.array(texts)]))
elif reason == "small":
columns = {name: values[:9999] for name, values in columns.items()}
elif reason == "atomic_schema":
fields[1] = T.StructField("b", T.IntegerType())
frame = pd.DataFrame(columns)
assert prepare_string_input(frame, T.StructType(fields)) is None


def test_arrow_string_columns_match_the_row_path(protocol):
texts = ["x" * (i % 5) if i % 4 else None for i in range(12000)]
schema = T.StructType([T.StructField("v", T.StringType())])
tables = []
for dtype in ("string[pyarrow]", object):
with _arrow_input_service(protocol) as (spark, state):
spark.createDataFrame(pd.DataFrame({"v": pd.array(texts, dtype=dtype)}), schema).toArrow()
tables.append(state["input_tables"][0])
assert tables[0].equals(tables[1])


@pytest.mark.parametrize("reason", [
Expand Down
Loading