Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
dc29a19
[SPARK-60676][CONNECT] Use Arrow columns for signed int64 pandas inputs
dbtsai Oct 29, 2026
3e79f91
[SPARK-60676][CONNECT] Validate integer upload scales against mock RPC
dbtsai Oct 29, 2026
8c705c0
[SPARK-60676][CONNECT] Cover leading null inference and non-nullable …
dbtsai Oct 29, 2026
fb996d9
[SPARK-60676][CONNECT] Exercise named schemas through mock rename rel…
dbtsai Oct 29, 2026
79d3faf
[SPARK-60676][CONNECT] Avoid Arrow configuration overhead for small N…
dbtsai Oct 29, 2026
e266e8e
[SPARK-60676][CONNECT] Retain IPC row fallback for NUL field names
dbtsai Oct 29, 2026
9942f21
[SPARK-60676][CONNECT] Include pandas regressions in the existing Arr…
dbtsai Oct 29, 2026
5edd700
[SPARK-60676][CONNECT] Preserve integer pandas schema and fallback se…
dbtsai Oct 29, 2026
080aa16
[SPARK-60676][TESTS] Preserve subclass column labels on pandas 3
dbtsai Oct 30, 2026
abbc23a
[SPARK-60676][CONNECT] Preserve customized pandas columns hooks
dbtsai Oct 30, 2026
7961bde
[SPARK-60676][CONNECT] Preserve rebound pandas index storage ordering
dbtsai Oct 30, 2026
3e3a8f5
Preserve integer pandas fallback with missing Arrow aliases
dbtsai Oct 31, 2026
f058ac9
[SPARK-60676][TESTS] Retain current master coverage after rebase
dbtsai Nov 5, 2026
121a141
Preserve current pandas label iteration and fallback order after rebase
dbtsai Nov 5, 2026
aca1d02
Keep canonical string storage guard module identity
dbtsai Nov 5, 2026
ecdd67a
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
e7ae71a
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
40e5266
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
280bdee
[SPARK-60727][CONNECT] Keep canonical double input on inline transport
dbtsai Oct 30, 2026
11ff3b7
[SPARK-60727][CONNECT] Normalize Float helper terminal newline
dbtsai Oct 30, 2026
b9498d1
Reconcile current master interfaces and formatting after rebase
dbtsai Nov 5, 2026
2bd4638
Align float label regression tests with current native iteration
dbtsai Nov 5, 2026
cfde2b6
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
b415494
Merge remote-tracking branch 'apache/master' into SPARK-60727-pandas-…
HyukjinKwon Nov 6, 2026
489a663
[SPARK-60727][CONNECT] Drop the converter guards that the float64 pat…
HyukjinKwon Nov 6, 2026
9fbcd15
[SPARK-60727][CONNECT] Do not pin the reads of master's pandas row pa…
HyukjinKwon Nov 6, 2026
a31846b
Merge remote-tracking branch 'apache/master' into SPARK-60727-pandas-…
HyukjinKwon Nov 6, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions crates/pyspark-rs/src/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -871,6 +871,31 @@ impl PySparkSession {
});
}
}
let ipc = py
.import("pyspark.sql._pandas_input")?
.getattr("prepare_float64_input")?
.call1((data, schema))?;
if !ipc.is_none() {
let fields = match &spec {
Spec::Struct(fields) => fields,
_ => {
return Err(pyo3::exceptions::PyValueError::new_err(
"float64 input requires a struct schema",
));
}
};
let schema = DataType::Struct {
fields: fields.clone(),
};
let frame = self
.session
.create_dataframe_from_inline_arrow_ipc(ipc.extract()?, schema);
// Match pandas row input: explicit nonempty field names travel
// through toDF as well as the local relation's logical schema.
return Ok(PyDataFrame::new(
frame.to_df(fields.iter().map(|field| field.name.as_str()).collect()),
));
}
return self.create_from_pandas(py, data, spec);
}
// Preserve schema parsing, ignored-argument warnings, and pandas/Arrow routing.
Expand Down
17 changes: 17 additions & 0 deletions crates/spark-connect/src/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -413,6 +413,23 @@ impl SparkSession {
Ok(DataFrame::new(self.clone(), plan))
}

/// Internal binding bridge for Arrow IPC already validated and encoded from
/// local rows or columns. Retains the row path's inline transport policy.
#[doc(hidden)]
pub fn create_dataframe_from_inline_arrow_ipc(
&self,
data: Vec<u8>,
schema: DataType,
) -> DataFrame {
DataFrame::new(
self.clone(),
LogicalPlan::LocalRelation {
schema: Some(schema),
data: Some(data),
},
)
}

/// Create a local DataFrame from Arrow batches without materializing rows.
///
/// `schema` optionally supplies the Spark schema applied by the server. With
Expand Down
4 changes: 4 additions & 0 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,10 @@ and explicit `StructType` schemas whose fields are all `LongType`. Other dtypes,
DDL/atomic schemas, schema conversions, NUL names, and duplicate/non-string input names
retain the existing row conversion path. `verifySchema` remains ignored for
pandas Arrow input, matching Spark Connect.
Pandas frames of at least 100,000 rows whose columns are all NumPy `float64`,
with an explicit `StructType` of `DoubleType` fields, are encoded to Arrow IPC
from their columns, with pandas' missing values as nulls, and sent inline as the
row path sends them, without the importer's configuration RPC.
Inputs at or above `spark.sql.session.localRelationCacheThreshold` are serialized
into independent IPC streams and uploaded as session-scoped cache artifacts.
The plan carries `ChunkedCachedLocalRelation` hashes instead of embedding the
Expand Down
110 changes: 110 additions & 0 deletions python/pyspark/sql/_pandas_input.py
Original file line number Diff line number Diff line change
Expand Up @@ -420,3 +420,113 @@ def prepare_timestamp_table(columns, schema, timezone):
for column in columns],
names=schema.names,
)


def prepare_float64_table(frame, schema):
"""Keep explicit native double columns columnar without changing pandas nulls."""
import numpy as np
import pandas as pd
from pandas.core import frame as frame_module, generic as generic_module
from pandas.core.frame import DataFrame
from pandas.core.series import Series
from types import FunctionType, GetSetDescriptorType, WrapperDescriptorType

from pyspark import _pyspark as native
from pyspark.sql import types as sql_types

# First-use Arrow imports outweigh the conversion saving for smaller frames.
# Keep those on the existing row path.
if (pd.DataFrame is not DataFrame or type(frame) is not DataFrame
or type(schema) is not native.StructType or len(frame) < 100000
or any(name in vars(frame) for name in ("astype", "notna"))):
return None
if not _has_canonical_columns(frame.columns):
return None
# Class or instance conversion hooks must retain the row path's calls.
for name, module in (("astype", generic_module), ("notna", frame_module),
("where", generic_module)):
method = getattr(DataFrame, name)
if (type(method) is not FunctionType or method.__name__ != name
or method.__module__ != module.__name__ or method.__globals__ is not vars(module)):
return None
if (DataFrame.astype is not generic_module.NDFrame.astype
or DataFrame.where is not generic_module.NDFrame.where):
return None
width = len(frame.columns)
if not width or not frame.columns.is_unique or not all(
type(name) is str and "\x00" not in name for name in frame.columns
):
return None
double_dtype = np.dtype("float64")
if not all(type(dtype) is type(double_dtype) and dtype == double_dtype
for dtype in frame.dtypes):
return None
# A custom ExtensionArray can advertise the literal NumPy dtype while its
# Arrow hook emits other values. Accept only pandas' concrete NumPy storage.
try:
from pandas.core.arrays.numpy_ import NumpyExtensionArray
except ImportError:
try:
from pandas.core.arrays.numpy_ import PandasArray as NumpyExtensionArray
except ImportError:
return None
columns = [frame.iloc[:, i] for i in range(width)]
if any(type(column) is not Series or type(column.array) is not NumpyExtensionArray
for column in columns):
return None
# Native row conversion reads Rust fields, bypassing mutable Python accessors.
for name in ("DoubleType", "StructField", "StructType"):
cls = getattr(native, name)
if (getattr(sql_types, name, None) is not cls
or cls.__getattribute__ is not object.__getattribute__
or any("__class__" in vars(base) for base in cls.__mro__ if base is not object)):
return None
for cls, names, descriptor_type in (
(native.StructType, ("fields", "names"), GetSetDescriptorType),
(native.StructType, ("__iter__",), WrapperDescriptorType),
(native.StructField, ("name", "dataType", "nullable", "metadata"), GetSetDescriptorType),
):
for name in names:
descriptor = vars(cls).get(name)
if (type(descriptor) is not descriptor_type
or descriptor.__objclass__ is not cls or descriptor.__name__ != name):
return None
fields = schema.fields
if len(fields) != width or not all(
type(field) is native.StructField and type(field.dataType) is native.DoubleType
and type(field.name) is str and "\x00" not in field.name for field in fields
) or len(set(schema.names)) != width:
return None

# Use the same pandas missing-value semantics as the row path, including the
# legacy use_inf_as_na option. Infinities and signed zero otherwise stay intact.
valid = frame.notna()
if any(not field.nullable and not valid.iloc[:, i].all()
for i, field in enumerate(fields)):
return None

import pyarrow as pa

return pa.Table.from_arrays(
[pa.array(column, type=pa.float64(),
mask=~valid.iloc[:, i], from_pandas=False)
for i, column in enumerate(columns)],
# Match the row serializer's physical fields. The complete Spark
# metadata remains in the explicit logical schema, not Arrow metadata.
schema=pa.schema([pa.field(field.name, pa.float64(), nullable=field.nullable)
for field in fields]),
)


def prepare_float64_input(frame, schema):
"""Serialize supported double columns with the existing row transport policy."""
table = prepare_float64_table(frame, schema)
if table is None:
return None

import pyarrow as pa

sink = pa.BufferOutputStream()
with pa.ipc.new_stream(sink, table.schema) as writer:
writer.write_table(table)
return sink.getvalue().to_pybytes()
Loading
Loading