Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
dc29a19
[SPARK-60676][CONNECT] Use Arrow columns for signed int64 pandas inputs
dbtsai Oct 29, 2026
3e79f91
[SPARK-60676][CONNECT] Validate integer upload scales against mock RPC
dbtsai Oct 29, 2026
8c705c0
[SPARK-60676][CONNECT] Cover leading null inference and non-nullable …
dbtsai Oct 29, 2026
fb996d9
[SPARK-60676][CONNECT] Exercise named schemas through mock rename rel…
dbtsai Oct 29, 2026
79d3faf
[SPARK-60676][CONNECT] Avoid Arrow configuration overhead for small N…
dbtsai Oct 29, 2026
e266e8e
[SPARK-60676][CONNECT] Retain IPC row fallback for NUL field names
dbtsai Oct 29, 2026
9942f21
[SPARK-60676][CONNECT] Include pandas regressions in the existing Arr…
dbtsai Oct 29, 2026
5edd700
[SPARK-60676][CONNECT] Preserve integer pandas schema and fallback se…
dbtsai Oct 29, 2026
080aa16
[SPARK-60676][TESTS] Preserve subclass column labels on pandas 3
dbtsai Oct 30, 2026
abbc23a
[SPARK-60676][CONNECT] Preserve customized pandas columns hooks
dbtsai Oct 30, 2026
7961bde
[SPARK-60676][CONNECT] Preserve rebound pandas index storage ordering
dbtsai Oct 30, 2026
3e3a8f5
Preserve integer pandas fallback with missing Arrow aliases
dbtsai Oct 31, 2026
f058ac9
[SPARK-60676][TESTS] Retain current master coverage after rebase
dbtsai Nov 5, 2026
121a141
Preserve current pandas label iteration and fallback order after rebase
dbtsai Nov 5, 2026
aca1d02
Keep canonical string storage guard module identity
dbtsai Nov 5, 2026
ecdd67a
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
e7ae71a
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
40e5266
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
280bdee
[SPARK-60727][CONNECT] Keep canonical double input on inline transport
dbtsai Oct 30, 2026
11ff3b7
[SPARK-60727][CONNECT] Normalize Float helper terminal newline
dbtsai Oct 30, 2026
b9498d1
Reconcile current master interfaces and formatting after rebase
dbtsai Nov 5, 2026
2bd4638
Align float label regression tests with current native iteration
dbtsai Nov 5, 2026
26be6ab
[SPARK-60720][CONNECT] Encode guarded pandas String input inline
dbtsai Oct 30, 2026
09d0b13
[SPARK-60720][CONNECT] Remove extra EOF blank lines
dbtsai Oct 30, 2026
4304062
Reconcile current master interfaces and formatting after rebase
dbtsai Nov 5, 2026
b2274e4
Align string label regression tests with current native iteration
dbtsai Nov 5, 2026
cfde2b6
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
e31aad6
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
b415494
Merge remote-tracking branch 'apache/master' into SPARK-60727-pandas-…
HyukjinKwon Nov 6, 2026
489a663
[SPARK-60727][CONNECT] Drop the converter guards that the float64 pat…
HyukjinKwon Nov 6, 2026
9fbcd15
[SPARK-60727][CONNECT] Do not pin the reads of master's pandas row pa…
HyukjinKwon Nov 6, 2026
121a7a8
Merge branch 'SPARK-60727-pandas-float64' into SPARK-60720-pandas-str…
HyukjinKwon Nov 6, 2026
6eca4a4
[SPARK-60720][CONNECT] Encode each pandas string once, and drop the c…
HyukjinKwon Nov 6, 2026
6104f54
[SPARK-60720][CONNECT] Do not pin the reads of master's pandas row pa…
HyukjinKwon Nov 6, 2026
a31846b
Merge remote-tracking branch 'apache/master' into SPARK-60727-pandas-…
HyukjinKwon Nov 6, 2026
74bf14b
Merge branch 'SPARK-60727-pandas-float64' into SPARK-60720-pandas-str…
HyukjinKwon Nov 6, 2026
8b24c32
Merge remote-tracking branch 'apache/master' into SPARK-60720-pandas-…
HyukjinKwon Nov 6, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions crates/pyspark-rs/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ mod group;
mod infer;
mod ml;
mod observation;
mod pandas_input;
mod pipelines;
mod profiler;
mod pyarrow_export;
Expand Down Expand Up @@ -130,6 +131,8 @@ fn _pyspark(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_function(wrap_pyfunction!(types::parse_datatype_json, m)?)?;
m.add_function(wrap_pyfunction!(column::column_reference_name, m)?)?;

m.add_function(wrap_pyfunction!(pandas_input::_pandas_string_ipc, m)?)?;

// Spark Declarative Pipelines (SDP) command execution
m.add_class::<pipelines::PyPipelineRunStream>()?;
m.add_function(wrap_pyfunction!(
Expand Down
98 changes: 98 additions & 0 deletions crates/pyspark-rs/src/pandas_input.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

//! Fused validation and inline encoding of guarded pandas string columns.

use std::sync::Arc;

use arrow::array::{ArrayRef, StringBuilder};
use arrow::datatypes::{DataType as ArrowType, Field, Schema};
use arrow::ipc::writer::StreamWriter;
use arrow::record_batch::RecordBatch;
use pyo3::prelude::*;
use pyo3::types::{PyBytes, PyString};

use crate::types::PyStructType;

/// The Python eligibility helper authenticates pandas storage and native schema.
/// Validate each object before accepting it: extraction must never call a custom
/// numeric/string conversion hook or turn a pandas missing sentinel into null.
#[pyfunction]
pub(crate) fn _pandas_string_ipc<'py>(
py: Python<'py>,
columns: Vec<Bound<'py, PyAny>>,
schema: PyRef<'py, PyStructType>,
) -> PyResult<Option<Bound<'py, PyBytes>>> {
if columns.len() != schema.fields.len() {
return Ok(None);
}
// Reject unsupported objects before copying any string data. Otherwise a
// late subclass would pay for most of an Arrow build before the row path.
// Keep this pass to exact type/null checks; UTF-8 extraction stays below.
for (column, field) in columns.iter().zip(&schema.fields) {
if !matches!(&field.data_type, spark_connect::types::DataType::String { collation }
if collation == "UTF8_BINARY")
{
return Ok(None);
}
for value in column.try_iter()? {
let value = value?;
if !(value.is_none() && field.nullable || value.is_exact_instance_of::<PyString>()) {
return Ok(None);
}
}
}
// Encode each string once, while building. A lone surrogate cannot be encoded: it
// falls back to the row path, which drops the partial Arrow buffers. The type pass
// above keeps a late subclass from paying for the build.
let mut arrays: Vec<ArrayRef> = Vec::with_capacity(columns.len());
let mut fields = Vec::with_capacity(columns.len());
for (column, field) in columns.iter().zip(&schema.fields) {
let mut builder = StringBuilder::with_capacity(column.len()?, column.len()? * 8);
for value in column.try_iter()? {
let value = value?;
if value.is_none() && field.nullable {
builder.append_null();
} else if let Ok(value) = value.cast_exact::<PyString>() {
let Ok(bytes) = value.encode_utf8() else {
return Ok(None);
};
let Ok(text) = std::str::from_utf8(bytes.as_bytes()) else {
return Ok(None);
};
// append_value copies while the bytes object and GIL are live.
// This avoids to_cow's owned String on the abi3-py39 build.
builder.append_value(text);
} else {
return Ok(None);
}
}
arrays.push(Arc::new(builder.finish()));
// Like the row encoder, Spark metadata lives in the logical schema;
// Arrow fields carry the name, standard string type and nullability.
fields.push(Field::new(&field.name, ArrowType::Utf8, field.nullable));
}
let bytes = py
.detach(move || -> Result<Vec<u8>, arrow::error::ArrowError> {
let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), arrays)?;
let mut writer = StreamWriter::try_new(Vec::new(), &batch.schema())?;
writer.write(&batch)?;
writer.into_inner()
})
.map_err(|error| pyo3::exceptions::PyRuntimeError::new_err(error.to_string()))?;
Ok(Some(PyBytes::new(py, &bytes)))
}
25 changes: 25 additions & 0 deletions crates/pyspark-rs/src/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -896,6 +896,31 @@ impl PySparkSession {
frame.to_df(fields.iter().map(|field| field.name.as_str()).collect()),
));
}
let ipc = py
.import("pyspark.sql._pandas_input")?
.getattr("prepare_string_input")?
.call1((data, schema))?;
if !ipc.is_none() {
let fields = match &spec {
Spec::Struct(fields) => fields,
_ => {
return Err(pyo3::exceptions::PyValueError::new_err(
"string input requires a struct schema",
));
}
};
let schema = DataType::Struct {
fields: fields.clone(),
};
let frame = self
.session
.create_dataframe_from_inline_arrow_ipc(ipc.extract()?, schema);
// Match pandas row input: explicit nonempty field names travel
// through toDF as well as the local relation's logical schema.
return Ok(PyDataFrame::new(
frame.to_df(fields.iter().map(|field| field.name.as_str()).collect()),
));
}
return self.create_from_pandas(py, data, spec);
}
// Preserve schema parsing, ignored-argument warnings, and pandas/Arrow routing.
Expand Down
99 changes: 99 additions & 0 deletions python/pyspark/sql/_pandas_input.py
Original file line number Diff line number Diff line change
Expand Up @@ -530,3 +530,102 @@ def prepare_float64_input(frame, schema):
with pa.ipc.new_stream(sink, table.schema) as writer:
writer.write_table(table)
return sink.getvalue().to_pybytes()


def prepare_string_input(frame, schema):
"""Prepare inline IPC for explicit columns of exact strings or None.

Inspect every object before Arrow conversion: str subclasses can override
conversion hooks, and pandas treats several other objects as missing values.
Those cases must keep the existing row conversion and its error ordering.
"""
import numpy as np
import pandas as pd
from pandas.core import frame as frame_module, generic as generic_module
from pandas.core.frame import DataFrame
from pandas.core.series import Series
from types import FunctionType, GetSetDescriptorType, MethodDescriptorType, WrapperDescriptorType

from pyspark._pyspark import StringType, StructField, StructType as NativeStructType
from pyspark.sql import types as sql_types

# Keep small frames on the established row path.
if (pd.DataFrame is not DataFrame or type(frame) is not DataFrame
or type(schema) is not NativeStructType
or len(frame) < 10000
or any(name in vars(frame) for name in ("astype", "notna", "where"))):
return None
# Preserve the direct normalization hooks used by the existing row path.
for name, module in (("astype", generic_module), ("notna", frame_module),
("where", generic_module)):
method = getattr(DataFrame, name)
if (type(method) is not FunctionType or method.__name__ != name
or method.__module__ != module.__name__ or method.__globals__ is not vars(module)):
return None
if (DataFrame.astype is not generic_module.NDFrame.astype
or DataFrame.where is not generic_module.NDFrame.where):
return None
# The row path reads native fields directly. Class attributes are mutable,
# so customized Python schema access must not select a different Arrow path.
for name, cls in (("StringType", StringType), ("StructField", StructField),
("StructType", NativeStructType)):
if (getattr(sql_types, name, None) is not cls
or cls.__getattribute__ is not object.__getattribute__
or any("__class__" in vars(base) for base in cls.__mro__ if base is not object)):
return None
for cls, names, descriptor_type in (
(NativeStructType, ("fields", "names"), GetSetDescriptorType),
(NativeStructType, ("__iter__",), WrapperDescriptorType),
(StructField, ("name", "dataType", "nullable", "metadata"), GetSetDescriptorType),
(StringType, ("isUTF8BinaryCollation",), MethodDescriptorType),
):
for name in names:
descriptor = vars(cls).get(name)
if (type(descriptor) is not descriptor_type
or descriptor.__objclass__ is not cls or descriptor.__name__ != name):
return None
if not _has_canonical_columns(frame.columns):
return None
width = len(frame.columns)
if not width or len(schema.fields) != width or not frame.columns.is_unique:
return None
if not all(type(name) is str and "\x00" not in name for name in frame.columns):
return None
if not all(
type(field) is StructField and type(field.dataType) is StringType
and field.dataType.isUTF8BinaryCollation()
and type(field.name) is str and "\x00" not in field.name
for field in schema.fields
) or len(set(schema.names)) != width:
return None
if not all(type(dtype) is type(np.dtype("object")) and dtype == np.dtype("object")
for dtype in frame.dtypes):
return None
# A custom ExtensionArray may report a genuine NumPy dtype while its
# Arrow hook emits different values from to_numpy(). Trust only pandas'
# concrete NumPy backing array, including its name in older pandas releases.
try:
from pandas.core.arrays.numpy_ import NumpyExtensionArray
except ImportError:
try:
from pandas.core.arrays.numpy_ import PandasArray as NumpyExtensionArray
except ImportError:
return None
columns = [frame.iloc[:, i] for i in range(width)]
if any(type(column) is not Series or type(column.array) is not NumpyExtensionArray
for column in columns):
return None

from pyspark._pyspark import _pandas_string_ipc

# Preflight exact object types before the native Arrow buffer traversal.
# Unsupported values, non-nullable None, and surrogates return to the row
# path before serialization. Keep its inline transport and configuration
# behavior rather than selecting the public Arrow import/upload policy.
arrays = [column.array._ndarray for column in columns]
if any(type(array) is not np.ndarray or array.ndim != 1
or type(array.dtype) is not type(np.dtype("object"))
or array.dtype != np.dtype("object") or len(array) != len(frame)
for array in arrays):
return None
return _pandas_string_ipc(arrays, schema)
Loading
Loading