diff --git a/crates/pyspark-rs/src/lib.rs b/crates/pyspark-rs/src/lib.rs index 15796178..807fce24 100644 --- a/crates/pyspark-rs/src/lib.rs +++ b/crates/pyspark-rs/src/lib.rs @@ -36,6 +36,7 @@ mod group; mod infer; mod ml; mod observation; +mod pandas_input; mod pipelines; mod profiler; mod pyarrow_export; @@ -130,6 +131,8 @@ fn _pyspark(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(types::parse_datatype_json, m)?)?; m.add_function(wrap_pyfunction!(column::column_reference_name, m)?)?; + m.add_function(wrap_pyfunction!(pandas_input::_pandas_string_ipc, m)?)?; + // Spark Declarative Pipelines (SDP) command execution m.add_class::()?; m.add_function(wrap_pyfunction!( diff --git a/crates/pyspark-rs/src/pandas_input.rs b/crates/pyspark-rs/src/pandas_input.rs new file mode 100644 index 00000000..ab6eb8fc --- /dev/null +++ b/crates/pyspark-rs/src/pandas_input.rs @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Fused validation and inline encoding of guarded pandas string columns. + +use std::sync::Arc; + +use arrow::array::{ArrayRef, StringBuilder}; +use arrow::datatypes::{DataType as ArrowType, Field, Schema}; +use arrow::ipc::writer::StreamWriter; +use arrow::record_batch::RecordBatch; +use pyo3::prelude::*; +use pyo3::types::{PyBytes, PyString}; + +use crate::types::PyStructType; + +/// The Python eligibility helper authenticates pandas storage and native schema. +/// Validate each object before accepting it: extraction must never call a custom +/// numeric/string conversion hook or turn a pandas missing sentinel into null. +#[pyfunction] +pub(crate) fn _pandas_string_ipc<'py>( + py: Python<'py>, + columns: Vec>, + schema: PyRef<'py, PyStructType>, +) -> PyResult>> { + if columns.len() != schema.fields.len() { + return Ok(None); + } + // Reject unsupported objects before copying any string data. Otherwise a + // late subclass would pay for most of an Arrow build before the row path. + // Keep this pass to exact type/null checks; UTF-8 extraction stays below. + for (column, field) in columns.iter().zip(&schema.fields) { + if !matches!(&field.data_type, spark_connect::types::DataType::String { collation } + if collation == "UTF8_BINARY") + { + return Ok(None); + } + for value in column.try_iter()? { + let value = value?; + if !(value.is_none() && field.nullable || value.is_exact_instance_of::()) { + return Ok(None); + } + } + } + // A lone surrogate must fall back before allocating partial Arrow buffers. + // Drop each temporary UTF-8 bytes object immediately rather than retaining + // an additional full copy of every column. Keep the cheap type pass above + // so a late subclass does not pay for this UTF-8 traversal. + for (column, field) in columns.iter().zip(&schema.fields) { + for value in column.try_iter()? { + let value = value?; + if value.is_none() && field.nullable { + continue; + } + let Ok(value) = value.cast_exact::() else { + return Ok(None); + }; + if value.encode_utf8().is_err() { + return Ok(None); + } + } + } + let mut arrays: Vec = Vec::with_capacity(columns.len()); + let mut fields = Vec::with_capacity(columns.len()); + for (column, field) in columns.iter().zip(&schema.fields) { + let mut builder = StringBuilder::with_capacity(column.len()?, column.len()? * 8); + for value in column.try_iter()? { + let value = value?; + if value.is_none() && field.nullable { + builder.append_null(); + } else if let Ok(value) = value.cast_exact::() { + let Ok(bytes) = value.encode_utf8() else { + return Ok(None); + }; + let Ok(text) = std::str::from_utf8(bytes.as_bytes()) else { + return Ok(None); + }; + // append_value copies while the bytes object and GIL are live. + // This avoids to_cow's owned String on the abi3-py39 build. + builder.append_value(text); + } else { + return Ok(None); + } + } + arrays.push(Arc::new(builder.finish())); + // Like the row encoder, Spark metadata lives in the logical schema; + // Arrow fields carry the name, standard string type and nullability. + fields.push(Field::new(&field.name, ArrowType::Utf8, field.nullable)); + } + let bytes = py + .detach(move || -> Result, arrow::error::ArrowError> { + let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), arrays)?; + let mut writer = StreamWriter::try_new(Vec::new(), &batch.schema())?; + writer.write(&batch)?; + writer.into_inner() + }) + .map_err(|error| pyo3::exceptions::PyRuntimeError::new_err(error.to_string()))?; + Ok(Some(PyBytes::new(py, &bytes))) +} diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs index 4b9ce5c5..8345a595 100644 --- a/crates/pyspark-rs/src/session.rs +++ b/crates/pyspark-rs/src/session.rs @@ -765,6 +765,54 @@ impl PySparkSession { } let spec = schema_spec(py, schema)?; if is_loaded_instance(py, data, "pandas", "DataFrame")? { + // Guarded integer columns share the existing Arrow importer. Schema + // validation above and unsupported pandas conversions keep master's + // normal ordering and row fallback. + let table = py + .import("pyspark.sql._pandas_input")? + .getattr("prepare_int64_table")? + .call1((data, schema))?; + if !table.is_none() { + let frame = + crate::pyarrow_import::create_dataframe(py, &self.session, &table, schema)?; + // pandas applies explicit field names with toDF even though the + // local relation already carries them. Empty input stays local. + return Ok(match &spec { + Spec::Struct(fields) if !fields.is_empty() && table.len()? > 0 => { + PyDataFrame::new( + frame + .dataframe + .to_df(fields.iter().map(|field| field.name.as_str()).collect()), + ) + } + _ => frame, + }); + } + let ipc = py + .import("pyspark.sql._pandas_input")? + .getattr("prepare_string_input")? + .call1((data, schema))?; + if !ipc.is_none() { + let fields = match &spec { + Spec::Struct(fields) => fields, + _ => { + return Err(pyo3::exceptions::PyValueError::new_err( + "string input requires a struct schema", + )); + } + }; + let schema = DataType::Struct { + fields: fields.clone(), + }; + let frame = self + .session + .create_dataframe_from_inline_arrow_ipc(ipc.extract()?, schema); + // Match pandas row input: explicit nonempty field names travel + // through toDF as well as the local relation's logical schema. + return Ok(PyDataFrame::new( + frame.to_df(fields.iter().map(|field| field.name.as_str()).collect()), + )); + } return self.create_from_pandas(py, data, spec); } self.create_from_local(py, data, spec) diff --git a/crates/spark-connect/src/session.rs b/crates/spark-connect/src/session.rs index 96c3b1f6..e7290132 100644 --- a/crates/spark-connect/src/session.rs +++ b/crates/spark-connect/src/session.rs @@ -360,6 +360,23 @@ impl SparkSession { Ok(DataFrame::new(self.clone(), plan)) } + /// Internal binding bridge for Arrow IPC already validated and encoded from + /// local rows or columns. Retains the row path's inline transport policy. + #[doc(hidden)] + pub fn create_dataframe_from_inline_arrow_ipc( + &self, + data: Vec, + schema: DataType, + ) -> DataFrame { + DataFrame::new( + self.clone(), + LogicalPlan::LocalRelation { + schema: Some(schema), + data: Some(data), + }, + ) + } + /// Create a local DataFrame from Arrow batches without materializing rows. /// /// `schema` optionally supplies the Spark schema applied by the server. With diff --git a/docs/architecture.md b/docs/architecture.md index 5cce8782..089232b3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -142,6 +142,16 @@ timestamp conversion helpers normalize the table, then arrow-rs imports its those batches to IPC for the local relation sent to Spark. No Python rows are materialized; casts, buffer alignment, and IPC serialization may still copy. Empty tables require an explicit schema, matching PySpark Connect. +Pandas inputs with only NumPy `int64` or pandas nullable `Int64` columns also +use this importer, preserving integer precision and null masks without Python +rows. Small frames (fewer than 1,000 rows) with only NumPy columns retain the +row path to avoid configuration RPC overhead. Nullable `Int64` columns always +use Arrow so schema inference does not depend on the first value being non-null. +This initial path accepts inferred schemas, complete string column names, +and explicit `StructType` schemas whose fields are all `LongType`. Other dtypes, +DDL/atomic schemas, schema conversions, NUL names, and duplicate/non-string input names +retain the existing row conversion path. `verifySchema` remains ignored for +pandas Arrow input, matching Spark Connect. Inputs at or above `spark.sql.session.localRelationCacheThreshold` are serialized into independent IPC streams and uploaded as session-scoped cache artifacts. The plan carries `ChunkedCachedLocalRelation` hashes instead of embedding the diff --git a/python/pyspark/sql/_pandas_input.py b/python/pyspark/sql/_pandas_input.py new file mode 100644 index 00000000..1c6ba1d6 --- /dev/null +++ b/python/pyspark/sql/_pandas_input.py @@ -0,0 +1,616 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +"""Conservative columnar pandas input; unsupported conversions use the row path.""" + + +def _has_canonical_columns(columns): + """Decline label containers whose row-path iteration can run custom hooks.""" + import sys + from types import FunctionType + + import numpy as np + import pandas as pd + from pandas.core import base as base_module + from pandas.core.indexes import base as index_module + + Index = index_module.Index + if (pd.Index is not Index or type(columns) is not Index + or Index.__getattribute__ is not object.__getattribute__ + or "__iter__" in vars(columns)): + return False + + def ordinary(method, name, module): + return (type(method) is FunctionType and method.__name__ == name + and method.__module__ == module.__name__ + and method.__globals__ is vars(module)) + + # The native row path iterates columns before normalizing any cells. + # Decline overrides without invoking them, so fallback preserves that order. + if (Index.__iter__ is not base_module.IndexOpsMixin.__iter__ + or not ordinary(Index.__iter__, "__iter__", base_module)): + return False + values = vars(Index).get("_values") + if (type(values) is not property + or not ordinary(values.fget, "_values", index_module)): + return False + if any("_data" in vars(base) for base in Index.__mro__): + return False + backing = vars(columns).get("_data") + if type(backing) is np.ndarray: + return True + + # Arrow-backed labels already hold PyArrow storage. Do not import optional + # Arrow modules just to decide whether a fallback input is eligible. + pa = sys.modules.get("pyarrow") + if pa is None: + return False + + try: + from pandas.core.arrays import numpy_ as numpy_module, string_ as string_module + from pandas.core.arrays import base as array_base_module, _mixins as mixins_module + from pandas._libs.arrays import NDArrayBacked + from pandas.core.arrays import string_arrow as string_arrow_module + from pandas.core.arrays.arrow import array as arrow_module + except ImportError: + # Older pandas releases may lack these concrete string storage types. + return False + + python_strings = tuple(vars(string_module).get(name) for name in + ("StringArray", "StringArrayNumpySemantics")) + arrow_strings = tuple(vars(string_arrow_module).get(name) for name in + ("ArrowStringArray", "ArrowStringArrayNumpySemantics")) + cls = type(backing) + if not any(cls is candidate for candidate in python_strings + arrow_strings): + return False + if (cls.__getattribute__ is not object.__getattribute__ + or any(name in vars(backing) for name in + ("__iter__", "__getitem__", "__len__", "_box_func", + "tolist", "to_numpy", "isna", "fillna", "_hasna")) + or getattr(cls, "tolist", None) is not getattr( + vars(string_module).get("BaseStringArray"), "tolist", None) + or not ordinary(getattr(cls, "tolist", None), "tolist", string_module)): + return False + if any(cls is candidate for candidate in python_strings): + owner, module = vars(numpy_module).get("NumpyExtensionArray"), numpy_module + string_array = vars(string_module).get("StringArray") + if owner is None or string_array is None: + return False + try: + storage = object.__getattribute__(backing, "_ndarray") + except AttributeError: + return False + if type(storage) is not np.ndarray: + return False + # ExtensionArray iteration dispatches through len and indexed boxing. + for name, hook_module in (("__iter__", array_base_module), + ("__getitem__", mixins_module), ("_box_func", mixins_module)): + if not ordinary(getattr(cls, name, None), name, hook_module): + return False + if (getattr(cls, "__len__", None) is not NDArrayBacked.__len__ + or getattr(cls, "ndim", None) is not NDArrayBacked.ndim + or any("_ndarray" in vars(base) for base in cls.__mro__ + if base is not NDArrayBacked)): + return False + isna = vars(string_array).get("isna", getattr(owner, "isna", None)) + isna_module = string_module if "isna" in vars(string_array) else module + else: + owner, module = vars(arrow_module).get("ArrowExtensionArray"), arrow_module + if owner is None: + return False + try: + storage = object.__getattribute__(backing, "_pa_array") + except AttributeError: + return False + if (type(storage) is not pa.ChunkedArray + or not ordinary(getattr(cls, "__iter__", None), "__iter__", module)): + return False + isna, isna_module = getattr(owner, "isna", None), module + hasna = getattr(cls, "_hasna", None) + if (type(hasna) is not property + or not ordinary(hasna.fget, "_hasna", module) + or getattr(cls, "fillna", None) is not getattr(owner, "fillna", None) + or not ordinary(getattr(cls, "fillna", None), "fillna", module)): + return False + # Keep the existing storage/conversion authentication as well as the + # iterator hooks: these objects also supply labels to pandas normalization. + return (getattr(cls, "to_numpy", None) is getattr(owner, "to_numpy", None) + and ordinary(getattr(cls, "to_numpy", None), "to_numpy", module) + and getattr(cls, "isna", None) is isna + and ordinary(getattr(cls, "isna", None), "isna", isna_module)) + + +def prepare_int64_table(frame, schema): + """Return an Arrow table only when each column is already a signed 64-bit integer. + + In particular, never coerce a mixed pandas frame to a common NumPy dtype: + doing so can round integers above 2**53. Other dtypes and schema conversions + deliberately retain the existing row path until their semantics are covered. + ``verifySchema`` is ignored for pandas Arrow input, as in Spark Connect. + """ + import sys + from types import GetSetDescriptorType, WrapperDescriptorType + + import numpy as np + import pandas as pd + try: + from pandas.core.arrays.integer import Int64Dtype, IntegerArray + from pandas.core.arrays.numpy_ import NumpyExtensionArray + from pandas.core.frame import DataFrame + from pandas.core.series import Series + except ImportError: + # Older supported pandas versions call the NumPy wrapper PandasArray. + # Keep their existing row conversion when these concrete types are absent. + return None + + from pyspark._pyspark import ( + AtomicType, BooleanType, ByteType, ShortType, IntegerType, LongType, StructField, StructType, + ) + from pyspark.sql import types as sql_types + + # A subclass or custom extension array can change the values exposed to Arrow. + # Check concrete identities before consulting dtype equality or conversion hooks. + if pd.DataFrame is not DataFrame or type(frame) is not DataFrame: + return None + + if not _has_canonical_columns(frame.columns): + return None + width = len(frame.columns) + if not width or not frame.columns.is_unique or not all( + type(name) is str and "\x00" not in name for name in frame.columns + ): + return None + dtypes = frame.dtypes + numpy_dtype = np.dtype("int64") + if not all((type(dtype) is type(numpy_dtype) and dtype == numpy_dtype) + or type(dtype) is Int64Dtype + for dtype in dtypes): + return None + # The importer's configuration RPCs outweigh row conversion for small NumPy + # frames. Nullable extension columns still use Arrow for reliable null typing. + if len(frame) < 1000 and not any(type(dtype) is Int64Dtype for dtype in dtypes): + return None + + # The native row path reads native schema fields directly. The Arrow converter + # instead uses Python descriptors and imported class aliases, including aliases + # captured before this helper was loaded. Customized schemas retain the row path. + if sys.modules.get("pyspark.sql.types") is not sql_types: + return None + importer = sys.modules.get("pyspark.sql._arrow_input") + if vars(sql_types).get("AtomicType") is not AtomicType: + return None + if importer is not None: + if getattr(importer, "StructType", None) is not StructType: + return None + # The importer uses this alias while validating schema widths. + if vars(importer).get("AtomicType") is not AtomicType: + return None + converter = sys.modules.get("pyspark.sql.pandas.types") + namespaces = [vars(sql_types)] + if converter is not None: + namespaces.append(vars(converter)) + if importer is not None: + # An importer may still hold functions from a removed converter module. + for name in ("from_arrow_schema", "to_arrow_schema"): + # Later importers resolve these converters locally inside prepare_table. + if name not in vars(importer): + continue + namespace = getattr(getattr(importer, name, None), "__globals__", None) + if type(namespace) is not dict: + return None + namespaces.append(namespace) + for name, cls in ( + ("BooleanType", BooleanType), ("ByteType", ByteType), ("ShortType", ShortType), + ("IntegerType", IntegerType), ("LongType", LongType), + ("StructField", StructField), ("StructType", StructType), + ): + if any(namespace.get(name) is not cls for namespace in namespaces): + return None + for cls in (LongType, StructField, StructType): + if (cls.__getattribute__ is not object.__getattribute__ + or any("__class__" in vars(base) for base in cls.__mro__ if base is not object)): + return None + for cls, names, descriptor_type in ( + (StructType, ("fields", "names"), GetSetDescriptorType), + (StructType, ("__iter__",), WrapperDescriptorType), + (StructField, ("name", "dataType", "nullable", "metadata"), GetSetDescriptorType), + ): + for name in names: + descriptor = vars(cls).get(name) + if (type(descriptor) is not descriptor_type + or descriptor.__objclass__ is not cls or descriptor.__name__ != name): + return None + + names = list(frame.columns) + fields = None + if type(schema) is StructType: + fields = schema.fields + if len(fields) != width or not all( + type(field) is StructField and type(field.dataType) is LongType + and type(field.name) is str and "\x00" not in field.name + for field in fields + ): + return None + names = schema.names + if len(set(names)) != width: + return None + elif schema is not None: + if type(schema) not in (list, tuple) or len(schema) != width or not all( + type(name) is str and "\x00" not in name for name in schema + ): + return None + + columns = [frame.iloc[:, i] for i in range(width)] + for i, (column, dtype) in enumerate(zip(columns, dtypes)): + if type(column) is not Series: + return None + expected = IntegerArray if type(dtype) is Int64Dtype else NumpyExtensionArray + if type(column.array) is not expected: + return None + # The native row path rejects invalid nullability before any Config RPC. + if (fields is not None and not fields[i].nullable + and type(dtype) is Int64Dtype and column.hasnans): + return None + + import pyarrow as pa + + # Arrow consumes each Series' integer buffer and nullable mask independently. + return pa.Table.from_arrays( + [pa.array(column, type=pa.int64(), from_pandas=True) for column in columns], + names=names, + ) + + +def prepare_string_input(frame, schema): + """Prepare inline IPC for eligible object or Arrow-backed string columns. + + Inspect every object before Arrow conversion: str subclasses can override + conversion hooks, and pandas treats several other objects as missing values. + Those cases must keep the existing row conversion and its error ordering. + """ + import sys + + import numpy as np + import pandas as pd + from pandas.core import frame as frame_module, generic as generic_module + from pandas.core.frame import DataFrame + from pandas.core.series import Series + from types import FunctionType, GetSetDescriptorType, MethodDescriptorType, WrapperDescriptorType + + from pyspark._pyspark import ( + BooleanType, ByteType, ShortType, IntegerType, LongType, FloatType, + DoubleType, DecimalType, StringType, StructField, StructType as NativeStructType, + ) + from pyspark.sql import types as sql_types + + # Keep small frames on the established row path. + if (pd.DataFrame is not DataFrame or type(frame) is not DataFrame + or type(schema) is not NativeStructType + or len(frame) < 10000 + or any(name in vars(frame) for name in ("astype", "notna", "where"))): + return None + # Preserve the direct normalization hooks used by the existing row path. + for name, module in (("astype", generic_module), ("notna", frame_module), + ("where", generic_module)): + method = getattr(DataFrame, name) + if (type(method) is not FunctionType or method.__name__ != name + or method.__module__ != module.__name__ or method.__globals__ is not vars(module)): + return None + if (DataFrame.astype is not generic_module.NDFrame.astype + or DataFrame.where is not generic_module.NDFrame.where): + return None + # Earlier converter branches must not recognize a string as another type. + # Check a loaded converter too: it keeps its own imported type aliases. + if sys.modules.get("pyspark.sql.types") is not sql_types: + return None + importer = sys.modules.get("pyspark.sql._arrow_input") + if importer is not None and getattr(importer, "StructType", None) is not NativeStructType: + return None + converter = sys.modules.get("pyspark.sql.pandas.types") + for name, cls in ( + ("BooleanType", BooleanType), ("ByteType", ByteType), ("ShortType", ShortType), + ("IntegerType", IntegerType), ("LongType", LongType), ("FloatType", FloatType), + ("DoubleType", DoubleType), ("DecimalType", DecimalType), ("StringType", StringType), + ("StructField", StructField), ("StructType", NativeStructType), + ): + if (getattr(sql_types, name, None) is not cls + or converter is not None and getattr(converter, name, None) is not cls): + return None + # The row path reads native fields directly. Class attributes are mutable, + # so customized Python schema access must not select a different Arrow path. + for name, cls in (("StringType", StringType), ("StructField", StructField), + ("StructType", NativeStructType)): + if (getattr(sql_types, name, None) is not cls + or cls.__getattribute__ is not object.__getattribute__ + or any("__class__" in vars(base) for base in cls.__mro__ if base is not object)): + return None + for cls, names, descriptor_type in ( + (NativeStructType, ("fields", "names"), GetSetDescriptorType), + (NativeStructType, ("__iter__",), WrapperDescriptorType), + (StructField, ("name", "dataType", "nullable", "metadata"), GetSetDescriptorType), + (StringType, ("isUTF8BinaryCollation",), MethodDescriptorType), + ): + for name in names: + descriptor = vars(cls).get(name) + if (type(descriptor) is not descriptor_type + or descriptor.__objclass__ is not cls or descriptor.__name__ != name): + return None + if not _has_canonical_columns(frame.columns): + return None + width = len(frame.columns) + if not width or len(schema.fields) != width or not frame.columns.is_unique: + return None + if not all(type(name) is str and "\x00" not in name for name in frame.columns): + return None + if not all( + type(field) is StructField and type(field.dataType) is StringType + and field.dataType.isUTF8BinaryCollation() + and type(field.name) is str and "\x00" not in field.name + for field in schema.fields + ) or len(set(schema.names)) != width: + return None + dtypes = frame.dtypes + if not all(type(dtype) is type(np.dtype("object")) and dtype == np.dtype("object") + for dtype in dtypes): + return _prepare_arrow_string_input(frame, schema, dtypes) + # A custom ExtensionArray may report a genuine NumPy dtype while its + # Arrow hook emits different values from to_numpy(). Trust only pandas' + # concrete NumPy backing array, including its name in older pandas releases. + try: + from pandas.core.arrays.numpy_ import NumpyExtensionArray + except ImportError: + try: + from pandas.core.arrays.numpy_ import PandasArray as NumpyExtensionArray + except ImportError: + return None + columns = [frame.iloc[:, i] for i in range(width)] + if any(type(column) is not Series or type(column.array) is not NumpyExtensionArray + for column in columns): + return None + + from pyspark._pyspark import _pandas_string_ipc + + # Preflight exact object types before the native Arrow buffer traversal. + # Unsupported values, non-nullable None, and surrogates return to the row + # path before serialization. Keep its inline transport and configuration + # behavior rather than selecting the public Arrow import/upload policy. + arrays = [column.array._ndarray for column in columns] + if any(type(array) is not np.ndarray or array.ndim != 1 + or type(array.dtype) is not type(np.dtype("object")) + or array.dtype != np.dtype("object") or len(array) != len(frame) + for array in arrays): + return None + return _pandas_string_ipc(arrays, schema) + + + +def _prepare_arrow_string_input(frame, schema, dtypes): + """Encode canonical Arrow string storage after the common String guards. + + Bypass neither pandas conversion hooks nor its missing-value policy. Inspect + concrete storage directly only when the hooks used by row normalization are + ordinary pandas implementations. Other extension arrays keep the row path. + """ + from types import FunctionType + + import numpy as np + import pandas as pd + try: + import pyarrow as pa + from pyarrow import lib as arrow_lib + from pandas.core.arrays import base as base_module + from pandas.core.arrays import string_ as string_module + from pandas.core.arrays import string_arrow as string_arrow_module + from pandas.core.arrays.arrow import array as arrow_module + from pandas.core import frame as frame_module, series as series_module + from pandas.core.dtypes import base as dtype_module + from pandas.core.internals import blocks as blocks_module, managers as managers_module + from pandas._libs import internals as native_internals, missing as missing_module + except ImportError: + return None + + StringDtype = string_module.StringDtype + ArrowStringArray = string_arrow_module.ArrowStringArray + ArrowExtensionArray = arrow_module.ArrowExtensionArray + ExtensionArray = base_module.ExtensionArray + Series = series_module.Series + if (pd.StringDtype is not StringDtype or pd.Series is not Series + or pd.NA is not missing_module.NA + or pa.ChunkedArray is not arrow_lib.ChunkedArray + or pa.StringArray is not arrow_lib.StringArray + or pa.LargeStringArray is not arrow_lib.LargeStringArray + or not all(type(dtype) is StringDtype for dtype in dtypes)): + return None + + def ordinary(method, name, module): + return (type(method) is FunctionType and method.__name__ == name + and method.__module__ == module.__name__ + and method.__globals__ is vars(module)) + + def property_hook(cls, name, module): + descriptor = getattr(cls, name, None) + return (type(descriptor) is property and descriptor.fset is None + and descriptor.fdel is None + and ordinary(descriptor.fget, name, module)) + + # astype(object) delegates through ExtensionArray.astype, __array__ and + # to_numpy. Nullable conversion also uses isna, _hasna and __getitem__. + hooks = { + "astype": (ArrowStringArray, string_arrow_module), + "__array__": (ArrowExtensionArray, arrow_module), + "to_numpy": (ArrowExtensionArray, arrow_module), + "isna": (ArrowExtensionArray, arrow_module), + "fillna": (ArrowExtensionArray, arrow_module), + "__getitem__": (ArrowExtensionArray, arrow_module), + } + if (any(name in vars(frame) for name in ("isna",)) + or not ordinary(frame_module.DataFrame.isna, "isna", frame_module) + or not ordinary(ExtensionArray.astype, "astype", base_module) + or not property_hook(Series, "array", series_module) + or Series.__getattribute__ is not object.__getattribute__): + return None + for name, module in (("name", string_module), ("na_value", string_module), + ("type", string_module), ("kind", dtype_module)): + if not property_hook(StringDtype, name, module): + return None + # pandas 2 stores storage as a plain instance value; pandas 3 uses a property. + storage_descriptor = vars(StringDtype).get("storage") + if storage_descriptor is not None and not property_hook(StringDtype, "storage", string_module): + return None + for name, module in (("__eq__", string_module), ("__ne__", dtype_module), + ("__from_arrow__", string_module)): + if not ordinary(getattr(StringDtype, name, None), name, module): + return None + if (StringDtype.__getattribute__ is not object.__getattribute__ + or StringDtype.__setattr__ is not object.__setattr__ + or StringDtype.__new__ is not object.__new__ + or not ordinary(StringDtype.__init__, "__init__", string_module) + or string_arrow_module.StringDtype is not StringDtype + or arrow_module.StringDtype is not StringDtype): + return None + dtype_hooks = ("name", "na_value", "type", "kind", "__eq__", "__ne__", "__from_arrow__") + for dtype in dtypes: + state = vars(dtype) + if any(name in state for name in dtype_hooks): + return None + for name in ("_na_value", "_storage"): + if any(name in vars(base) for base in StringDtype.__mro__): + return None + storage = state.get("_storage") if storage_descriptor is not None else state.get("storage") + missing = state.get("_na_value") + if (type(storage) is not str or storage != "pyarrow" + or not (missing is pd.NA or type(missing) is float and np.isnan(missing))): + return None + + if (pd.arrays.ArrowStringArray is not ArrowStringArray + or ArrowStringArray.__name__ != "ArrowStringArray" + or ArrowStringArray.__qualname__ != "ArrowStringArray" + or ArrowStringArray.__module__ not in (string_arrow_module.__name__, "pandas.arrays")): + return None + numpy_strings = vars(string_arrow_module).get("ArrowStringArrayNumpySemantics") + if numpy_strings is not None and ( + not isinstance(numpy_strings, type) + or numpy_strings.__name__ != "ArrowStringArrayNumpySemantics" + or numpy_strings.__qualname__ != "ArrowStringArrayNumpySemantics" + or numpy_strings.__module__ != string_arrow_module.__name__ + or numpy_strings.__bases__ != (ArrowStringArray,) + ): + return None + classes = (ArrowStringArray, numpy_strings) + # Read authenticated manager storage without constructing a Series: iloc + # calls len(backing) before the concrete array type can be checked. A custom + # extension subclass must reach row normalization before running that hook. + manager = vars(frame).get("_mgr") + BlockManager, ExtensionBlock = managers_module.BlockManager, blocks_module.ExtensionBlock + if (type(BlockManager) is not type + or BlockManager.__name__ != "BlockManager" + or BlockManager.__qualname__ != "BlockManager" + or BlockManager.__module__ != managers_module.__name__ + or BlockManager.__bases__ != (native_internals.BlockManager, managers_module.BaseBlockManager) + or type(ExtensionBlock) is not type + or ExtensionBlock.__name__ != "ExtensionBlock" + or ExtensionBlock.__qualname__ != "ExtensionBlock" + or ExtensionBlock.__module__ != blocks_module.__name__ + or ExtensionBlock.__bases__ != (blocks_module.EABackedBlock,) + or any("_mgr" in vars(base) for base in type(frame).__mro__) + or type(manager) is not BlockManager + or BlockManager.__getattribute__ is not object.__getattribute__ + or not ordinary(BlockManager.iget_values, "iget_values", managers_module) + or not ordinary(ExtensionBlock.iget, "iget", blocks_module) + or not property_hook(BlockManager, "blknos", managers_module) + or not property_hook(BlockManager, "blklocs", managers_module) + or BlockManager.blocks is not native_internals.BlockManager.blocks + or ExtensionBlock.values is not native_internals.Block.values + or BlockManager._rebuild_blknos_and_blklocs is not + native_internals.BlockManager._rebuild_blknos_and_blklocs + or any(name in vars(manager) for name in + ("iget_values", "blknos", "blklocs", "_rebuild_blknos_and_blklocs"))): + return None + if (ExtensionBlock.__getattribute__ is not object.__getattribute__ + or any(type(block) is not ExtensionBlock or "iget" in vars(block) + for block in manager.blocks)): + return None + # The canonical properties can expose customized cached index arrays. + # Decline subclasses before iget_values can invoke their indexing hooks. + for cache in (manager.blknos, manager.blklocs): + if (type(cache) is not np.ndarray or cache.ndim != 1 + or cache.dtype != np.dtype("intp") or len(cache) != len(schema.fields)): + return None + arrays = [] + for i, field in enumerate(schema.fields): + array = manager.iget_values(i) + cls = type(array) + if (not any(cls is candidate for candidate in classes) + or cls.__getattribute__ is not object.__getattribute__ + or cls.__setattr__ is not object.__setattr__ + or cls.__new__ is not object.__new__ + or any("__class__" in vars(base) for base in cls.__mro__ if base is not object)): + return None + state = vars(array) + if any(name in state for name in (*hooks, "dtype", "_hasna", "__len__", + "_from_pyarrow_array", "__init__")): + return None + for name, (owner, module) in hooks.items(): + method = getattr(cls, name, None) + if method is not getattr(owner, name, None) or not ordinary(method, name, module): + return None + if (not property_hook(cls, "dtype", string_arrow_module) + or not property_hook(cls, "_hasna", arrow_module) + or not ordinary(cls.__init__, "__init__", string_arrow_module) + or not ordinary(ArrowExtensionArray.__init__, "__init__", arrow_module)): + return None + length_owner = ArrowStringArray if "__len__" in vars(ArrowStringArray) else ArrowExtensionArray + length_module = string_arrow_module if length_owner is ArrowStringArray else arrow_module + if not ordinary(getattr(cls, "__len__", None), "__len__", length_module): + return None + has_factory = any("_from_pyarrow_array" in vars(base) for base in cls.__mro__) + if storage_descriptor is not None and not has_factory: + return None + if has_factory: + factory = getattr(cls, "_from_pyarrow_array", None) + if not ordinary(factory, "_from_pyarrow_array", string_arrow_module): + return None + # Never invoke a descriptor installed over either private backing field. + if any(name in vars(base) for base in cls.__mro__ for name in ("_pa_array", "_dtype")): + return None + backing, dtype = state.get("_pa_array"), state.get("_dtype") + if (type(backing) is not arrow_lib.ChunkedArray or dtype is not dtypes.iloc[i] + or len(backing) != len(frame) + or backing.type not in (pa.string(), pa.large_string()) + or any(type(chunk) not in (arrow_lib.StringArray, arrow_lib.LargeStringArray) + for chunk in backing.chunks)): + return None + try: + # Full validation rejects malformed offsets and UTF-8 before either + # casting or IPC can bypass the row path's decoding error/order. + backing.validate(full=True) + if not field.nullable and backing.null_count: + return None + arrays.append(backing.cast(pa.string(), safe=True)) + except (pa.ArrowException, OverflowError): + return None + + fields = [pa.field(field.name, pa.string(), nullable=field.nullable) + for field in schema.fields] + table = pa.Table.from_arrays(arrays, schema=pa.schema(fields)) + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, table.schema) as writer: + writer.write_table(table) + # Keep the existing inline LocalRelation bridge: the public Table importer + # consults configuration and can upload artifacts before returning a plan. + return sink.getvalue().to_pybytes() diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index 459a65e1..c0bc3803 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -26,17 +26,19 @@ import subprocess from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager -from datetime import datetime, timezone +from datetime import date, datetime, timezone from decimal import Decimal from pathlib import Path from urllib.parse import quote import grpc +import numpy as np import pandas as pd import pyarrow as pa import pytest from google.protobuf import descriptor_pb2, descriptor_pool, message_factory -from pyspark.sql import SparkSession +from pyspark.sql import SparkSession, types as T +from pyspark.sql._pandas_input import prepare_int64_table @pytest.fixture(scope="module") @@ -403,6 +405,7 @@ def _arrow_input_service( protocol, *, upload_error=None, + config_error=False, user_id=None, user_agent=None, allow_file_artifacts=False, @@ -424,7 +427,8 @@ def _arrow_input_service( } config.update(overrides) state = {"cache": {}, "uploads": 0, "relations": [], "message_sizes": [], - "config_requests": [], "upload_contexts": [], "status_contexts": []} + "config_requests": [], "input_tables": [], + "upload_contexts": [], "status_contexts": []} server_id = "00000000-0000-0000-0000-000000000001" def attribution(request, context): @@ -446,6 +450,8 @@ def configure(raw, context): operation = request.operation.WhichOneof("op_type") keys = getattr(request.operation, operation).keys state["config_requests"].append((operation, list(keys))) + if config_error: + context.abort(grpc.StatusCode.UNAVAILABLE, "configuration unavailable") return protocol("ConfigResponse", session_id=request.session_id, pairs=[dict(key=k, value=config[k]) for k in keys if config.get(k) is not None] @@ -509,6 +515,9 @@ def execute(raw, context): state["column_names"] = list(relation.to_df.column_names) relation = relation.to_df.input state["relations"].append(relation) + # Column-name schemas wrap the local input with a rename relation. + if relation.HasField("to_df"): + relation = relation.to_df.input if relation.HasField("chunked_cached_local_relation"): cached = relation.chunked_cached_local_relation payloads = [state["cache"]["cache/" + h] for h in cached.dataHashes] @@ -523,6 +532,7 @@ def execute(raw, context): if relation.local_relation.HasField("schema") else None ) rows, first, last = 0, None, None + batches = [] state["arrow_schemas"], state["batch_rows"] = [], [] for payload in payloads: reader = pa.ipc.open_stream(payload) @@ -531,12 +541,14 @@ def execute(raw, context): state["batch_rows"].append(stream_rows) for batch in reader: stream_rows.append(batch.num_rows) + batches.append(batch) rows += batch.num_rows if batch.num_rows: if first is None: first = batch.column(0)[0].as_py() last = batch.column(0)[-1].as_py() state["result"] = (rows, first, last) + state["input_tables"].append(pa.Table.from_batches(batches)) result = pa.table({"rows": [rows]}) sink = pa.BufferOutputStream() with pa.ipc.new_stream(sink, result.schema) as writer: @@ -889,3 +901,1771 @@ def import_module(name, *args, **kwargs): spark.createDataFrame(data) assert imported == [] assert state["config_requests"] == [] + + +# Signed-int64 pandas input uses the same Arrow importer and mock service. + +@pytest.mark.parametrize("storage", ["object", "default", "python", "pyarrow", "python_nan", "pyarrow_nan"]) +@pytest.mark.parametrize("dtype", ["int64", "Int64"]) +def test_integer_canonical_columns_storage_stays_columnar(protocol, monkeypatch, storage, dtype): + from pyspark.sql._pandas_input import _has_canonical_columns + + values = [2**53 + 1, -(2**63), 2**63 - 1, 0] * 250 + if dtype == "Int64": + values[1] = None + frame = pd.DataFrame({"original": pd.Series(values, dtype=dtype)}) + if storage == "object": + frame.columns = pd.Index(["original"], dtype=object) + elif storage != "default": + kind = storage.removesuffix("_nan") + label_dtype = (pd.StringDtype(storage=kind, na_value=np.nan) + if storage.endswith("_nan") else pd.StringDtype(storage=kind)) + frame.columns = pd.Index(pd.array(["original"], dtype=label_dtype)) + assert _has_canonical_columns(frame.columns) + assert prepare_int64_table(frame, None).column(0).to_pylist() == values + + def no_rows(*args, **kwargs): + raise AssertionError("ordinary columns must retain integer columnar construction") + + monkeypatch.setattr(pd.DataFrame, "astype", no_rows) + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(frame).toArrow() + assert state["input_tables"][0].column(0).to_pylist() == values + assert state["config_requests"] + + +@pytest.mark.parametrize("dtype", ["int64", "Int64"]) +@pytest.mark.parametrize("schema_kind", ["inferred", "names", "explicit"]) +@pytest.mark.parametrize("location", [ + "subclass", "class", "owner", "backing_subclass_item", "backing_class_iter", + "backing_class_getitem", "backing_instance_box", +]) +@pytest.mark.parametrize("behavior", ["raise", "mutate"]) +def test_integer_columns_iterator_hooks_keep_native_order( + protocol, monkeypatch, dtype, schema_kind, location, behavior +): + from pandas.core.base import IndexOpsMixin + + values = [2**53 + 1, -(2**63), 2**63 - 1, 0] * 250 + if dtype == "Int64": + values[1] = None + frame = pd.DataFrame({"original": pd.Series(values, dtype=dtype)}) + schema = None + expected_name = "original" + nullable = True + metadata = {} + if schema_kind == "names": + schema = ["renamed"] + expected_name = "renamed" + elif schema_kind == "explicit": + nullable, metadata = dtype == "Int64", {"source": "native"} + schema = T.StructType([T.StructField("renamed", T.LongType(), nullable, metadata)]) + expected_name = "renamed" + original_iter = pd.Index.__iter__ + original_notna, original_astype = frame.notna, frame.astype + calls = [] + invoked = False + + class ColumnsError(RuntimeError): + pass + + def notna(*args, **kwargs): + calls.append("notna") + return original_notna(*args, **kwargs) + + def astype(*args, **kwargs): + calls.append("astype") + return original_astype(*args, **kwargs) + + def before_hook(): + nonlocal invoked + if invoked: + return + invoked = True + calls.append("columns iterator") + if behavior == "raise": + raise ColumnsError("columns hook precedes pandas normalization") + # Install spies only when native iteration reaches the columns hook. + # Before eligibility, normalization methods are the canonical methods. + monkeypatch.setitem(vars(frame), "notna", notna) + monkeypatch.setitem(vars(frame), "astype", astype) + frame.iat[0, 0] = 2**53 + 7 + frame.iat[-1, 0] = None if dtype == "Int64" else -(2**63) + 1 + + def iterate(index): + if index is columns: + before_hook() + return original_iter(index) + + if location == "subclass": + class Columns(pd.Index): + pass + monkeypatch.setattr(Columns, "__iter__", iterate) + labels = pd.Index(["original"], dtype=object) + frame.columns = Columns._simple_new(labels._values, name=None) + assert type(frame.columns) is Columns + elif location == "backing_subclass_item": + # IndexOpsMixin iteration calls ndarray.item, rather than ndarray.__iter__. + class Names(np.ndarray): + def item(self, *args): + before_hook() + return super().item(*args) + backing = np.asarray(["original"], dtype=object).view(Names) + frame.columns = pd.Index._simple_new(backing, name=None) + assert type(frame.columns._values) is Names + elif location.startswith("backing_"): + frame.columns = pd.Index(pd.array(["original"], dtype=pd.StringDtype(storage="python"))) + backing = frame.columns._values + if location == "backing_class_iter": + original = type(backing).__iter__ + + def backing_iter(array): + if array is backing: + before_hook() + return original(array) + + monkeypatch.setattr(type(backing), "__iter__", backing_iter) + elif location == "backing_class_getitem": + original = type(backing).__getitem__ + + def backing_getitem(array, key): + if array is backing: + before_hook() + return original(array, key) + + monkeypatch.setattr(type(backing), "__getitem__", backing_getitem) + else: + original = backing._box_func + + def box(value): + before_hook() + return original(value) + + monkeypatch.setitem(vars(backing), "_box_func", box) + else: + monkeypatch.setattr(pd.Index if location == "class" else IndexOpsMixin, + "__iter__", iterate) + columns = frame.columns + + with _arrow_input_service(protocol) as (spark, state): + if behavior == "raise": + with pytest.raises(ColumnsError, + match="columns hook precedes pandas normalization"): + spark.createDataFrame(frame, schema) + assert calls == ["columns iterator"] + assert state["config_requests"] == [] + assert state["input_tables"] == [] + else: + spark.createDataFrame(frame, schema).toArrow() + expected = values.copy() + expected[0], expected[-1] = 2**53 + 7, (None if dtype == "Int64" else -(2**63) + 1) + table = state["input_tables"][0] + assert table.num_rows == 1000 + assert table.column_names == [expected_name] + assert table.column(0).type == pa.int64() + assert table.column(0).to_pylist() == expected + assert table.column(0).null_count == sum(value is None for value in expected) + assert table.schema.field(0).nullable is nullable + assert table.schema.field(0).metadata is None + import json + logical = json.loads(state["relations"][0].local_relation.schema) + assert logical == {"type": "struct", "fields": [{ + "name": expected_name, "type": "long", "nullable": nullable, + "metadata": metadata, + }]} + assert state["config_requests"] == [] + assert calls == ["columns iterator", "notna", "astype"] + + +def test_integer_rebound_index_values_keeps_native_error(protocol, monkeypatch): + frame = pd.DataFrame({"original": np.arange(1000, dtype=np.int64)}) + schema = T.StructType([T.StructField("renamed", T.LongType())]) + columns = frame.columns + original = pd.Index._values.fget + calls = [] + + def values(index): + if index is columns: + calls.append("_values") + raise RuntimeError("columns backing hook precedes normalization") + return original(index) + + monkeypatch.setattr(pd.Index, "_values", property(values)) + with _arrow_input_service(protocol) as (spark, state): + with pytest.raises(RuntimeError, + match="columns backing hook precedes normalization"): + spark.createDataFrame(frame, schema) + assert calls == ["_values"] + assert state["config_requests"] == [] + assert state["input_tables"] == [] + + +@pytest.mark.parametrize("schema_kind", ["inferred", "names", "explicit"]) +@pytest.mark.parametrize("verify", [True, False]) +def test_integer_columns_preserve_values_without_python_rows(protocol, monkeypatch, schema_kind, verify): + frame = pd.DataFrame({ + "a": pd.Series([2**53 + 1, None, -(2**63)], dtype="Int64"), + "b": np.array([2**63 - 1, -1, 0], dtype=np.int64), + }) + schema = None + names = ["a", "b"] + if schema_kind == "names": + schema = names = ["renamed_a", "renamed_b"] + elif schema_kind == "explicit": + schema = T.StructType([T.StructField("a", T.LongType()), + T.StructField("b", T.LongType(), False, {"source": "test"})]) + import pyspark.sql._arrow_input as helper + original = helper.prepare_table + observed = [] + + def capture(*args): + result = original(*args) + observed.append(result) + return result + + def no_rows(*args, **kwargs): + raise AssertionError("supported pandas input must not materialize Python rows") + + monkeypatch.setattr(helper, "prepare_table", capture) + monkeypatch.setattr(pd.DataFrame, "astype", no_rows) + with _arrow_input_service(protocol) as (spark, state): + df = spark.createDataFrame(frame, schema=schema, verifySchema=verify) + # Every nonempty pandas input applies its names through toDF, including + # explicit struct schemas, just as the current row fallback does. + assert not df.isLocal() + assert df.toArrow().column(0).to_pylist() == [3] + assert state["column_names"] == names + assert state["result"] == (3, 2**53 + 1, -(2**63)) + table, resolved, renamed = observed[0] + assert table.to_pydict() == {"a": [2**53 + 1, None, -(2**63)], + "b": [2**63 - 1, -1, 0]} + assert (renamed or resolved.names) == names + if schema_kind == "explicit": + assert resolved == schema + assert not table.schema.field(1).nullable + assert resolved.fields[1].metadata == {"source": "test"} + + +@pytest.mark.parametrize("dtype", ["int64", "Int64"]) +def test_empty_integer_frame_requires_explicit_schema(protocol, dtype): + frame = pd.DataFrame({"a": pd.Series([], dtype=dtype)}) + with _arrow_input_service(protocol) as (spark, state): + with pytest.raises(ValueError): + spark.createDataFrame(frame) + df = spark.createDataFrame(frame, T.StructType([T.StructField("a", T.LongType())])) + assert df.isLocal() + + +@pytest.mark.parametrize("values", [ + [1.0, float("nan")], [True, False], ["x", None], + [datetime(2024, 1, 1), None], + [datetime(2024, 1, 1, tzinfo=timezone.utc), None], + [date(2024, 1, 1), None], [Decimal("1.20"), None], + [[1, 2], None], [{"a": 1}, None], +]) +def test_other_dtypes_keep_row_fallback(values): + frame = pd.DataFrame({"a": values}) + assert prepare_int64_table(frame, None) is None + assert prepare_int64_table(frame, T.StructType([T.StructField("a", T.LongType())])) is None + + +@pytest.mark.parametrize("schema", [ + "a long", T.LongType(), ["a", "b"], [1], + T.StructType([T.StructField("a", T.IntegerType())]), + T.StructType([T.StructField("a", T.DoubleType())]), + T.StructType([]), +]) +def test_schema_conversions_keep_row_fallback(schema): + assert prepare_int64_table(pd.DataFrame({"a": [1, 2]}), schema) is None + + +def test_duplicate_names_fallback_and_nullable_masks(): + frame = pd.DataFrame([[2**53 + 1, 2**53 + 3]], columns=["x", "x"], dtype="int64") + assert prepare_int64_table(frame, None) is None + assert prepare_int64_table(pd.DataFrame({0: [1]}), None) is None + frame = pd.DataFrame({"a": pd.Series([None, None], dtype="Int64")}) + assert prepare_int64_table(frame, None).column(0).to_pylist() == [None, None] + + +@pytest.mark.parametrize("dtype", ["int32", "uint64", "Int32", "UInt64", "int64[pyarrow]"]) +def test_other_integer_dtypes_keep_row_fallback(dtype): + assert prepare_int64_table(pd.DataFrame({"a": pd.Series([1, 2], dtype=dtype)}), None) is None + + +@pytest.mark.parametrize("rows", [100, 1000, 100000, 1000000]) +@pytest.mark.parametrize("explicit", [False, True]) +def test_integer_scales_reach_mock_service(protocol, monkeypatch, rows, explicit): + values = np.arange(rows, dtype=np.int64) + (2**53 + 1) + frame = pd.DataFrame({name: values for name in ("a", "b", "c")}) + schema = T.StructType([T.StructField(name, T.LongType()) for name in frame.columns]) + + def no_rows(*args, **kwargs): + raise AssertionError("integer input must stay columnar at every scale") + + if rows >= 1000: + monkeypatch.setattr(pd.DataFrame, "astype", no_rows) + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(frame, schema if explicit else None).toArrow() + assert state["result"] == (rows, 2**53 + 1, 2**53 + rows) + if frame.memory_usage(index=False).sum() > 1048576: + assert state["relations"][0].HasField("chunked_cached_local_relation") + + +def test_nullable_inference_does_not_depend_on_first_value(protocol): + frame = pd.DataFrame({"a": pd.Series([None, 2**53 + 1], dtype="Int64")}) + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(frame).toArrow() + assert state["result"] == (2, None, 2**53 + 1) + + +@pytest.mark.parametrize("verify", [True, False]) +@pytest.mark.parametrize("config_error", [False, True]) +def test_non_nullable_schema_is_preserved(protocol, verify, config_error): + frame = pd.DataFrame({"a": pd.Series([1, None], dtype="Int64")}) + schema = T.StructType([T.StructField("a", T.LongType(), False)]) + assert prepare_int64_table(frame, schema) is None + with _arrow_input_service(protocol, config_error=config_error) as (spark, state): + with pytest.raises(RuntimeError, match="null"): + spark.createDataFrame(frame, schema, verifySchema=verify) + assert not state["relations"] + assert not state["uploads"] + assert state["config_requests"] == [] + + +def test_small_numpy_frames_avoid_importer_configuration_overhead(): + assert prepare_int64_table(pd.DataFrame({"a": np.arange(999, dtype=np.int64)}), None) is None + assert prepare_int64_table(pd.DataFrame({"a": np.arange(1000, dtype=np.int64)}), None) is not None + assert prepare_int64_table(pd.DataFrame({"a": pd.Series([None], dtype="Int64")}), None) is not None + + +@pytest.mark.parametrize("schema_kind", ["inferred", "names", "explicit"]) +def test_nul_names_keep_lossless_ipc_row_fallback(protocol, schema_kind): + import pyarrow as pa + + name = "a\x00b" + frame = pd.DataFrame({name if schema_kind == "inferred" else "original": + np.arange(1000, dtype=np.int64)}) + schema = None + if schema_kind == "names": + schema = [name] + elif schema_kind == "explicit": + schema = T.StructType([T.StructField(name, T.LongType())]) + # C Stream field names are NUL-terminated; IPC preserves the full string. + assert prepare_int64_table(frame, schema) is None + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + assert state["result"] == (1000, 0, 999) + payload = state["relations"][0].local_relation.data + assert pa.ipc.open_stream(payload).schema.names == [name] + + +@pytest.mark.parametrize("nullable", [False, True]) +@pytest.mark.parametrize("rows", [999, 1000]) +def test_integer_explicit_renamed_schema_preserves_all_ipc_cells(protocol, monkeypatch, nullable, rows): + values = ([2**53 + 1, -(2**63), 2**63 - 1, 0] * 250)[:rows] + other = [None if nullable and i % 7 == 0 else value + for i, value in enumerate(reversed(values))] + frame = pd.DataFrame({"original": np.asarray(values, dtype="int64"), + "second": pd.Series(other, dtype="Int64" if nullable else "int64")}) + schema = T.StructType([ + T.StructField("renamed", T.LongType(), False, {"source": "native", "id": 3}), + T.StructField("nullable", T.LongType(), True, {"tag": [1, 2]}), + ]) + + def no_rows(*args, **kwargs): + raise AssertionError("trusted integer columns must stay columnar") + + if rows >= 1000 or nullable: + monkeypatch.setattr(pd.DataFrame, "astype", no_rows) + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + table = state["input_tables"][0] + assert table.column_names == ["renamed", "nullable"] + assert [column.to_pylist() for column in table.columns] == [values, other] + assert table.schema.field(0).type == pa.int64() + assert table.schema.field(0).nullable is False + assert table.schema.field(1).nullable is True + if rows >= 1000 or nullable: + from pyspark.sql.pandas.types import from_arrow_schema + assert from_arrow_schema(table.schema).json() == schema.json(), ( + from_arrow_schema(table.schema).json(), schema.json(), table.schema) + assert state["config_requests"] + else: + assert state["config_requests"] == [] + + +def test_integer_dataframe_and_name_subclasses_keep_fallback(monkeypatch): + class Frame(pd.DataFrame): + pass + + class Name(str): + pass + + class Names(list): + pass + + frame = pd.DataFrame({"a": np.arange(1000, dtype="int64")}) + assert prepare_int64_table(Frame(frame), None) is None + assert prepare_int64_table(frame, Names(["a"])) is None + assert prepare_int64_table(frame, [Name("a")]) is None + # Pandas 3 normalizes constructor labels to plain strings by default. + # An object Index keeps the subclass this fallback regression exercises. + named_frame = frame.copy() + named_frame.columns = pd.Index([Name("a")], dtype=object) + assert type(named_frame.columns[0]) is Name + assert prepare_int64_table(named_frame, None) is None + monkeypatch.setattr(pd, "DataFrame", Frame) + assert prepare_int64_table(Frame(frame), None) is None + + +def test_integer_missing_internal_pandas_array_class_keeps_fallback(): + import sys + + script = r''' +import sys +import types +import numpy as np +import pandas as pd +from pyspark.sql._pandas_input import prepare_int64_table +frame = pd.DataFrame({"a": np.arange(1000, dtype="int64")}) +nullable = pd.DataFrame({"a": pd.Series([None, 1], dtype="Int64")}) +# Older pandas exposes PandasArray rather than NumpyExtensionArray. Missing +# optional internals must disable the optimization without breaking pandas input. +sys.modules["pandas.core.arrays.numpy_"] = types.ModuleType("pandas.core.arrays.numpy_") +assert prepare_int64_table(frame, None) is None +assert prepare_int64_table(nullable, None) is None +''' + result = subprocess.run([sys.executable, "-c", script], + capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.parametrize("class_spoof", [False, True]) +def test_integer_custom_extension_dtype_cannot_select_arrow(protocol, class_spoof): + from pandas.api.extensions import ExtensionArray, ExtensionDtype, take + + arrow_calls = [] + + class Int64EqualDtype(ExtensionDtype): + name = "int64_equal" + type = int + kind = "i" + na_value = None + + @classmethod + def construct_array_type(cls): + return DifferentArrowArray + + def __eq__(self, other): + return (isinstance(other, Int64EqualDtype) + or isinstance(other, np.dtype) and other == np.dtype("int64")) + + def __hash__(self): + return hash(self.name) + + class DifferentArrowArray(ExtensionArray): + def __init__(self, values): + self._values = np.asarray(values, dtype=object) + + @property + def dtype(self): + return Int64EqualDtype() + + @property + def nbytes(self): + return self._values.nbytes + + def __len__(self): + return len(self._values) + + def __getitem__(self, key): + result = self._values[key] + return type(self)(result) if isinstance(result, np.ndarray) else result + + def isna(self): + return pd.isna(self._values) + + def take(self, indices, allow_fill=False, fill_value=None): + return type(self)(take(self._values, indices, allow_fill=allow_fill, + fill_value=fill_value)) + + def copy(self): + return type(self)(self._values.copy()) + + @classmethod + def _from_sequence(cls, scalars, *, dtype=None, copy=False): + values = np.asarray(scalars, dtype=object) + return cls(values.copy() if copy else values) + + def to_numpy(self, dtype=None, copy=False, na_value=None): + if dtype is not None: + return self._values.astype(dtype, copy=copy) + return self._values.copy() if copy else self._values + + def __arrow_array__(self, type=None): + arrow_calls.append("arrow") + return pa.array([-7] * len(self), type=type) + + values = [2**53 + 1, None, -(2**63), 0] * 250 + frame = pd.DataFrame({"a": pd.Series(DifferentArrowArray(values), copy=False)}) + if class_spoof: + Int64EqualDtype.__getattribute__ = lambda self, name: ( + type(np.dtype("int64")) if name == "__class__" + else object.__getattribute__(self, name)) + assert frame.dtypes.iloc[0] == np.dtype("int64") + assert isinstance(frame.dtypes.iloc[0], np.dtype) is class_spoof + schema = T.StructType([T.StructField("renamed", T.LongType())]) + assert prepare_int64_table(frame, schema) is None + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + assert state["config_requests"] == [] + assert state["input_tables"][0].column(0).to_pylist() == values + assert arrow_calls == [] + + +@pytest.mark.parametrize("custom", ["dtype", "array"]) +def test_integer_nullable_dtype_and_array_subclasses_keep_fallback(protocol, custom): + from pandas.core.arrays.integer import IntegerArray, Int64Dtype + + arrow_calls = [] + + class CustomDtype(Int64Dtype): + pass + + class CustomArray(IntegerArray): + @property + def dtype(self): + if custom == "dtype": + return CustomDtype() + return Int64Dtype() + + def __arrow_array__(self, type=None): + arrow_calls.append("arrow") + return pa.array([-7] * len(self), type=type) + + values = [2**53 + 1, 0, -(2**63), 7] * 250 + array = CustomArray(np.asarray(values, dtype="int64"), np.zeros(1000, dtype=bool)) + frame = pd.DataFrame({"a": pd.Series(array, copy=False)}) + assert type(frame["a"].array) is CustomArray + schema = T.StructType([T.StructField("a", T.LongType())]) + assert prepare_int64_table(frame, schema) is None + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + assert state["config_requests"] == [] + assert state["input_tables"][0].column(0).to_pylist() == values + assert arrow_calls == [] + + +def _integer_guard_process(mode, argument, before_import=False): + """Class mutation and converter imports must be isolated from other tests.""" + import sys + + if mode == "captured_alias": + import pyspark.sql._arrow_input as importer + if not hasattr(importer, "to_arrow_schema"): + pytest.skip("this importer resolves converters locally and holds no captured aliases") + + script = r''' +import importlib +import importlib.util +import sys +import tempfile +from pathlib import Path +import numpy as np +import pandas as pd +from pyspark.sql import types as T +from pyspark import _pyspark as native + +spec = importlib.util.spec_from_file_location("integer_guard_capture", sys.argv[4]) +capture = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = capture +spec.loader.exec_module(capture) +values = [2**53 + 1, -(2**63), 2**63 - 1, 0] * 250 +frame = pd.DataFrame({"renamed": np.asarray(values, dtype="int64")}) +schema = T.StructType([T.StructField("renamed", T.LongType(), False, {"source": "native"})]) +mode, argument, before_import = sys.argv[1:4] +if before_import == "True": + sys.modules.pop("pyspark.sql._pandas_input", None) +if mode == "descriptor": + class_name, attribute = argument.split(":") + cls = getattr(native, class_name) + if attribute == "alias": + setattr(T, class_name, object) + elif attribute == "__getattribute__": + setattr(cls, attribute, lambda self, name: object.__getattribute__(self, name)) + elif attribute == "__class__": + setattr(cls, "__getattribute__", lambda self, name: ( + object if name == "__class__" else object.__getattribute__(self, name))) + elif attribute == "__iter__": + setattr(cls, attribute, lambda self: iter([T.StructField("changed", T.BooleanType())])) + else: + value = {"names": ["changed"], "fields": [], "name": "changed", + "dataType": T.BooleanType(), "nullable": True, + "metadata": {"changed": True}}[attribute] + setattr(cls, attribute, property(lambda self: value)) +elif mode in ("public_alias", "converter_alias", "captured_alias"): + replacement = native.BooleanType if argument == "LongType" else native.LongType + if mode == "public_alias": + sys.modules.pop("pyspark.sql.pandas.types", None) + assert "pyspark.sql.pandas.types" not in sys.modules + setattr(T, argument, replacement) + else: + converter = importlib.import_module("pyspark.sql.pandas.types") + if mode == "captured_alias": + importer = importlib.import_module("pyspark.sql._arrow_input") + setattr(converter, argument, replacement) + if mode == "captured_alias": + sys.modules.pop("pyspark.sql.pandas.types") + current = importlib.import_module("pyspark.sql.pandas.types") + assert current is not converter + assert importer.to_arrow_schema.__globals__ is vars(converter) +elif mode == "atomic_alias": + scope, replacement_name = argument.split(":") + replacement = None if replacement_name == "none" else native.LongType + if scope == "public_before": + sys.modules.pop("pyspark.sql._arrow_input", None) + T.AtomicType = replacement + elif scope == "public_after": + importlib.import_module("pyspark.sql._arrow_input") + T.AtomicType = replacement + else: + importer = importlib.import_module("pyspark.sql._arrow_input") + if replacement_name == "missing": + del importer.AtomicType + else: + importer.AtomicType = replacement +elif mode == "importer_alias": + importer = importlib.import_module("pyspark.sql._arrow_input") + importer.StructType = None if argument == "none" else native.LongType +else: + raise AssertionError(mode) +class Factory: + def mktemp(self, name): + return Path(tempfile.mkdtemp(prefix=name)) +with capture._arrow_input_service(capture.protocol.__wrapped__(Factory())) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + assert state["config_requests"] == [], state["config_requests"] + table = state["input_tables"][0] + assert table.schema.names == ["renamed"], table.schema + assert table.column(0).type == __import__("pyarrow").int64() + assert table.column(0).to_pylist() == values +''' + result = subprocess.run([sys.executable, "-c", script, mode, argument, + str(before_import), __file__], + capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.parametrize("class_name,attribute", [ + ("StructType", "fields"), ("StructType", "names"), ("StructType", "__iter__"), + ("StructField", "name"), ("StructField", "dataType"), ("StructField", "nullable"), + ("StructField", "metadata"), ("StructType", "__getattribute__"), + ("StructField", "__getattribute__"), ("LongType", "__getattribute__"), + ("StructType", "__class__"), ("StructField", "__class__"), ("LongType", "__class__"), + ("StructType", "alias"), ("StructField", "alias"), ("LongType", "alias"), +]) +@pytest.mark.parametrize("before_import", [False, True]) +def test_integer_custom_schema_descriptors_keep_native_row_path(class_name, attribute, before_import): + _integer_guard_process("descriptor", f"{class_name}:{attribute}", before_import) + + +@pytest.mark.parametrize("class_name", ["BooleanType", "ByteType", "ShortType", "IntegerType", "LongType"]) +@pytest.mark.parametrize("mode", ["public_alias", "converter_alias", "captured_alias"]) +def test_integer_rebound_converter_aliases_keep_native_row_path(class_name, mode): + _integer_guard_process(mode, class_name) + + +@pytest.mark.parametrize("replacement", ["none", "long"]) +def test_integer_rebound_importer_schema_alias_keeps_native_row_path(replacement): + _integer_guard_process("importer_alias", replacement) + + +@pytest.mark.parametrize("scope", ["public_before", "public_after", "importer"]) +@pytest.mark.parametrize("replacement", ["none", "long"]) +def test_integer_rebound_atomic_alias_keeps_native_row_path(scope, replacement): + _integer_guard_process("atomic_alias", f"{scope}:{replacement}") + + +def test_integer_deleted_importer_atomic_alias_keeps_native_row_path(): + _integer_guard_process("atomic_alias", "importer:missing") + + +def test_integer_warm_importer_without_global_converter_aliases(protocol, monkeypatch): + import pyspark.sql._arrow_input as importer + + original = importer.prepare_table + + def local_imports(table, schema, *configs): + # Newer importers import converters within prepare_table. Exercise that + # module shape even on the pinned PR118 dependency used by this branch. + from pyspark.sql.pandas.types import from_arrow_schema, to_arrow_schema + table, resolved, names = original(table, schema, *configs) + assert from_arrow_schema(table.schema).fields + assert to_arrow_schema(resolved).names == table.column_names + return table, resolved, names + + frame = pd.DataFrame({"a": np.arange(1000, dtype="int64")}) + schema = T.StructType([T.StructField("renamed", T.LongType())]) + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(pa.table({"warm": [1, 2]})).toArrow() + # Keep original's module globals available in the dependency importer; + # the full-stack validation separately uses the actual local-import helper. + import types + substitute = types.ModuleType(importer.__name__) + substitute.StructType = T.StructType + substitute.AtomicType = T.AtomicType + substitute.prepare_table = local_imports + monkeypatch.setitem(__import__("sys").modules, importer.__name__, substitute) + + def no_rows(*args, **kwargs): + raise AssertionError("warm integer input must stay columnar") + + monkeypatch.setattr(pd.DataFrame, "astype", no_rows) + for _ in range(3): + spark.createDataFrame(frame, schema).toArrow() + for table in state["input_tables"][1:]: + assert table.column_names == ["renamed"] + assert table.column(0).to_pylist() == list(range(1000)) + + + + + + + +@pytest.mark.parametrize("dtype", ["int64", "Int64"]) +@pytest.mark.parametrize("schema_kind", ["inferred", "names", "explicit"]) +def test_integer_rebound_index_data_keeps_native_read_order( + protocol, monkeypatch, dtype, schema_kind +): + import json + + values = [2**53 + 1, -(2**63), 2**63 - 1, 0] * 250 + if dtype == "Int64": + values[1] = None + frame = pd.DataFrame({"original": pd.Series(values, dtype=dtype)}) + schema, expected_name, nullable, metadata = None, "original", True, {} + if schema_kind == "names": + schema, expected_name = ["renamed"], "renamed" + elif schema_kind == "explicit": + expected_name, nullable, metadata = "renamed", dtype == "Int64", {"source": "native"} + schema = T.StructType([T.StructField(expected_name, T.LongType(), nullable, metadata)]) + columns, backing = frame.columns, vars(frame.columns)["_data"] + original_notna, original_astype = frame.notna, frame.astype + calls = [] + normalizing = False + allowed_reads = None + + class ColumnsDataError(RuntimeError): + pass + + def notna(*args, **kwargs): + nonlocal normalizing + normalizing = True + calls.append("notna") + return original_notna(*args, **kwargs) + + def astype(*args, **kwargs): + calls.append("astype") + return original_astype(*args, **kwargs) + + def data(index): + if index is columns: + if not normalizing: + calls.append("_data") + if allowed_reads is not None and calls.count("_data") > allowed_reads: + raise ColumnsDataError("eligibility added a columns storage read") + # Install spies only when the native columns storage hook runs. + # Canonical frame methods remain intact before eligibility. + monkeypatch.setitem(vars(frame), "notna", notna) + monkeypatch.setitem(vars(frame), "astype", astype) + return backing + return vars(index)["_data"] + + def set_data(index, value): + # Preserve ordinary storage initialization for indexes pandas creates + # while normalizing values; only reads of the input columns are hooked. + vars(index)["_data"] = value + + monkeypatch.setattr(pd.Index, "_data", property(data, set_data), raising=False) + import pyspark.sql._pandas_input as helper + + with _arrow_input_service(protocol) as (spark, state): + # Record current master's fallback rather than assuming the old tolist + # path's single read. Index iteration reads NumPy-backed _values three + # times, and extension-backed _values twice, before normalization. + with monkeypatch.context() as baseline: + baseline.setattr(helper, "prepare_int64_table", lambda *_: None) + spark.createDataFrame(frame, schema).toArrow() + baseline_calls = calls.copy() + allowed_reads = baseline_calls.index("notna") + assert allowed_reads > 0 + assert baseline_calls == ["_data"] * allowed_reads + ["notna", "astype"] + calls.clear() + normalizing = False + # Restore canonical methods before eligibility, just as for the first + # call. The storage hook installs them when native iteration starts. + monkeypatch.delitem(vars(frame), "notna") + monkeypatch.delitem(vars(frame), "astype") + spark.createDataFrame(frame, schema).toArrow() + assert calls == baseline_calls + assert not state["config_requests"] + assert len(state["input_tables"]) == 2 + table = state["input_tables"][1] + assert table.equals(state["input_tables"][0], check_metadata=True) + assert table.column(0).to_pylist() == values + assert table.column(0).null_count == sum(value is None for value in values) + assert table.schema.equals( + pa.schema([pa.field(expected_name, pa.int64(), nullable=nullable)]), + check_metadata=True, + ) + assert json.loads(state["relations"][0].local_relation.schema) == { + "type": "struct", "fields": [ + {"name": expected_name, "type": "long", "nullable": nullable, + "metadata": metadata} + ], + } + + +@pytest.mark.parametrize("location", [ + "subclass", "class", "owner", "backing_subclass_item", "backing_class_iter", + "backing_class_getitem", "backing_instance_box", "arrow_class_iter", +]) +@pytest.mark.parametrize("behavior", ["raise", "mutate"]) +def test_string_columns_iterator_hooks_keep_native_order( + protocol, monkeypatch, location, behavior +): + from pandas.core.base import IndexOpsMixin + + values = ["plain"] * 10000 + frame = pd.DataFrame({"original": values}, dtype=object) + schema = T.StructType([T.StructField("renamed", T.StringType())]) + original_iter = pd.Index.__iter__ + original_notna, original_astype = frame.notna, frame.astype + calls = [] + invoked = False + + class ColumnsError(RuntimeError): + pass + + def notna(*args, **kwargs): + calls.append("notna") + return original_notna(*args, **kwargs) + + def astype(*args, **kwargs): + calls.append("astype") + return original_astype(*args, **kwargs) + + def before_hook(): + nonlocal invoked + if invoked: + return + invoked = True + calls.append("columns iterator") + if behavior == "raise": + raise ColumnsError("columns hook precedes pandas normalization") + # Install spies only when native iteration reaches the columns hook. + # Before eligibility, normalization methods are the canonical methods. + monkeypatch.setitem(vars(frame), "notna", notna) + monkeypatch.setitem(vars(frame), "astype", astype) + frame.iat[0, 0], frame.iat[-1, 0] = "changed", None + + def iterate(index): + if index is columns: + before_hook() + return original_iter(index) + + if location == "subclass": + class Columns(pd.Index): + pass + monkeypatch.setattr(Columns, "__iter__", iterate) + labels = pd.Index(["original"], dtype=object) + frame.columns = Columns._simple_new(labels._values, name=None) + assert type(frame.columns) is Columns + elif location == "backing_subclass_item": + # IndexOpsMixin iteration uses ndarray.item, not ndarray.__iter__. + class Names(np.ndarray): + def item(self, *args): + before_hook() + return super().item(*args) + backing = np.asarray(["original"], dtype=object).view(Names) + frame.columns = pd.Index._simple_new(backing, name=None) + assert type(frame.columns._values) is Names + elif location.startswith(("backing_", "arrow_")): + storage = "pyarrow" if location.startswith("arrow_") else "python" + frame.columns = pd.Index(pd.array(["original"], dtype=pd.StringDtype(storage=storage))) + backing = frame.columns._values + if location.endswith("class_iter"): + original = type(backing).__iter__ + + def backing_iter(array): + if array is backing: + before_hook() + return original(array) + + monkeypatch.setattr(type(backing), "__iter__", backing_iter) + elif location.endswith("class_getitem"): + original = type(backing).__getitem__ + + def backing_getitem(array, key): + if array is backing: + before_hook() + return original(array, key) + + monkeypatch.setattr(type(backing), "__getitem__", backing_getitem) + else: + original = backing._box_func + + def box(value): + before_hook() + return original(value) + + monkeypatch.setitem(vars(backing), "_box_func", box) + else: + monkeypatch.setattr(pd.Index if location == "class" else IndexOpsMixin, + "__iter__", iterate) + columns = frame.columns + + with _arrow_input_service(protocol) as (spark, state): + if behavior == "raise": + with pytest.raises(ColumnsError, + match="columns hook precedes pandas normalization"): + spark.createDataFrame(frame, schema) + assert calls == ["columns iterator"] + assert state["config_requests"] == [] + assert state["input_tables"] == [] + assert state["uploads"] == 0 + else: + spark.createDataFrame(frame, schema).toArrow() + expected = values.copy() + expected[0], expected[-1] = "changed", None + import json + + table = state["input_tables"][0] + assert table.schema.names == ["renamed"] + assert table.column(0).type == pa.string() + assert table.column(0).to_pylist() == expected + assert table.column(0).is_null().to_pylist() == [False] * 9999 + [True] + assert table.schema.equals(pa.schema([pa.field("renamed", pa.string())]), + check_metadata=True) + assert json.loads(state["relations"][0].local_relation.schema) == json.loads(schema.json()) + assert state["config_requests"] == [] + assert state["uploads"] == 0 + assert calls == ["columns iterator", "notna", "astype"] + + + +def test_string_rebound_index_values_keeps_native_error(protocol, monkeypatch): + + frame = pd.DataFrame({"original": ["plain"] * 10000}, dtype=object) + schema = T.StructType([T.StructField("renamed", T.StringType())]) + columns = frame.columns + original = pd.Index._values.fget + calls = [] + + def values(index): + if index is columns: + calls.append("_values") + raise RuntimeError("columns backing hook precedes normalization") + return original(index) + + monkeypatch.setattr(pd.Index, "_values", property(values)) + assert calls == [] + with _arrow_input_service(protocol) as (spark, state): + with pytest.raises(RuntimeError, + match="columns backing hook precedes normalization"): + spark.createDataFrame(frame, schema) + assert calls == ["_values"] + assert state["config_requests"] == [] + assert state["input_tables"] == [] + + + +@pytest.mark.parametrize("nullable", [False, True]) +def test_string_rebound_index_data_keeps_native_read_order(protocol, monkeypatch, nullable): + import json + + values = ["plain", "é", "漢字", ""] * 2500 + if nullable: + values[1] = None + frame = pd.DataFrame({"original": values}, dtype=object) + schema = T.StructType([ + T.StructField("renamed", T.StringType(), nullable, {"source": "native"}) + ]) + columns, backing = frame.columns, vars(frame.columns)["_data"] + original_notna, original_astype = frame.notna, frame.astype + calls = [] + normalizing = False + allowed_reads = None + + class ColumnsDataError(RuntimeError): + pass + + def notna(*args, **kwargs): + nonlocal normalizing + normalizing = True + calls.append("notna") + return original_notna(*args, **kwargs) + + def astype(*args, **kwargs): + calls.append("astype") + return original_astype(*args, **kwargs) + + def data(index): + if index is columns: + if not normalizing: + calls.append("_data") + if allowed_reads is not None and calls.count("_data") > allowed_reads: + raise ColumnsDataError("eligibility added a columns storage read") + # Canonical normalization remains intact before eligibility. + monkeypatch.setitem(vars(frame), "notna", notna) + monkeypatch.setitem(vars(frame), "astype", astype) + return backing + return vars(index)["_data"] + + def set_data(index, value): + # Keep ordinary storage initialization for indexes created by pandas. + vars(index)["_data"] = value + + monkeypatch.setattr(pd.Index, "_data", property(data, set_data), raising=False) + import pyspark.sql._pandas_input as helper + + with _arrow_input_service(protocol) as (spark, state): + # Measure the current native fallback's storage reads, rather than the + # former tolist path's single-read contract. + with monkeypatch.context() as baseline: + baseline.setattr(helper, "prepare_string_input", lambda *_: None) + spark.createDataFrame(frame, schema).toArrow() + baseline_calls = calls.copy() + allowed_reads = baseline_calls.index("notna") + assert allowed_reads > 0 + assert baseline_calls == ["_data"] * allowed_reads + ["notna", "astype"] + calls.clear() + normalizing = False + # Restore canonical methods before the candidate eligibility check; + # native label iteration will install the spies through the storage hook. + monkeypatch.delitem(vars(frame), "notna") + monkeypatch.delitem(vars(frame), "astype") + spark.createDataFrame(frame, schema).toArrow() + assert calls == baseline_calls + assert not state["config_requests"] + assert state["uploads"] == 0 + assert len(state["input_tables"]) == 2 + assert len(state["relations"]) == 2 + table = state["input_tables"][1] + assert table.equals(state["input_tables"][0], check_metadata=True) + assert table.column(0).to_pylist() == values + assert table.column(0).is_null().to_pylist() == [value is None for value in values] + assert table.schema.equals( + pa.schema([pa.field("renamed", pa.string(), nullable=nullable)]), + check_metadata=True, + ) + assert json.loads(state["relations"][1].local_relation.schema) == { + "type": "struct", "fields": [ + {"name": "renamed", "type": "string", "nullable": nullable, + "metadata": {"source": "native"}} + ], + } + + + +@pytest.mark.parametrize("kind", [ + "object", "default", "python", "pyarrow", "python_numpy", "pyarrow_numpy", +]) +def test_string_canonical_column_storage_stays_eligible(kind): + from pyspark.sql._pandas_input import _has_canonical_columns, prepare_string_input + + frame = pd.DataFrame({"original": ["plain"] * 10000}, dtype=object) + schema = T.StructType([T.StructField("renamed", T.StringType())]) + if kind == "object": + frame.columns = pd.Index(np.asarray(["original"], dtype=object), dtype=object) + elif kind == "default": + frame.columns = pd.Index(["original"]) + elif kind.endswith("_numpy"): + from pandas.core.arrays import string_ as strings, string_arrow as arrow_strings + + module = strings if kind.startswith("python") else arrow_strings + name = ("StringArrayNumpySemantics" if kind.startswith("python") + else "ArrowStringArrayNumpySemantics") + cls = getattr(module, name, None) + if cls is None: + pytest.skip("older infer-string storage class is absent") + frame.columns = pd.Index._simple_new(cls._from_sequence(["original"]), name=None) + else: + frame.columns = pd.Index(pd.array(["original"], dtype=pd.StringDtype(storage=kind))) + assert _has_canonical_columns(frame.columns) + assert type(prepare_string_input(frame, schema)) is bytes + + + +@pytest.mark.parametrize("missing", ["module", "owner", "base"]) +def test_string_absent_private_columns_storage_keeps_row_fallback(monkeypatch, missing): + import builtins + from pandas.core.arrays import numpy_ as numpy_module, string_ as string_module + from pyspark.sql._pandas_input import _has_canonical_columns + + columns = pd.Index(pd.array(["original"], dtype=pd.StringDtype(storage="python"))) + if missing == "module": + original = builtins.__import__ + + def without_arrow(name, *args, **kwargs): + if name == "pandas.core.arrays.arrow": + raise ImportError("private storage module is absent") + return original(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", without_arrow) + elif missing == "owner": + monkeypatch.delattr(numpy_module, "NumpyExtensionArray") + else: + monkeypatch.delattr(string_module, "BaseStringArray") + assert not _has_canonical_columns(columns) + + + +@pytest.mark.parametrize("cached", [False, True]) +@pytest.mark.parametrize("verify", [False, True]) +@pytest.mark.parametrize("large", ["false", "true"]) +def test_string_columns_preserve_every_ipc_cell(protocol, monkeypatch, cached, verify, large): + import json + + from pyspark.sql.pandas.types import from_arrow_metadata + + strings = ["", "plain", "é", "e\u0301", "\x00embedded\x00", "漢字", "🙂", + "\U0010ffff", "\uffff", "\u202eright-to-left"] * 1000 + nullable = [None if i % 7 == 0 else value for i, value in enumerate(reversed(strings))] + expected = [strings, nullable, [None] * len(strings)] + frame = pd.DataFrame(dict(zip(["original", "b", "c"], expected)), dtype=object) + schema = T.StructType([ + T.StructField("renamed.`text", T.StringType(), False, {"source": "string", "id": 3}), + T.StructField("nullable", T.StringType(), True, {"tag": [1, 2]}), + T.StructField("all_null", T.StringType()), + ]) + + def no_rows(*args, **kwargs): + raise AssertionError("string fast path must not materialize Python rows") + + for method in ("to_numpy", "iterrows", "itertuples"): + monkeypatch.setattr(pd.DataFrame, method, no_rows) + monkeypatch.setattr(pd.DataFrame, "values", property(no_rows)) + monkeypatch.setattr(pd.Series, "tolist", no_rows) + overrides = { + "spark.sql.session.localRelationCacheThreshold": "1" if cached else "999999999", + "spark.sql.execution.arrow.useLargeVarTypes": large, + "spark.sql.session.localRelationChunkSizeRows": "997", + } + with _arrow_input_service(protocol, **overrides) as (spark, state): + result = spark.createDataFrame(frame, schema, verifySchema=verify) + assert not result.isLocal() + result.toArrow() + assert state["column_names"] == schema.names + table = state["input_tables"][0] + assert table.num_rows == len(strings) + assert table.schema.names == schema.names + for column, values, field in zip(table.columns, expected, schema.fields): + assert column.type == pa.string() + assert column.to_pylist() == values + actual_field = table.schema.field(field.name) + assert actual_field.nullable == field.nullable + assert from_arrow_metadata(actual_field.metadata) is None + spark_field = json.loads(state["relations"][0].local_relation.schema)["fields"][ + schema.names.index(field.name)] + assert spark_field["metadata"] == field.metadata + assert state["config_requests"] == [] + assert state["uploads"] == 0 + + + +@pytest.mark.parametrize("reason", [ + "small", "empty", "inferred", "names", "ddl", "cast", "atomic", "mixed_schema", + "width", "duplicate_input", "duplicate_schema", "input_nul", "schema_nul", + "input_number", "datetime", "date", "nan", "pd_na", "nat", "mixed", "bytes", + "nonnullable_null", "frame_subclass", "string_subclass", "string_dtype", + "arrow_string_dtype", "late_unsupported", "surrogate", "late_surrogate", "collation", +]) +def test_string_column_guard_preserves_row_fallback(reason): + from pyspark.sql._pandas_input import prepare_string_input + + class Frame(pd.DataFrame): + pass + + class Text(str): + def __str__(self): + raise AssertionError("guard must not call conversion hooks") + + class Unsupported: + def __str__(self): + raise AssertionError("guard must not stringify unsupported objects") + + values = ["plain"] * 10000 + frame = pd.DataFrame({"text": values}, dtype=object) + schema = T.StructType([T.StructField("text", T.StringType())]) + if reason == "small": + frame = frame.iloc[:9999] + elif reason == "empty": + frame = frame.iloc[:0] + elif reason in ("inferred", "names", "ddl", "cast", "atomic"): + schema = {"inferred": None, "names": ["text"], "ddl": "text STRING", + "cast": T.StructType([T.StructField("text", T.LongType())]), + "atomic": T.StringType()}[reason] + elif reason == "mixed_schema": + frame["long"] = np.arange(10000) + schema.add("long", T.LongType()) + elif reason == "width": + schema.add("extra", T.StringType()) + elif reason in ("duplicate_input", "duplicate_schema"): + frame["other"] = values + schema.add("other", T.StringType()) + if reason == "duplicate_input": + frame.columns = ["text", "text"] + else: + schema = T.StructType([T.StructField("text", T.StringType())] * 2) + elif reason in ("input_nul", "input_number"): + frame.columns = ["text\x00" if reason == "input_nul" else 1] + elif reason == "schema_nul": + schema = T.StructType([T.StructField("text\x00", T.StringType())]) + elif reason == "frame_subclass": + frame = Frame(frame) + elif reason in ("string_dtype", "arrow_string_dtype"): + frame = pd.DataFrame({"text": pd.Series( + values, dtype=pd.StringDtype(storage="pyarrow" if reason == "arrow_string_dtype" + else "python"))}) + elif reason == "collation": + schema = T.StructType([T.StructField("text", T.StringType("UTF8_LCASE"))]) + else: + value = { + "datetime": datetime(2020, 1, 1), "date": date(2020, 1, 1), + "nan": float("nan"), "pd_na": pd.NA, "nat": pd.NaT, "mixed": 42, + "bytes": b"plain", "nonnullable_null": None, "string_subclass": Text("plain"), + "late_unsupported": Unsupported(), "surrogate": "\ud800", "late_surrogate": "\udfff", + }[reason] + if reason == "nonnullable_null": + schema = T.StructType([T.StructField("text", T.StringType(), False)]) + if reason in ("late_unsupported", "late_surrogate"): + # pandas 3 infers StringDtype for a newly assigned list column. + # Keep these unsupported-object fixtures on concrete object storage. + frame["other"] = pd.Series(values, dtype=object) + schema.add("other", T.StringType()) + backing = frame["other"].array._ndarray + assert type(backing) is np.ndarray and backing.dtype == np.dtype("object") + frame.iat[-1, -1] = value + if reason in ("late_unsupported", "late_surrogate"): + assert frame.iat[-1, -1] is value + result = prepare_string_input(frame, schema) + if reason == "arrow_string_dtype": + # Canonical Arrow-backed StringDtype now has its own inline path. + # The explicit backend suite checks every cell and storage identity. + assert type(result) is bytes + else: + assert result is None + + + +@pytest.mark.parametrize("reason", [ + "string_subclass", "frame_subclass", "nan", "pd_na", "nat", "string_dtype", + "nonnullable_null", "inferred", "names", "ddl", "small", "collation", "cast", "mixed", +]) +def test_string_public_fallback_keeps_row_semantics(protocol, monkeypatch, reason): + calls = [] + + class Text(str): + def __str__(self): + calls.append("str") + return "changed" + + class Frame(pd.DataFrame): + pass + + values = ["plain"] * (9999 if reason == "small" else 10000) + if reason in ("nan", "pd_na", "nat", "nonnullable_null", "mixed"): + values[-1] = {"nan": float("nan"), "pd_na": pd.NA, "nat": pd.NaT, + "nonnullable_null": None, "mixed": 42}[reason] + elif reason == "string_subclass": + values[-1] = Text("plain") + frame_type = Frame if reason == "frame_subclass" else pd.DataFrame + frame = frame_type({"text": values}, dtype="string" if reason == "string_dtype" else object) + schema = T.StructType([T.StructField("text", T.StringType(), reason != "nonnullable_null")]) + if reason == "inferred": + schema = None + elif reason == "names": + schema = ["renamed"] + elif reason == "ddl": + schema = "text STRING" + elif reason == "collation": + schema = T.StructType([T.StructField("text", T.StringType("UTF8_LCASE"))]) + elif reason == "cast": + schema = T.StructType([T.StructField("text", T.LongType())]) + astype = pd.DataFrame.astype + + def record_rows(self, *args, **kwargs): + calls.append("astype") + return astype(self, *args, **kwargs) + + monkeypatch.setattr(pd.DataFrame, "astype", record_rows) + with _arrow_input_service(protocol) as (spark, state): + if reason in ("nonnullable_null", "cast", "mixed"): + from pyspark.errors import PySparkRuntimeError + + message = ("Column 'text' is declared as non-nullable but contains null values" + if reason == "nonnullable_null" else "Type mismatch in row data") + with pytest.raises(PySparkRuntimeError, match=message): + spark.createDataFrame(frame, schema) + assert calls == ["astype"] + assert state["config_requests"] == [] + assert state["relations"] == [] + return + spark.createDataFrame(frame, schema).toArrow() + assert calls == ["astype"] # Native extraction keeps str.__str__ overrides uncalled. + table = state["input_tables"][0] + assert table.schema.names == ["renamed" if reason == "names" else "text"] + expected = ["plain"] * len(values) + if reason in ("nan", "pd_na", "nat"): + expected[-1] = None + assert table.column(0).to_pylist() == expected + assert state["config_requests"] == [] + + + +@pytest.mark.parametrize("surrogate", ["\ud800", "\udfff"]) +def test_string_surrogate_fallback_precedes_importer_configuration(protocol, monkeypatch, surrogate): + values = ["plain"] * 10000 + values[-1] = surrogate + frame = pd.DataFrame({"text": values}, dtype=object) + schema = T.StructType([T.StructField("text", T.StringType())]) + astype = pd.DataFrame.astype + calls = [] + + def record_rows(self, *args, **kwargs): + calls.append("astype") + return astype(self, *args, **kwargs) + + monkeypatch.setattr(pd.DataFrame, "astype", record_rows) + with _arrow_input_service(protocol) as (spark, state): + state["config_error"] = grpc.StatusCode.INVALID_ARGUMENT + with pytest.raises(TypeError, match="unsupported value type for createDataFrame: str"): + spark.createDataFrame(frame, schema) + assert calls == ["astype"] + assert state["config_requests"] == [] + assert state["relations"] == [] + assert state["uploads"] == 0 + + + +@pytest.mark.parametrize("class_name,attribute", [ + ("StructType", "names"), ("StructType", "fields"), ("StructType", "__iter__"), + ("StructField", "name"), ("StructField", "dataType"), ("StructField", "nullable"), + ("StructField", "metadata"), ("StringType", "isUTF8BinaryCollation"), + ("StructType", "__getattribute__"), ("StructField", "__getattribute__"), + ("StringType", "__getattribute__"), ("StructType", "__class__"), + ("StructField", "__class__"), ("StringType", "__class__"), + ("StructType", "alias"), ("StructField", "alias"), ("StringType", "alias"), +]) +@pytest.mark.parametrize("before_import", [False, True]) +def test_string_custom_schema_descriptors_keep_native_row_path( + protocol, class_name, attribute, before_import +): + import sys + + script = r''' +import sys +import tempfile +from pathlib import Path +import pandas as pd +from pyspark.sql import types as T + +import importlib.util +spec = importlib.util.spec_from_file_location("string_schema_capture", sys.argv[4]) +capture = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = capture +spec.loader.exec_module(capture) +_arrow_input_service, protocol = capture._arrow_input_service, capture.protocol +frame = pd.DataFrame({"text": ["plain"] * 10000}, dtype=object) +schema = T.StructType([T.StructField("text", T.StringType(), True, {"source": "native"})]) +class_name, attribute, before_import = sys.argv[1:4] +if before_import == "True": + # The capture module imports prepare_int64_table; force the public call to + # import a new helper after the class or public alias has been customized. + sys.modules.pop("pyspark.sql._pandas_input", None) +assert ("pyspark.sql._pandas_input" in sys.modules) == (before_import == "False") +cls = getattr(T, class_name) +if attribute == "alias": + setattr(T, class_name, object) +else: + if attribute == "__getattribute__": + replacement = lambda self, name: object.__getattribute__(self, name) + elif attribute == "__iter__": + replacement = lambda self: iter([T.StructField("changed", T.StringType())]) + elif attribute == "__class__": + attribute = "__getattribute__" + replacement = lambda self, name: object if name == "__class__" else ( + object.__getattribute__(self, name)) + elif attribute == "isUTF8BinaryCollation": + replacement = lambda self: False + else: + value = {"names": ["changed"], "fields": [], "name": "changed", + "dataType": T.StringType(), "nullable": False, "metadata": {"changed": True}}[attribute] + replacement = property(lambda self: value) + setattr(cls, attribute, replacement) +class Factory: + def mktemp(self, name): + return Path(tempfile.mkdtemp(prefix=name)) +with _arrow_input_service(protocol.__wrapped__(Factory())) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + assert state["config_requests"] == [], state["config_requests"] + table = state["input_tables"][0] + assert table.schema.names == ["text"], table.schema + assert table.column(0).to_pylist() == ["plain"] * 10000 +''' + result = subprocess.run([sys.executable, "-c", script, class_name, attribute, + str(before_import), __file__], + capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stdout + result.stderr + + + +def test_string_rebound_pandas_alias_keeps_subclass_fallback(monkeypatch): + from pyspark.sql._pandas_input import prepare_string_input + + class Frame(pd.DataFrame): + pass + + frame = Frame({"text": ["plain"] * 10000}, dtype=object) + schema = T.StructType([T.StructField("text", T.StringType())]) + monkeypatch.setattr(pd, "DataFrame", Frame) + assert prepare_string_input(frame, schema) is None + + + +@pytest.mark.parametrize("class_name", [ + "BooleanType", "ByteType", "ShortType", "IntegerType", "LongType", + "FloatType", "DoubleType", "DecimalType", +]) +@pytest.mark.parametrize("converter_loaded", [False, True]) +def test_string_rebound_preceding_converter_alias_keeps_native_row_path( + protocol, class_name, converter_loaded +): + import sys + + script = r''' +import importlib.util +import sys +import tempfile +from pathlib import Path +import pandas as pd +from pyspark.sql import types as T + +spec = importlib.util.spec_from_file_location("string_alias_capture", sys.argv[3]) +capture = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = capture +spec.loader.exec_module(capture) +_arrow_input_service, protocol = capture._arrow_input_service, capture.protocol +values = ["plain", "", "\x00", "é", "漢字", "🙂", None, "tail"] * 1250 +frame = pd.DataFrame({"text": values}, dtype=object) +schema = T.StructType([T.StructField("text", T.StringType())]) +class_name, converter_loaded = sys.argv[1:3] +if converter_loaded == "True": + import pyspark.sql.pandas.types as converter + setattr(converter, class_name, T.StringType) +else: + sys.modules.pop("pyspark.sql.pandas.types", None) + setattr(T, class_name, T.StringType) + assert "pyspark.sql.pandas.types" not in sys.modules +class Factory: + def mktemp(self, name): + return Path(tempfile.mkdtemp(prefix=name)) +with _arrow_input_service(protocol.__wrapped__(Factory())) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + assert state["config_requests"] == [], state["config_requests"] + table = state["input_tables"][0] + assert table.schema.names == ["text"], table.schema + assert table.column(0).to_pylist() == values +''' + result = subprocess.run([sys.executable, "-c", script, class_name, + str(converter_loaded), __file__], + capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stdout + result.stderr + + + +@pytest.mark.parametrize("replacement", ["string", "none"]) +def test_string_rebound_importer_schema_alias_keeps_native_row_path(protocol, replacement): + import sys + + script = r''' +import importlib.util +import json +import sys +import tempfile +from pathlib import Path +import pandas as pd +from pyspark.sql import types as T + +spec = importlib.util.spec_from_file_location("string_importer_capture", sys.argv[2]) +capture = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = capture +spec.loader.exec_module(capture) +_arrow_input_service, protocol = capture._arrow_input_service, capture.protocol +strings = ["plain", "", "\x00", "é", "漢字", "🙂", "e\u0301", "tail"] * 1250 +nullable = [None if i % 7 == 0 else value for i, value in enumerate(reversed(strings))] +expected = [strings, nullable, [None] * len(strings)] +frame = pd.DataFrame(dict(zip(["original", "b", "c"], expected)), dtype=object) +schema = T.StructType([ + T.StructField("text", T.StringType(), False, {"source": "native", "id": 3}), + T.StructField("nullable", T.StringType(), True, {"tag": [1, 2]}), + T.StructField("all_null", T.StringType()), +]) +import pyspark.sql._arrow_input as importer +importer.StructType = T.StringType if sys.argv[1] == "string" else None +class Factory: + def mktemp(self, name): + return Path(tempfile.mkdtemp(prefix=name)) +with _arrow_input_service(protocol.__wrapped__(Factory())) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + assert state["config_requests"] == [], state["config_requests"] + table = state["input_tables"][0] + assert table.schema.names == schema.names, table.schema + spark_fields = json.loads(state["relations"][0].local_relation.schema)["fields"] + for column, values, field, spark_field in zip(table.columns, expected, schema.fields, spark_fields): + assert column.to_pylist() == values + actual_field = table.schema.field(field.name) + assert actual_field.nullable == field.nullable + assert actual_field.metadata is None + assert spark_field["metadata"] == field.metadata +''' + result = subprocess.run([sys.executable, "-c", script, replacement, __file__], + capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stdout + result.stderr + + + +@pytest.mark.parametrize("dtype_mode", ["equal", "class_spoof", "exact_numpy"]) +def test_string_custom_extension_dtype_cannot_impersonate_numpy_object(protocol, dtype_mode): + from pandas.api.extensions import ExtensionArray, ExtensionDtype, take + from pyspark.sql._pandas_input import prepare_string_input + + arrow_calls = [] + + class ObjectEqualDtype(ExtensionDtype): + name = "object_equal_string" + type = str + kind = "O" + na_value = None + + @classmethod + def construct_array_type(cls): + return DifferentArrowArray + + def __eq__(self, other): + return (isinstance(other, ObjectEqualDtype) + or isinstance(other, np.dtype) and other.kind == "O") + + def __hash__(self): + return hash(self.name) + + class DifferentArrowArray(ExtensionArray): + def __init__(self, values): + self._values = np.asarray(values, dtype=object) + + @property + def dtype(self): + return ObjectEqualDtype() + + @property + def nbytes(self): + return self._values.nbytes + + def __len__(self): + return len(self._values) + + def __getitem__(self, key): + result = self._values[key] + return type(self)(result) if isinstance(result, np.ndarray) else result + + def isna(self): + return pd.isna(self._values) + + def take(self, indices, allow_fill=False, fill_value=None): + return type(self)(take(self._values, indices, allow_fill=allow_fill, + fill_value=fill_value)) + + def copy(self): + return type(self)(self._values.copy()) + + @classmethod + def _from_sequence(cls, scalars, *, dtype=None, copy=False): + values = np.asarray(scalars, dtype=object) + return cls(values.copy() if copy else values) + + def to_numpy(self, dtype=None, copy=False, na_value=None): + if dtype is not None: + return self._values.astype(dtype, copy=copy) + return self._values.copy() if copy else self._values + + def __arrow_array__(self, type=None): + arrow_calls.append("arrow") + # A scan of to_numpy() cannot establish the values this hook emits. + return pa.array(["different"] * len(self), type=type) + + array = DifferentArrowArray(["plain"] * 10000) + frame = pd.DataFrame({"text": pd.Series(array, copy=False)}) + schema = T.StructType([T.StructField("text", T.StringType())]) + if dtype_mode == "exact_numpy": + # An ExtensionArray may change its advertised dtype after construction. + # The dtype and NumPy scan both pass while its Arrow hook changes cells. + DifferentArrowArray.dtype = property(lambda self: np.dtype("object")) + DifferentArrowArray.__array__ = lambda self, dtype=None, copy=None: self._values + elif dtype_mode == "class_spoof": + def spoof_class(self, name): + if name == "__class__": + return type(np.dtype("object")) + return object.__getattribute__(self, name) + + ObjectEqualDtype.__getattribute__ = spoof_class + dtype = frame.dtypes.iloc[0] + assert type(frame) is pd.DataFrame + assert type(frame.iloc[:, 0].array) is DifferentArrowArray + assert type(dtype) is (type(np.dtype("object")) if dtype_mode == "exact_numpy" + else ObjectEqualDtype) + assert isinstance(dtype, np.dtype) is (dtype_mode != "equal") + assert dtype == np.dtype("object") + assert prepare_string_input(frame, schema) is None + with _arrow_input_service(protocol) as (spark, state): + if dtype_mode == "exact_numpy": + # This deliberately inconsistent EA already fails pandas' row-path + # astype(object). Preserve that error before importer configuration. + with pytest.raises(TypeError, match="cannot set astype"): + spark.createDataFrame(frame, schema) + assert state["input_tables"] == [] + else: + spark.createDataFrame(frame, schema).toArrow() + assert state["input_tables"][0].column(0).to_pylist() == ["plain"] * 10000 + assert state["config_requests"] == [] + assert arrow_calls == [] + + + +@pytest.mark.parametrize("method", ["astype", "notna"]) +def test_string_instance_conversion_hooks_keep_row_fallback(protocol, method): + frame = pd.DataFrame({"text": ["plain"] * 10000}, dtype=object) + schema = T.StructType([T.StructField("text", T.StringType())]) + original = getattr(frame, method) + calls = [] + expected = ["plain"] * 10000 + if method == "astype": + expected[0] = "changed" + + def customized(*args, **kwargs): + calls.append(method) + result = original(*args, **kwargs) + result.iat[0, 0] = "changed" + return result + else: + expected[0] = None + + def customized(*args, **kwargs): + calls.append(method) + result = original(*args, **kwargs) + result.iat[0, 0] = False + return result + setattr(frame, method, customized) + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + assert calls == [method] + assert state["config_requests"] == [] + assert state["uploads"] == 0 + assert state["input_tables"][0].column(0).to_pylist() == expected + + + +@pytest.mark.parametrize("method", ["astype", "notna", "where"]) +@pytest.mark.parametrize("before_import", [False, True]) +def test_string_class_conversion_hooks_keep_row_fallback( + protocol, monkeypatch, method, before_import +): + import importlib + import sys + + name = "pyspark.sql._pandas_input" + if before_import: + monkeypatch.delitem(sys.modules, name, raising=False) + else: + importlib.import_module(name) + frame = pd.DataFrame({"text": ["plain"] * 10000}, dtype=object) + schema = T.StructType([T.StructField("text", T.StringType())]) + original = getattr(pd.DataFrame, method) + calls = [] + expected = ["plain"] * 10000 + expected[0] = None if method == "notna" else "changed" + + def customized(self, *args, **kwargs): + calls.append(method) + result = original(self, *args, **kwargs) + result.iat[0, 0] = False if method == "notna" else "changed" + return result + + monkeypatch.setattr(pd.DataFrame, method, customized) + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + assert calls == [method] + assert state["config_requests"] == [] + assert state["uploads"] == 0 + assert state["input_tables"][0].column(0).to_pylist() == expected + + + +def test_string_instance_where_override_keeps_existing_row_behavior(protocol): + from pyspark.sql._pandas_input import prepare_string_input + + frame = pd.DataFrame({"text": ["plain"] * 10000}, dtype=object) + schema = T.StructType([T.StructField("text", T.StringType())]) + + def customized(*args, **kwargs): + raise AssertionError("row normalization calls where on the astype result") + + frame.where = customized + assert prepare_string_input(frame, schema) is None + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + assert state["config_requests"] == [] + assert state["input_tables"][0].column(0).to_pylist() == ["plain"] * 10000 + + + +def test_string_native_backing_does_not_add_series_conversion_hooks(protocol, monkeypatch): + frame = pd.DataFrame({"text": ["plain"] * 10000}, dtype=object) + schema = T.StructType([T.StructField("text", T.StringType())]) + + def customized(*args, **kwargs): + raise AssertionError("native string input must not call Series.to_numpy") + + monkeypatch.setattr(pd.Series, "to_numpy", customized) + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + assert state["config_requests"] == [] + assert state["input_tables"][0].column(0).to_pylist() == ["plain"] * 10000 + + + +def test_string_large_inline_input_keeps_row_transport_and_schema(protocol): + values = ["漢字-🙂-" + str(i) for i in range(100000)] + frame = pd.DataFrame({"text": values}, dtype=object) + schema = T.StructType([T.StructField("renamed", T.StringType(), False, + {"source": "inline-string", "size": len(values)})]) + with _arrow_input_service(protocol, **{ + "spark.sql.session.localRelationCacheThreshold": "1", + "spark.sql.session.localRelationSizeLimit": "1", + "spark.sql.execution.arrow.useLargeVarTypes": "true", + }) as (spark, state): + result = spark.createDataFrame(frame, schema) + assert not result.isLocal() + result.toArrow() + assert state["column_names"] == schema.names + assert state["config_requests"] == [] + assert state["uploads"] == 0 + table = state["input_tables"][0] + assert table.schema.names == ["renamed"] + assert table.column(0).type == pa.string() + assert table.schema.field(0).nullable is False + assert table.schema.field(0).metadata is None + assert table.column(0).to_pylist() == values + assert __import__("json").loads(state["relations"][0].local_relation.schema) == ( + __import__("json").loads(schema.json())) diff --git a/python/tests/test_arrow_string_input.py b/python/tests/test_arrow_string_input.py new file mode 100644 index 00000000..2542e448 --- /dev/null +++ b/python/tests/test_arrow_string_input.py @@ -0,0 +1,879 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Explicit Arrow-backed StringDtype input checks against the offline RPC service.""" + +import json +from types import MethodType + +import numpy as np +import pandas as pd +import pyarrow as pa +import pytest +from pandas.core.arrays import string_arrow as string_arrow_module +from pandas.core.internals import blocks as blocks_module, managers as managers_module +from pyspark.sql import types as T +from pyspark.sql import _pandas_input as pandas_input +from pyspark.sql._pandas_input import prepare_string_input + +from tests.test_arrow_handoff import _arrow_input_service, protocol # noqa: F401 + + +@pytest.fixture(params=["pd_na", "nan"]) +def missing_kind(request): + return request.param + + +def _dtype(missing_kind, storage="pyarrow"): + return pd.StringDtype( + storage=storage, na_value=pd.NA if missing_kind == "pd_na" else np.nan + ) + + +def _frame(values, missing_kind="pd_na"): + """Always request the backend; pandas' default string inference is not evidence.""" + frame = pd.DataFrame({ + name: pd.Series(pd.array(column, dtype=_dtype(missing_kind))) + for name, column in values.items() + }) + # Isolate data-array hooks from pandas 3's independently inferred label + # storage. Canonical extension-backed labels have their own existing tests. + frame.columns = pd.Index(list(values), dtype=object) + return frame + + +def _backing(frame, index=0): + return frame.iloc[:, index].array + + +def _assert_backend(frame, missing_kind): + expected_class = string_arrow_module.ArrowStringArray + if missing_kind == "nan": + expected_class = vars(string_arrow_module).get( + "ArrowStringArrayNumpySemantics", expected_class + ) + dtypes = frame.dtypes + for index, column in enumerate(frame.columns): + array = frame[column].array + assert type(array) is expected_class + assert type(array.dtype) is pd.StringDtype + assert array.dtype is dtypes.iloc[index] + assert array.dtype.storage == "pyarrow" + if missing_kind == "pd_na": + assert array.dtype.na_value is pd.NA + else: + assert type(array.dtype.na_value) is float and np.isnan(array.dtype.na_value) + assert type(array._pa_array) is pa.ChunkedArray + assert all(type(chunk) in (pa.StringArray, pa.LargeStringArray) + for chunk in array._pa_array.chunks) + + +def _schema(names=("text",), nullable=True): + return T.StructType([T.StructField(name, T.StringType(), nullable) for name in names]) + + +def _assert_inline(state, schema, expected): + assert state["config_requests"] == [] + assert state["uploads"] == 0 + assert state["cache"] == {} + assert state["column_names"] == schema.names + assert len(state["relations"]) == 1 + relation = state["relations"][0] + assert relation.HasField("local_relation") + assert json.loads(relation.local_relation.schema) == json.loads(schema.json()) + table = state["input_tables"][0] + assert table.schema.names == schema.names + assert table.schema.metadata is None + assert table.num_rows == len(expected[0]) + for index, (field, values) in enumerate(zip(schema.fields, expected)): + actual_field = table.schema.field(index) + assert actual_field.type == pa.string() + assert actual_field.nullable == field.nullable + # Spark metadata belongs to LocalRelation's logical schema on the row path. + assert actual_field.metadata is None + assert table.column(index).to_pylist() == values + + +@pytest.mark.parametrize("verify", [False, True]) +@pytest.mark.parametrize("arrow_type", [pa.string(), pa.large_string()], ids=["string", "large"]) +def test_arrow_string_preserve_every_cell_and_inline_schema( + protocol, monkeypatch, missing_kind, verify, arrow_type +): + strings = ["", "plain", "é", "e\u0301", "\x00embedded\x00", "漢字", "🙂", + "\U0010ffff", "\uffff", "\u202eright-to-left"] * 1000 + nullable = [None if i % 7 == 0 else value for i, value in enumerate(reversed(strings))] + expected = [strings, nullable, [None] * len(strings)] + frame = _frame(dict(zip(["original", "b", "c"], expected)), missing_kind) + # These are ordinary concrete Arrow string buffers with deliberately different + # chunk boundaries. Every chunk is sliced from a larger buffer with offset > 0. + partitions = ([1, 0, 997, 1001, 8001], [3333, 1, 6666], [5001, 4999]) + for index, (values, sizes) in enumerate(zip(expected, partitions)): + chunks, start = [], 0 + for size in sizes: + chunk = pa.array(["padding"] + values[start:start + size] + ["padding"], + type=arrow_type).slice(1, size) + assert chunk.offset == 1 + chunks.append(chunk) + start += size + assert start == len(strings) + _backing(frame, index)._pa_array = pa.chunked_array(chunks, type=arrow_type) + assert [len(chunk) for chunk in _backing(frame)._pa_array.chunks] == partitions[0] + _assert_backend(frame, missing_kind) + schema = T.StructType([ + T.StructField("renamed.`text", T.StringType(), False, + {"source": "arrow-string", "id": 3}), + T.StructField("nullable", T.StringType(), True, {"tag": [1, 2]}), + T.StructField("all_null", T.StringType()), + ]) + ipc = prepare_string_input(frame, schema) + assert type(ipc) is bytes + private_table = pa.ipc.open_stream(ipc).read_all() + assert [column.to_pylist() for column in private_table.columns] == expected + + def no_rows(*args, **kwargs): + raise AssertionError("canonical Arrow strings must not materialize Python rows") + + for method in ("to_numpy", "iterrows", "itertuples"): + monkeypatch.setattr(pd.DataFrame, method, no_rows) + monkeypatch.setattr(pd.DataFrame, "values", property(no_rows)) + monkeypatch.setattr(pd.Series, "tolist", no_rows) + with _arrow_input_service(protocol, config_error=True, **{ + "spark.sql.session.localRelationCacheThreshold": "1", + "spark.sql.session.localRelationSizeLimit": "1", + "spark.sql.session.localRelationChunkSizeRows": "997", + "spark.sql.execution.arrow.useLargeVarTypes": "true", + }) as (spark, state): + result = spark.createDataFrame(frame, schema, verifySchema=verify) + assert not result.isLocal() + result.toArrow() + _assert_inline(state, schema, expected) + + +@pytest.mark.parametrize("rows,eligible", [(0, False), (9999, False), (10000, True)]) +def test_arrow_string_threshold_is_explicit(missing_kind, rows, eligible): + frame = _frame({"text": [""] * rows}, missing_kind) + _assert_backend(frame, missing_kind) + result = prepare_string_input(frame, _schema()) + assert (type(result) is bytes) is eligible + if not eligible: + assert result is None + + +@pytest.mark.parametrize("reason", [ + "inferred", "names", "ddl", "atomic", "cast", "mixed_schema", "width", + "duplicate_input", "duplicate_schema", "input_nul", "input_number", "schema_nul", + "collation", "nonnullable_null", "python", "generic_arrow", "object_mixed", + "python_mixed", "frame_subclass", "array_subclass", "dtype_subclass", + "array_storage", "binary_storage", "integer_storage", "short_storage", "invalid_utf8", + "invalid_offsets", +]) +def test_arrow_string_guard_keeps_unsupported_inputs_on_rows(missing_kind, reason): + values = ["plain"] * 10000 + frame = _frame({"text": values}, missing_kind) + schema = _schema() + if reason in ("inferred", "names", "ddl", "atomic", "cast"): + schema = { + "inferred": None, "names": ["text"], "ddl": "text STRING", + "atomic": T.StringType(), + "cast": T.StructType([T.StructField("text", T.LongType())]), + }[reason] + elif reason == "mixed_schema": + frame["long"] = np.arange(10000) + schema.add("long", T.LongType()) + elif reason == "width": + schema.add("extra", T.StringType()) + elif reason in ("duplicate_input", "duplicate_schema"): + frame["other"] = pd.Series(pd.array(values, dtype=_dtype(missing_kind))) + schema.add("other", T.StringType()) + if reason == "duplicate_input": + frame.columns = ["text", "text"] + else: + schema = _schema(("text", "text")) + elif reason in ("input_nul", "input_number"): + frame.columns = ["text\x00" if reason == "input_nul" else 1] + elif reason == "schema_nul": + schema = _schema(("text\x00",)) + elif reason == "collation": + schema = T.StructType([T.StructField("text", T.StringType("UTF8_LCASE"))]) + elif reason == "nonnullable_null": + frame = _frame({"text": values[:-1] + [None]}, missing_kind) + schema = _schema(nullable=False) + elif reason == "python": + frame = pd.DataFrame({"text": pd.Series(values, dtype=_dtype(missing_kind, "python"))}) + elif reason == "generic_arrow": + frame = pd.DataFrame({"text": pd.Series(values, dtype=pd.ArrowDtype(pa.large_string()))}) + elif reason in ("object_mixed", "python_mixed"): + dtype = object if reason == "object_mixed" else _dtype(missing_kind, "python") + frame["other"] = pd.Series(values, dtype=dtype) + schema.add("other", T.StringType()) + elif reason == "frame_subclass": + class Frame(pd.DataFrame): + pass + + frame = Frame(frame) + elif reason == "array_subclass": + array = _backing(frame) + + class Array(type(array)): + pass + + array.__class__ = Array + elif reason == "dtype_subclass": + class Dtype(pd.StringDtype): + pass + + _backing(frame)._dtype = Dtype(storage="pyarrow", na_value=_dtype(missing_kind).na_value) + elif reason == "array_storage": + _backing(frame)._pa_array = pa.array(values, type=pa.large_string()) + elif reason == "binary_storage": + _backing(frame)._pa_array = pa.chunked_array([pa.array([b"plain"] * 10000)]) + elif reason == "integer_storage": + _backing(frame)._pa_array = pa.chunked_array([pa.array([1] * 10000)]) + elif reason == "short_storage": + _backing(frame)._pa_array = pa.chunked_array([pa.array(values[:-1])]) + elif reason in ("invalid_utf8", "invalid_offsets"): + offsets = np.arange(10001, dtype=np.int64) + data = b"a" * 10000 + if reason == "invalid_utf8": + data = data[:-1] + b"\xff" + else: + offsets[2] = 0 + invalid = pa.Array.from_buffers(pa.large_string(), 10000, [ + None, pa.py_buffer(offsets), pa.py_buffer(data), + ]) + _backing(frame)._pa_array = pa.chunked_array([invalid]) + assert prepare_string_input(frame, schema) is None + + +@pytest.mark.parametrize("location", ["instance", "class"]) +@pytest.mark.parametrize("method", [ + "astype", "isna", "__array__", "to_numpy", "_hasna", "fillna", "__getitem__", "__len__", +]) +def test_arrow_string_custom_array_hooks_decline_without_calling( + monkeypatch, missing_kind, location, method +): + frame = _frame({"text": ["plain"] * 10000}, missing_kind) + array = _backing(frame) + calls = [] + + def customized(*args, **kwargs): + calls.append(method) + raise AssertionError("eligibility must not invoke customized conversion hooks") + + if location == "instance": + # _hasna is a data descriptor with no setter. Populate the instance + # dictionary directly so the guard can reject every customized name. + monkeypatch.setitem(vars(array), method, MethodType(customized, array)) + else: + replacement = property(customized) if method == "_hasna" else customized + monkeypatch.setattr(type(array), method, replacement) + assert prepare_string_input(frame, _schema()) is None + assert calls == [] + + +@pytest.mark.parametrize("attribute", [ + "name", "na_value", "storage", "type", "kind", "__eq__", "__ne__", "__from_arrow__", +]) +def test_arrow_string_custom_dtype_hooks_decline_without_calling( + monkeypatch, missing_kind, attribute +): + frame = _frame({"text": ["plain"] * 10000}, missing_kind) + calls = [] + + def customized(*args, **kwargs): + calls.append(attribute) + raise AssertionError("eligibility must not invoke customized dtype hooks") + + replacement = customized if attribute.startswith("__") else property(customized) + # pandas 2 stores storage directly on each dtype; pandas 3 has a property. + monkeypatch.setattr(pd.StringDtype, attribute, replacement, raising=False) + assert prepare_string_input(frame, _schema()) is None + assert calls == [] + + +def test_arrow_string_custom_array_dtype_descriptor_declines(monkeypatch, missing_kind): + frame = _frame({"text": ["plain"] * 10000}, missing_kind) + array = _backing(frame) + dtype = array.dtype + # frame.dtypes already consults this descriptor in the established public + # path. The contract here is to decline, rather than serialize raw storage. + monkeypatch.setattr(type(array), "dtype", property(lambda self: dtype)) + assert prepare_string_input(frame, _schema()) is None + + +@pytest.mark.parametrize("location", [ + "manager_instance", "manager_class", "block_instance", "block_class", +]) +def test_arrow_string_custom_storage_extraction_hooks_decline_without_calling( + monkeypatch, missing_kind, location +): + frame = _frame({"text": ["plain"] * 10000}, missing_kind) + manager = vars(frame)["_mgr"] + block = manager.blocks[0] + calls = [] + + def customized(*args, **kwargs): + calls.append(location) + raise AssertionError("eligibility must not invoke customized storage extraction") + + target = manager if location.startswith("manager_") else block + method = "iget_values" if location.startswith("manager_") else "iget" + if location.endswith("_instance"): + monkeypatch.setitem(vars(target), method, MethodType(customized, target)) + else: + monkeypatch.setattr(type(target), method, customized) + assert prepare_string_input(frame, _schema()) is None + assert calls == [] + + +@pytest.mark.parametrize("cache_name", ["blknos", "blklocs"]) +def test_arrow_string_public_cached_index_subclass_matches_native_row_control( + protocol, monkeypatch, missing_kind, cache_name +): + calls = [] + + class CachedIndexError(Exception): + pass + + class CustomIndexes(np.ndarray): + def __getitem__(self, key): + calls.append(cache_name) + raise CachedIndexError("custom cached index must not be read during eligibility") + + values = ["plain", "\x00", "é", "🙂"] * 2500 + tables = [] + for row_control in (False, True): + frame = _frame({"text": values}, missing_kind) + _assert_backend(frame, missing_kind) + manager = vars(frame)["_mgr"] + cache = getattr(manager, cache_name) + assert type(cache) is np.ndarray + assert cache.dtype == np.dtype("intp") + assert cache.shape == (1,) + custom_cache = cache.view(CustomIndexes) + setattr(manager, "_" + cache_name, custom_cache) + assert type(getattr(manager, cache_name)) is CustomIndexes + schema = _schema() + calls.clear() + # Both cached arrays normally feed iget_values through __getitem__. + # Decline this subclass before indexing, while preserving ordinary row + # normalization, which does not need to index the original caches. + assert prepare_string_input(frame, schema) is None + assert calls == [] + with monkeypatch.context() as control: + if row_control: + control.setattr(pandas_input, "prepare_string_input", lambda *args: None) + with _arrow_input_service(protocol, config_error=True) as (spark, state): + result = spark.createDataFrame(frame, schema) + assert not result.isLocal() + result.toArrow() + _assert_inline(state, schema, [values]) + tables.append(state["input_tables"][0]) + assert calls == [] + with pytest.raises(CachedIndexError, match="custom cached index must not be read"): + custom_cache[0] + assert calls == [cache_name] + assert tables[0].equals(tables[1], check_metadata=True) + + +@pytest.mark.parametrize("target", ["manager", "block"]) +def test_arrow_string_public_storage_class_alias_matches_native_row_control( + protocol, monkeypatch, missing_kind, target +): + calls = [] + + class StorageError(Exception): + pass + + errors = [] + for row_control in (False, True): + with monkeypatch.context() as control: + # Start with ordinary storage before rebinding the public internal + # class. Matching the new alias must not authenticate a subclass. + frame = _frame({"text": ["plain"] * 10000}, missing_kind) + manager = vars(frame)["_mgr"] + if target == "manager": + class CustomManager(managers_module.BlockManager): + def apply(self, *args, **kwargs): + calls.append("manager.apply") + raise StorageError("manager.apply row sentinel") + + vars(frame)["_mgr"] = CustomManager( + manager.blocks, manager.axes, verify_integrity=False + ) + control.setattr(managers_module, "BlockManager", CustomManager) + else: + class CustomBlock(blocks_module.ExtensionBlock): + def astype(self, *args, **kwargs): + calls.append("block.astype") + raise StorageError("block.astype row sentinel") + + block = manager.blocks[0] + manager.blocks = (CustomBlock( + block.values, placement=block._mgr_locs, ndim=2, refs=block.refs + ),) + control.setattr(blocks_module, "ExtensionBlock", CustomBlock) + schema = _schema() + calls.clear() + assert prepare_string_input(frame, schema) is None + assert calls == [] + if row_control: + control.setattr(pandas_input, "prepare_string_input", lambda *args: None) + with _arrow_input_service(protocol, config_error=True) as (spark, state): + method = "manager.apply" if target == "manager" else "block.astype" + with pytest.raises(StorageError, + match=f"^{method} row sentinel$") as error: + spark.createDataFrame(frame, schema) + errors.append((type(error.value), str(error.value))) + assert calls == [method] + assert state["config_requests"] == [] + assert state["uploads"] == 0 + assert state["relations"] == [] + assert state["input_tables"] == [] + assert errors[0] == errors[1] + + +@pytest.mark.parametrize("location", ["class_none", "instance"]) +def test_arrow_string_custom_factory_declines_without_calling( + monkeypatch, missing_kind, location +): + frame = _frame({"text": ["plain"] * 10000}, missing_kind) + array = _backing(frame) + calls = [] + + def factory(*args, **kwargs): + calls.append("factory") + raise AssertionError("eligibility must not invoke a customized Arrow factory") + + if location == "class_none": + # pandas 3 normally defines this factory; pandas 2 must also decline an + # explicitly installed None rather than mistake it for absence. + monkeypatch.setattr(string_arrow_module.ArrowStringArray, + "_from_pyarrow_array", None, raising=False) + else: + monkeypatch.setitem(vars(array), "_from_pyarrow_array", MethodType(factory, array)) + assert prepare_string_input(frame, _schema()) is None + assert calls == [] + + +def test_arrow_string_deleted_factory_matches_pandas_layout(monkeypatch, missing_kind): + frame = _frame({"text": ["plain"] * 10000}, missing_kind) + requires_factory = type(vars(pd.StringDtype).get("storage")) is property + monkeypatch.delattr(string_arrow_module.ArrowStringArray, + "_from_pyarrow_array", raising=False) + result = prepare_string_input(frame, _schema()) + if requires_factory: + # pandas 3's constructor and row conversion use this factory. + assert result is None + else: + # pandas 2 ordinarily has no such factory; absence alone is not a hook. + assert type(result) is bytes + + +def test_arrow_string_custom_numpy_semantics_alias_declines_without_calling( + monkeypatch, missing_kind +): + frame = _frame({"text": ["plain"] * 10000}, missing_kind) + array = _backing(frame) + calls = [] + + class CustomNumpyStrings(string_arrow_module.ArrowStringArray): + def __arrow_array__(self, *args, **kwargs): + calls.append("arrow") + raise AssertionError("eligibility must not invoke an aliased subclass") + + # pandas 3 has no NumPy-semantics alias; creating one must be as conservative + # as rebinding the existing alias in pandas 2. Keep the actual custom type in + # the input, so an identity check against the rebound alias alone is unsafe. + array.__class__ = CustomNumpyStrings + monkeypatch.setattr(string_arrow_module, "ArrowStringArrayNumpySemantics", + CustomNumpyStrings, raising=False) + assert prepare_string_input(frame, _schema()) is None + assert calls == [] + + +def test_arrow_string_custom_public_array_alias_declines_without_calling( + monkeypatch, missing_kind +): + frame = _frame({"text": ["plain"] * 10000}, missing_kind) + calls = [] + + class CustomStrings(string_arrow_module.ArrowStringArray): + def __arrow_array__(self, *args, **kwargs): + calls.append("arrow") + raise AssertionError("eligibility must not invoke a rebound public class") + + monkeypatch.setattr(pd.arrays, "ArrowStringArray", CustomStrings) + assert prepare_string_input(frame, _schema()) is None + assert calls == [] + + +def test_arrow_string_public_fake_arrow_backing_matches_native_row_control( + protocol, monkeypatch, missing_kind +): + calls = [] + + class RowBackingError(Exception): + pass + + class CastBypassError(Exception): + pass + + class FakeChunkedArray: + def __init__(self, genuine): + self.genuine = genuine + + @property + def type(self): + calls.append("type") + return self.genuine.type + + @property + def chunks(self): + calls.append("chunks") + return self.genuine.chunks + + @property + def null_count(self): + calls.append("null_count") + return self.genuine.null_count + + def __len__(self): + calls.append("len") + return len(self.genuine) + + def validate(self, *args, **kwargs): + calls.append("validate") + return self.genuine.validate(*args, **kwargs) + + def to_numpy(self, *args, **kwargs): + calls.append("to_numpy") + raise RowBackingError("fake to_numpy row sentinel") + + def cast(self, *args, **kwargs): + calls.append("cast") + raise CastBypassError("fake cast bypass sentinel") + + errors, orders = [], [] + for row_control in (False, True): + # Construct the ordinary array while all public Arrow aliases are still + # genuine. Only its private backing and the public class alias change. + frame = _frame({"text": ["plain"] * 10000}, missing_kind) + array = _backing(frame) + ordinary_array_class = type(array) + fake = FakeChunkedArray(array._pa_array) + array._pa_array = fake + schema = _schema() + calls.clear() + with monkeypatch.context() as control: + control.setattr(pa, "ChunkedArray", FakeChunkedArray) + assert type(array) is ordinary_array_class + assert prepare_string_input(frame, schema) is None + assert calls == [] + if row_control: + control.setattr(pandas_input, "prepare_string_input", lambda *args: None) + with _arrow_input_service(protocol, config_error=True) as (spark, state): + with pytest.raises(RowBackingError, + match="^fake to_numpy row sentinel$") as error: + spark.createDataFrame(frame, schema) + errors.append((type(error.value), str(error.value))) + orders.append(list(calls)) + assert calls[-1] == "to_numpy" + assert "cast" not in calls + assert state["config_requests"] == [] + assert state["uploads"] == 0 + assert state["relations"] == [] + assert state["input_tables"] == [] + # The tempting direct-storage conversion has a different observable + # failure. It must never replace the public row sentinel above. + with pytest.raises(CastBypassError, match="^fake cast bypass sentinel$"): + fake.cast(pa.string(), safe=True) + assert errors[0] == errors[1] + assert orders[0] == orders[1] + + +def test_arrow_string_public_subclass_len_matches_native_row_control( + protocol, monkeypatch, missing_kind +): + frame = _frame({"text": ["plain"] * 10000}, missing_kind) + array = _backing(frame) + calls = [] + + class LengthError(Exception): + pass + + class CustomArray(type(array)): + def __len__(self): + calls.append("len") + raise LengthError("subclass len before astype") + + def astype(self, *args, **kwargs): + calls.append("astype") + raise AssertionError("astype must follow the failing isna length check") + + array.__class__ = CustomArray + schema = _schema() + assert prepare_string_input(frame, schema) is None + assert calls == [] + errors = [] + for row_control in (False, True): + calls.clear() + with monkeypatch.context() as control: + if row_control: + control.setattr(pandas_input, "prepare_string_input", lambda *args: None) + with _arrow_input_service(protocol, config_error=True) as (spark, state): + with pytest.raises(LengthError, + match="^subclass len before astype$") as error: + spark.createDataFrame(frame, schema) + errors.append((type(error.value), str(error.value))) + assert state["config_requests"] == [] + assert state["uploads"] == 0 + assert state["relations"] == [] + assert state["input_tables"] == [] + assert calls == ["len"] + assert errors[0] == errors[1] + + +def test_arrow_string_public_invalid_utf8_matches_native_row_control( + protocol, monkeypatch, missing_kind +): + errors = [] + for row_control in (False, True): + frame = _frame({"text": ["plain"] * 10000}, missing_kind) + validity = bytearray([0xFF] * 1250) + validity[0] = 0xFE + invalid = pa.Array.from_buffers(pa.large_string(), 10000, [ + pa.py_buffer(validity), pa.py_buffer(np.arange(10001, dtype=np.int64)), + pa.py_buffer(b"a" * 9999 + b"\xff"), + ], null_count=1) + _backing(frame)._pa_array = pa.chunked_array([invalid]) + schema = _schema(nullable=False) + assert prepare_string_input(frame, schema) is None + with monkeypatch.context() as control: + if row_control: + control.setattr(pandas_input, "prepare_string_input", lambda *args: None) + with _arrow_input_service(protocol, config_error=True) as (spark, state): + # Normalization decodes the final invalid string before the + # native row validator can reject the first, nonnullable null. + with pytest.raises((UnicodeDecodeError, pa.ArrowException)) as error: + spark.createDataFrame(frame, schema) + errors.append((type(error.value), str(error.value))) + assert state["config_requests"] == [] + assert state["uploads"] == 0 + assert state["relations"] == [] + assert state["input_tables"] == [] + assert errors[0] == errors[1] + + +@pytest.mark.parametrize("target", ["array_astype", "array_isna", "array_to_numpy", "frame_isna"]) +@pytest.mark.parametrize("behavior", ["mutate", "raise"]) +def test_arrow_string_public_hooks_match_native_row_control( + protocol, monkeypatch, missing_kind, target, behavior +): + frame = _frame({"text": ["plain"] * 10000}, missing_kind) + array = _backing(frame) + method = target.removeprefix("array_").removeprefix("frame_") + owner = frame if target == "frame_isna" else type(array) + original = getattr(owner, method) + calls = [] + expected = ["plain"] * 10000 + expected[0] = None if method == "isna" else "changed" + # A serializer that reads only the original Arrow buffers cannot reproduce + # these sentinels. This is the negative control for bypassing row hooks. + original_storage = array._pa_array.to_pylist() + assert original_storage != expected + + class HookError(Exception): + pass + + def customized(*args, **kwargs): + calls.append(target) + if behavior == "raise": + raise HookError(f"{target} sentinel") + result = original(*args, **kwargs).copy() + if target == "frame_isna": + result.iat[0, 0] = True + else: + result[0] = True if method == "isna" else "changed" + return result + + monkeypatch.setattr(owner, method, customized) + schema = _schema() + assert prepare_string_input(frame, schema) is None + assert calls == [] + tables = [] + for row_control in (False, True): + calls.clear() + with monkeypatch.context() as control: + if row_control: + control.setattr(pandas_input, "prepare_string_input", lambda *args: None) + with _arrow_input_service(protocol, config_error=True) as (spark, state): + if behavior == "raise": + with pytest.raises(HookError, match=f"^{target} sentinel$"): + spark.createDataFrame(frame, schema) + assert state["config_requests"] == [] + assert state["uploads"] == 0 + assert state["relations"] == [] + else: + result = spark.createDataFrame(frame, schema) + assert not result.isLocal() + result.toArrow() + _assert_inline(state, schema, [expected]) + tables.append(state["input_tables"][0]) + assert calls == [target] + if behavior == "mutate": + assert tables[0].equals(tables[1], check_metadata=True) + assert array._pa_array.to_pylist() == original_storage + + +def test_arrow_string_public_dtype_error_matches_native_row_control( + protocol, monkeypatch, missing_kind +): + frame = _frame({"text": ["plain"] * 10000}, missing_kind) + array = _backing(frame) + calls = [] + + class DtypeError(Exception): + pass + + def dtype(self): + calls.append("dtype") + raise DtypeError("dtype sentinel") + + monkeypatch.setattr(type(array), "dtype", property(dtype)) + # Eligibility and pandas normalization may consult array.dtype. Both public + # calls must preserve its exact descriptor error before any RPC. + for row_control in (False, True): + calls.clear() + with monkeypatch.context() as control: + if row_control: + control.setattr(pandas_input, "prepare_string_input", lambda *args: None) + with _arrow_input_service(protocol, config_error=True) as (spark, state): + with pytest.raises(DtypeError, match="^dtype sentinel$"): + spark.createDataFrame(frame, _schema()) + assert state["config_requests"] == [] + assert state["uploads"] == 0 + assert state["relations"] == [] + assert calls == ["dtype"] + + +def test_arrow_string_public_dtype_na_value_error_matches_native_row_control( + protocol, monkeypatch, missing_kind +): + frame = _frame({"text": [None] + ["plain"] * 9999}, missing_kind) + array = _backing(frame) + schema = _schema() + original_storage = array._pa_array + original_storage.validate(full=True) + assert original_storage.null_count == 1 + calls = [] + + class MissingValueError(Exception): + pass + + def na_value(self): + calls.append("na_value") + raise MissingValueError("StringDtype.na_value sentinel") + + monkeypatch.setattr(pd.StringDtype, "na_value", property(na_value)) + assert prepare_string_input(frame, schema) is None + assert calls == [] + errors = [] + for row_control in (False, True): + calls.clear() + with monkeypatch.context() as control: + if row_control: + control.setattr(pandas_input, "prepare_string_input", lambda *args: None) + with _arrow_input_service(protocol, config_error=True) as (spark, state): + with pytest.raises(MissingValueError, + match=r"^StringDtype\.na_value sentinel$") as error: + spark.createDataFrame(frame, schema) + errors.append((type(error.value), str(error.value))) + assert state["config_requests"] == [] + assert state["uploads"] == 0 + assert state["relations"] == [] + assert state["input_tables"] == [] + assert calls == ["na_value"] + assert errors[0] == errors[1] + # A direct serializer of this still-valid storage would succeed and omit + # the user-visible missing-value descriptor error proved above. + assert array._pa_array is original_storage + original_storage.validate(full=True) + + +def test_arrow_string_public_row_hook_error_order(protocol, monkeypatch, missing_kind): + frame = _frame({"text": ["plain"] * 10000}, missing_kind) + array = _backing(frame) + calls = [] + + class HookError(Exception): + pass + + def isna(self): + calls.append("isna") + raise HookError("isna before astype") + + def astype(self, *args, **kwargs): + calls.append("astype") + raise HookError("astype must follow notna") + + monkeypatch.setattr(type(array), "isna", isna) + monkeypatch.setattr(type(array), "astype", astype) + assert prepare_string_input(frame, _schema()) is None + assert calls == [] + with _arrow_input_service(protocol, config_error=True) as (spark, state): + with pytest.raises(HookError, match="isna before astype"): + spark.createDataFrame(frame, _schema()) + assert calls == ["isna"] + assert state["config_requests"] == [] + assert state["uploads"] == 0 + assert state["relations"] == [] + + +@pytest.mark.parametrize("reason", ["nonnullable_null", "cast", "width"]) +@pytest.mark.parametrize("verify", [False, True]) +def test_arrow_string_public_errors_precede_rpc(protocol, missing_kind, reason, verify): + from pyspark.errors import PySparkRuntimeError, PySparkValueError + + values = ["plain"] * 10000 + if reason == "nonnullable_null": + values[-1] = None + frame = _frame({"text": values}, missing_kind) + schema = _schema(nullable=reason != "nonnullable_null") + error = PySparkRuntimeError + match = "Column 'text' is declared as non-nullable but contains null values" + if reason == "cast": + schema = T.StructType([T.StructField("text", T.LongType())]) + match = "Type mismatch in row data" + elif reason == "width": + schema.add("extra", T.StringType()) + error = PySparkValueError + # The native row fallback now reports structured schema-width errors. + match = (r"^\[AXIS_LENGTH_MISMATCH\] Length mismatch: Expected axis has 2 elements, " + r"new values have 1 elements\.$") + assert prepare_string_input(frame, schema) is None + with _arrow_input_service(protocol, config_error=True) as (spark, state): + with pytest.raises(error, match=match) as raised: + spark.createDataFrame(frame, schema, verifySchema=verify) + if reason == "width": + assert type(raised.value) is PySparkValueError + assert raised.value.getCondition() == "AXIS_LENGTH_MISMATCH" + assert raised.value.getMessageParameters() == { + "expected_length": "2", "actual_length": "1", + } + assert state["config_requests"] == [] + assert state["uploads"] == 0 + assert state["relations"] == []