From 738d8a33726bd95d297ff94f2d95ba0314bbcbbf Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Thu, 19 Nov 2026 02:47:08 +0000 Subject: [PATCH] [SPARK-62066][CONNECT][PYTHON] Take local rows apart by the cached names of a StructType changed in place, raise PySpark's errors of the types that proto_schema_to_pyspark_data_type does not convert, and raise the MALFORMED conditions of no parameters, as PySpark PySpark's StructType caches its names as it is made and as add() adds a field, so an in-place change of its fields list, such as a StructField appended, popped or replaced, leaves names and fieldNames() as they were. Its conversion of local rows takes each row, and each struct value below the top level, apart by fieldNames(): the arity of a row and the keys of a dict are of the cached names, and the values convert by position with the fields as they are, so a field past them has no value (IndexError of row[i]); below the top level, a value past the fields has no converter, and fewer names than fields fail _deduplicate_field_names, both IndexError. A pandas DataFrame is named with toDF(*schema.fieldNames()). The drop-in took them apart by the names of the fields as they are. Now: - row_values and the struct converter take the cached names (infer::cached_names), which TypeObjects keeps of each struct below the top level, and convert by position; - infer::dedup_names_error raises the IndexError of fewer names than fields once the values are converted, before the Arrow schema; - the integer fast path of createDataFrame declines such a schema; - the pandas path renames the table by as many of the names as the fields, and applies toDF of fieldNames(). An element that is not a StructField still raises the drop-in's TypeError where the struct is lowered (dev/parity/known-differences.md). proto_schema_to_pyspark_data_type of a type of no kind, such as pb2.DataType() or an element type that an array does not set, raised a PySparkRuntimeError of the core ("DataType kind not set"), and of an unparsed type gave an UnparsedDataType. PySpark's last branch raises UNSUPPORTED_OPERATION of f"data type {schema}", of the proto text of the type, also of a nanosecond-precision timestamp, in the order of its conversion, of which a spatial type of an SRID that it does not take raises ST_INVALID_SRID_VALUE; it makes a user-defined type of its Python class, of no conversion of its SQL type. Now _data_type_of_proto finds the first such type in that order (connect_bridge::unconverted) and raises PySpark's error, of the text of the message of the drop-in's protobuf runtime, and drops the SQL type of a user-defined type that holds a type of no kind. MALFORMED_VARIANT, MALFORMED_GEOMETRY and MALFORMED_GEOGRAPHY of the conversion of local rows are raised of the condition alone (errors::raise_pyspark_condition), as PySpark's PySparkValueError(errorClass=...) of its converters, whose getMessageParameters() is None, where the drop-in's was {}. Co-authored-by: Isaac --- crates/pyspark-rs/src/connect_bridge.rs | 128 ++++++++- crates/pyspark-rs/src/errors.rs | 18 ++ crates/pyspark-rs/src/infer.rs | 245 ++++++++++++++---- crates/pyspark-rs/src/session.rs | 52 +++- dev/parity/known-differences.md | 14 + docs/compatibility.md | 11 + python/pyspark/sql/_pandas_input.py | 4 +- python/tests/test_dropin_e2e.py | 33 +++ python/tests/test_dropin_offline.py | 107 +++++++- .../test_dropin_types_columns_offline.py | 59 +++++ 10 files changed, 592 insertions(+), 79 deletions(-) diff --git a/crates/pyspark-rs/src/connect_bridge.rs b/crates/pyspark-rs/src/connect_bridge.rs index 6be449c8..48c61ce2 100644 --- a/crates/pyspark-rs/src/connect_bridge.rs +++ b/crates/pyspark-rs/src/connect_bridge.rs @@ -44,12 +44,134 @@ pub fn _data_type_proto<'py>( /// The type of the bytes of a `spark.connect.DataType`, as the client reads the types of the /// server, with the errors of PySpark's `proto_schema_to_pyspark_data_type` for a type that it -/// does not convert, such as a spatial type of an SRID that PySpark does not take. +/// does not convert (see [`unconverted`]), such as `pb2.DataType()` of no kind or a spatial type +/// of an SRID that PySpark does not take. PySpark makes a user-defined type of its Python class, +/// of no conversion of its SQL type, which is read only when the core can read it. #[pyfunction] pub fn _data_type_of_proto(py: Python<'_>, data_type: &[u8]) -> PyResult> { - let data_type = proto::DataType::decode(data_type) + let mut data_type = proto::DataType::decode(data_type) .map_err(|error| PyValueError::new_err(error.to_string()))?; + match unconverted(Some(&data_type)) { + Some(Unconverted::Kind(node)) => return Err(unsupported_data_type(py, &node)), + Some(Unconverted::Srid(srid)) => return Err(crate::types::invalid_schema_srid(py, srid)), + None => {} + } + drop_unread_sql_types(&mut data_type); let data_type = DataType::from_proto(&data_type).to_pyerr()?; - crate::dataframe::check_schema_conversion(py, Some(&data_type))?; crate::types::schema_to_py(py, &data_type) } + +/// Drops the SQL type of each user-defined type in `data_type` that holds a type of no kind, +/// which the core does not read and PySpark does not convert. +fn drop_unread_sql_types(data_type: &mut proto::DataType) { + use proto::data_type::Kind; + match &mut data_type.kind { + Some(Kind::Udt(udt)) => { + if udt.sql_type.as_deref().is_some_and(has_no_kind) { + udt.sql_type = None; + } + } + Some(Kind::Array(array)) => array + .element_type + .iter_mut() + .for_each(|t| drop_unread_sql_types(t)), + Some(Kind::Struct(st)) => st + .fields + .iter_mut() + .filter_map(|field| field.data_type.as_mut()) + .for_each(drop_unread_sql_types), + Some(Kind::Map(map)) => { + map.key_type + .iter_mut() + .for_each(|t| drop_unread_sql_types(t)); + map.value_type + .iter_mut() + .for_each(|t| drop_unread_sql_types(t)); + } + _ => {} + } +} + +/// Whether `data_type` is, or holds, a type of no kind, of which the core makes no type. +fn has_no_kind(data_type: &proto::DataType) -> bool { + use proto::data_type::Kind; + match &data_type.kind { + None => true, + Some(Kind::Array(array)) => array.element_type.as_deref().is_none_or(has_no_kind), + Some(Kind::Struct(st)) => st + .fields + .iter() + .any(|field| field.data_type.as_ref().is_none_or(has_no_kind)), + Some(Kind::Map(map)) => { + map.key_type.as_deref().is_none_or(has_no_kind) + || map.value_type.as_deref().is_none_or(has_no_kind) + } + Some(Kind::Udt(udt)) => udt.sql_type.as_deref().is_some_and(has_no_kind), + Some(_) => false, + } +} + +/// A type that PySpark's `proto_schema_to_pyspark_data_type` does not convert. +enum Unconverted { + /// A type of no kind, such as `pb2.DataType()` or the element type that an array does not + /// set, which PySpark reads as the empty message of the field, or of a kind that it has no + /// branch of, an unparsed type or a nanosecond-precision timestamp: the bytes of the type, of + /// whose proto text PySpark's last branch raises `UNSUPPORTED_OPERATION`. + Kind(Vec), + /// A spatial type of an SRID that PySpark's `GeometryType` or `GeographyType` does not take, + /// which raises `ST_INVALID_SRID_VALUE`. + Srid(i32), +} + +/// The first type of `data_type` that PySpark's `proto_schema_to_pyspark_data_type` does not +/// convert, in the order in which it converts them: the element of an array, the types of the +/// fields of a struct, and the key and then the value of a map. PySpark makes a user-defined type +/// of its Python class, of no conversion of its SQL type. +fn unconverted(data_type: Option<&proto::DataType>) -> Option { + use proto::data_type::Kind; + let Some(data_type) = data_type else { + return Some(Unconverted::Kind(Vec::new())); + }; + let spatial = |srid: i32, geographic: bool| { + (!crate::types::pyspark_converts_srid(srid, geographic)).then_some(Unconverted::Srid(srid)) + }; + match &data_type.kind { + None + | Some(Kind::Unparsed(_)) + | Some(Kind::TimestampNtzNanos(_)) + | Some(Kind::TimestampLtzNanos(_)) => Some(Unconverted::Kind(data_type.encode_to_vec())), + Some(Kind::Geometry(geometry)) => spatial(geometry.srid, false), + Some(Kind::Geography(geography)) => spatial(geography.srid, true), + Some(Kind::Array(array)) => unconverted(array.element_type.as_deref()), + Some(Kind::Struct(st)) => st + .fields + .iter() + .find_map(|field| unconverted(field.data_type.as_ref())), + Some(Kind::Map(map)) => { + unconverted(map.key_type.as_deref()).or_else(|| unconverted(map.value_type.as_deref())) + } + Some(_) => None, + } +} + +/// PySpark's `UNSUPPORTED_OPERATION` of `data type {schema}`, a `PySparkValueError`, of the proto +/// text of the type of the bytes `node`, as `str()` of its message gives it: empty of a type of no +/// kind. +fn unsupported_data_type(py: Python<'_>, node: &[u8]) -> PyErr { + let text = || -> PyResult { + let message = py + .import("pyspark.sql.connect.proto")? + .getattr("DataType")? + .call_method1("FromString", (PyBytes::new(py, node),))?; + Ok(message.str()?.to_cow()?.into_owned()) + }; + match text() { + Ok(text) => crate::errors::raise_pyspark( + py, + "PySparkValueError", + "UNSUPPORTED_OPERATION", + &[("operation", &format!("data type {text}"))], + ), + Err(err) => err, + } +} diff --git a/crates/pyspark-rs/src/errors.rs b/crates/pyspark-rs/src/errors.rs index 24ee8fda..310306b8 100644 --- a/crates/pyspark-rs/src/errors.rs +++ b/crates/pyspark-rs/src/errors.rs @@ -28,6 +28,24 @@ pub(crate) fn raise_pyspark( build().unwrap_or_else(|e| e) } +/// Raise a `pyspark.errors` exception (`class_name`) of the `error_class` condition alone, as +/// PySpark raises some of them without `messageParameters`, such as `MALFORMED_VARIANT` of the +/// conversion of local rows: its `getMessageParameters()` is None, where that of +/// [`raise_pyspark`] of no parameters is `{}`. +pub(crate) fn raise_pyspark_condition( + py: Python<'_>, + class_name: &str, + error_class: &str, +) -> PyErr { + let build = || -> PyResult { + let cls = py.import("pyspark.errors")?.getattr(class_name)?; + let kwargs = PyDict::new(py); + kwargs.set_item("errorClass", error_class)?; + Ok(PyErr::from_value(cls.call((), Some(&kwargs))?)) + }; + build().unwrap_or_else(|e| e) +} + /// As [`raise_pyspark`], of parameters that are Python objects, as PySpark formats them into its /// message: a str that UTF-8 cannot encode, such as one of a lone surrogate, shows as it is. pub(crate) fn raise_pyspark_of( diff --git a/crates/pyspark-rs/src/infer.rs b/crates/pyspark-rs/src/infer.rs index f3b4ee10..e648f2bb 100644 --- a/crates/pyspark-rs/src/infer.rs +++ b/crates/pyspark-rs/src/infer.rs @@ -7,7 +7,8 @@ use std::collections::{HashMap, HashSet}; use pyo3::exceptions::{ - PyAssertionError, PyNotImplementedError, PyOverflowError, PyTypeError, PyValueError, + PyAssertionError, PyIndexError, PyNotImplementedError, PyOverflowError, PyTypeError, + PyValueError, }; use pyo3::intern; use pyo3::prelude::*; @@ -1529,6 +1530,10 @@ pub(crate) struct TypeObjects<'py> { /// The objects of the fields of a struct, in their order; of the element of an array; of the /// key and the value of a map; or of the `sqlType()` of a user-defined type. children: Vec>, + /// The names by which PySpark's converter of a struct below the top of the schema takes its + /// values apart, `fieldNames()` of its type object, when they are not those of its fields as + /// they are (see [`cached_names`]). + names: Option>>, } impl<'py> TypeObjects<'py> { @@ -1536,41 +1541,55 @@ impl<'py> TypeObjects<'py> { /// makes its converters of them: of the `dataType` of each of its `fields`, its `elementType`, /// its `keyType` and `valueType`, and of the `sqlType()` of a user-defined type. pub(crate) fn of(obj: &Bound<'py, PyAny>, dt: &DataType) -> PyResult { + Self::of_place(obj, dt, false) + } + + /// As [`Self::of`], `nested` below the top of the schema, where PySpark's converter of a + /// struct reads its `fieldNames()`, which the top level takes of the rows (see + /// [`cached_names`]). + fn of_place(obj: &Bound<'py, PyAny>, dt: &DataType, nested: bool) -> PyResult { let py = obj.py(); + let mut names = None; let children = match dt { DataType::Udt { sql_type: Some(sql_type), .. - } => vec![Self::of( + } => vec![Self::of_place( &obj.call_method0(intern!(py, "sqlType"))?, sql_type, + true, )?], DataType::Struct { fields } => { + if nested { + names = cached_names(obj, fields)?; + } let objects = obj.getattr(intern!(py, "fields"))?; let mut children = Vec::with_capacity(fields.len()); for (field, object) in fields.iter().zip(objects.try_iter()?) { let data_type = object?.getattr(intern!(py, "dataType"))?; - children.push(Self::of(&data_type, &field.data_type)?); + children.push(Self::of_place(&data_type, &field.data_type, true)?); } children } - DataType::Array { element_type, .. } => vec![Self::of( + DataType::Array { element_type, .. } => vec![Self::of_place( &obj.getattr(intern!(py, "elementType"))?, element_type, + true, )?], DataType::Map { key_type, value_type, .. } => vec![ - Self::of(&obj.getattr(intern!(py, "keyType"))?, key_type)?, - Self::of(&obj.getattr(intern!(py, "valueType"))?, value_type)?, + Self::of_place(&obj.getattr(intern!(py, "keyType"))?, key_type, true)?, + Self::of_place(&obj.getattr(intern!(py, "valueType"))?, value_type, true)?, ], _ => Vec::new(), }; Ok(Self { object: Some(obj.clone()), children, + names, }) } @@ -1580,6 +1599,7 @@ impl<'py> TypeObjects<'py> { Self { object: Some(udt), children: Vec::new(), + names: None, } } @@ -1589,6 +1609,7 @@ impl<'py> TypeObjects<'py> { Self { object: None, children: fields, + names: None, } } @@ -1609,6 +1630,7 @@ impl<'py> TypeObjects<'py> { Self { object: None, children, + names: None, } } @@ -1626,7 +1648,7 @@ impl<'py> TypeObjects<'py> { } => { if let Some(udt) = &self.object { let sql = udt.call_method0(intern!(udt.py(), "sqlType"))?; - self.children = vec![Self::of(&sql, sql_type)?]; + self.children = vec![Self::of_place(&sql, sql_type, true)?]; } } DataType::Struct { fields } => { @@ -1658,6 +1680,12 @@ impl<'py> TypeObjects<'py> { self.object.as_ref() } + /// The names by which a value of the struct of this place is taken apart, when they are not + /// those of its fields as they are (see [`cached_names`]). + fn names(&self) -> Option<&[Bound<'py, PyAny>]> { + self.names.as_deref() + } + /// The objects of the field `index` of a struct. pub(crate) fn field(&self, index: usize) -> Option<&Self> { self.children.get(index) @@ -1708,6 +1736,7 @@ impl<'py> FieldObjects<'py> { TypeObjects { object: None, children: self.0, + names: None, } } } @@ -1879,6 +1908,8 @@ pub(crate) fn convert<'py>( // `metadata` attributes of a VariantVal, and the `srid` and `wkb` of a Geometry or a // Geography, which PyArrow converts as the fields of the structs of their Arrow types. A // spatial type of the schema of a Python subclass converts with its own (`spatial_value`). + // Another value raises the `MALFORMED_*` condition of no message parameters, as PySpark's + // `PySparkValueError(errorClass=...)` of its converter. DataType::Variant => { if obj.is_instance(variant_val_class(py)?)? { let value = obj.getattr(intern!(py, "value"))?; @@ -1890,11 +1921,10 @@ pub(crate) fn convert<'py>( }) })) } else { - Err(raise_pyspark( + Err(crate::errors::raise_pyspark_condition( py, "PySparkValueError", "MALFORMED_VARIANT", - &[], )) } } @@ -1903,11 +1933,10 @@ pub(crate) fn convert<'py>( let native = py.get_type::(); spatial_value(obj, objects.and_then(TypeObjects::object), &native, late) } else { - Err(raise_pyspark( + Err(crate::errors::raise_pyspark_condition( py, "PySparkValueError", "MALFORMED_GEOMETRY", - &[], )) } } @@ -1916,11 +1945,10 @@ pub(crate) fn convert<'py>( let native = py.get_type::(); spatial_value(obj, objects.and_then(TypeObjects::object), &native, late) } else { - Err(raise_pyspark( + Err(crate::errors::raise_pyspark_condition( py, "PySparkValueError", "MALFORMED_GEOGRAPHY", - &[], )) } } @@ -2383,39 +2411,119 @@ fn convert_integral(obj: &Bound<'_, PyAny>, dt: &DataType) -> PyResult { }) } -/// The values of `fields` from a mapping, by their names, as `mapping.get(name)`. +/// The values of a struct value from a mapping, as `mapping.get(name)` of each of `names`, or of +/// the names of `fields` without them. fn values_by_name<'py>( mapping: &Bound<'py, PyAny>, fields: &[StructField], + names: Option<&[Bound<'py, PyAny>]>, ) -> PyResult>> { let py = mapping.py(); + let Some(names) = names else { + let names = fields + .iter() + .map(|f| PyString::new(py, &f.name).into_any()) + .collect::>(); + return values_by_name(mapping, fields, Some(&names)); + }; if let Ok(dict) = mapping.cast::() { - return fields + return names .iter() - .map(|f| { + .map(|name| { Ok(dict - .get_item(&f.name)? + .get_item(name)? .unwrap_or_else(|| py.None().into_bound(py))) }) .collect(); } - fields + names .iter() - .map(|f| mapping.call_method1(intern!(py, "get"), (&f.name,))) + .map(|name| mapping.call_method1(intern!(py, "get"), (name,))) .collect() } -/// The values of a tuple or a Row, which must be as many as `fields`. +/// The values of a tuple or a Row, which must be as many as the names that the struct value is +/// taken apart by, `len`. fn values_by_position<'py>( py: Python<'py>, values: Vec>, - fields: &[StructField], + len: usize, ) -> PyResult>> { - if values.len() == fields.len() { + if values.len() == len { Ok(values) } else { - Err(axis_length_mismatch(py, fields.len(), values.len())) + Err(axis_length_mismatch(py, len, values.len())) + } +} + +/// The names by which PySpark's converters take a value of the struct `obj` of the given schema +/// apart, `obj.fieldNames()`, which `to_row` reads of the schema and the converter of a struct of +/// its type object, when they are not the names of `fields`, the fields of `obj` as they are: +/// PySpark's `StructType` caches its `names` as it is made (and as `add` adds a field), so of a +/// `fields` list changed in place, such as of a `StructField` appended, they are the names before +/// the change, while the converters are of the fields as they are. None of the type of another +/// class, or of names that are those of the fields. +pub(crate) fn cached_names<'py>( + obj: &Bound<'py, PyAny>, + fields: &[StructField], +) -> PyResult>>> { + if !obj.is_instance_of::() { + return Ok(None); + } + let names = obj + .call_method0(intern!(obj.py(), "fieldNames"))? + .try_iter()? + .collect::>>()?; + let same = names.len() == fields.len() + && names.iter().zip(fields).all(|(name, field)| { + name.cast::() + .is_ok_and(|name| name.to_cow().is_ok_and(|name| name == field.name)) + }); + Ok((!same).then_some(names)) +} + +/// The `IndexError` that PySpark's conversion of local rows raises once their values are +/// converted, as it makes the Arrow schema of the fields of the schema, whose +/// `_deduplicate_field_names` names the fields of each struct below the top level, in an array +/// or a map too, by the cached names of its type object (see [`cached_names`]): of fewer names +/// than fields, of a `fields` list that a field was added to in place. Its converters stop at a +/// user-defined type, of whose SQL type the schema has no struct. +pub(crate) fn dedup_names_error( + objects: Option<&TypeObjects<'_>>, + fields: &[StructField], +) -> Option { + fn short<'py>(objects: Option<&TypeObjects<'py>>, dt: &DataType) -> bool { + let Some(objects) = objects else { + return false; + }; + match dt { + DataType::Struct { fields } => { + objects + .names() + .is_some_and(|names| names.len() < fields.len()) + || fields + .iter() + .enumerate() + .any(|(i, field)| short(objects.field(i), &field.data_type)) + } + DataType::Array { element_type, .. } => short(objects.element(), element_type), + DataType::Map { + key_type, + value_type, + .. + } => { + let (key, value) = objects.key_and_value(); + short(key, key_type) || short(value, value_type) + } + _ => false, + } } + let objects = objects?; + fields + .iter() + .enumerate() + .any(|(i, field)| short(objects.field(i), &field.data_type)) + .then(|| PyIndexError::new_err("list index out of range")) } /// A struct value, as PySpark converts one: a tuple or a Row by position, a dict by name, @@ -2429,15 +2537,19 @@ fn convert_struct<'py>( budget: &ConversionBudget, ) -> PyResult { let py = obj.py(); + // Of the cached names of a `fields` list changed in place, if they differ (see + // `cached_names`). + let names = objects.and_then(TypeObjects::names); + let len = names.map_or(fields.len(), <[_]>::len); let values = if let Ok(tuple) = obj.cast::() { if !reads_as_stored(tuple, true)? { - return struct_by_index(obj, fields, objects, late, budget); + return struct_by_index(obj, fields, len, objects, late, budget); } - values_by_position(py, tuple.iter().collect(), fields)? + values_by_position(py, tuple.iter().collect(), len)? } else if obj.is_instance_of::() { - values_by_name(obj, fields)? + values_by_name(obj, fields, names)? } else if let Ok(attributes) = obj.getattr(intern!(py, "__dict__")) { - values_by_name(&attributes, fields)? + values_by_name(&attributes, fields, names)? } else { // PySpark asserts that another value has a `__dict__`. return Err(PyAssertionError::new_err(format!( @@ -2457,36 +2569,54 @@ fn struct_of<'py>( budget: &ConversionBudget, ) -> PyResult { let names: std::sync::Arc<[String]> = fields.iter().map(|f| f.name.clone()).collect(); - let values = values - .iter() - .zip(fields) - .enumerate() - .map(|(index, (value, f))| { - let field_objects = objects.and_then(|objects| objects.field(index)); - convert(value, &f.data_type, field_objects, f.nullable, late, budget) - }) - .collect::>>()?; - Ok(Value::Struct(StructValue::new(names, values))) + // Each value is converted with the field of its position, as PySpark's `field_convs[i]` of + // the values of the cached names of a `fields` list changed in place (see `cached_names`): + // past the fields, its `IndexError`, once those before are converted; short of them, the + // fields after are null, and the Arrow schema of the conversion raises once every value is + // converted (`dedup_names_error`). + let mut converted = Vec::with_capacity(fields.len()); + for (index, value) in values.iter().enumerate() { + let Some(f) = fields.get(index) else { + return Err(PyIndexError::new_err("list index out of range")); + }; + let field_objects = objects.and_then(|objects| objects.field(index)); + converted.push(convert( + value, + &f.data_type, + field_objects, + f.nullable, + late, + budget, + )?); + } + converted.resize_with(fields.len(), || Value::Null); + Ok(Value::Struct(StructValue::new(names, converted))) } /// A struct value of a tuple of a subclass of its own `__len__` or `__getitem__`, as PySpark's -/// converter reads one: `len(value)` must be the number of fields, and `value[i]` of each field -/// is read as the value before it is converted. +/// converter reads one: `len(value)` must be `len`, the number of the names that it is taken +/// apart by (see `cached_names`), and `value[i]` of each is read as the value before it is +/// converted with the field of its position, of which PySpark reads the converter first: past +/// the fields, its `IndexError`, and short of them, the fields after are null. fn struct_by_index<'py>( obj: &Bound<'py, PyAny>, fields: &[StructField], + len: usize, objects: Option<&TypeObjects<'py>>, late: &mut Late, budget: &ConversionBudget, ) -> PyResult { let py = obj.py(); - if obj.len()? != fields.len() { + if obj.len()? != len { // PySpark's message reads `len(value)` again. - return Err(axis_length_mismatch(py, fields.len(), obj.len()?)); + return Err(axis_length_mismatch(py, len, obj.len()?)); } let names: std::sync::Arc<[String]> = fields.iter().map(|f| f.name.clone()).collect(); let mut values = Vec::with_capacity(fields.len()); - for (index, f) in fields.iter().enumerate() { + for index in 0..len { + let Some(f) = fields.get(index) else { + return Err(PyIndexError::new_err("list index out of range")); + }; let field_objects = objects.and_then(|objects| objects.field(index)); let value = obj.get_item(index)?; values.push(convert( @@ -2498,6 +2628,7 @@ fn struct_by_index<'py>( budget, )?); } + values.resize_with(fields.len(), || Value::Null); Ok(Value::Struct(StructValue::new(names, values))) } @@ -2571,41 +2702,43 @@ pub(crate) fn variant_row(py: Python<'_>) -> PyErr { /// by `len(item)`, which must be the number of fields, and `tuple(item)`, of the `__len__` and /// `__iter__` of a subclass too, which may give fewer or more values than the fields: PySpark's /// `row[i]` of a missing one raises `IndexError` as its column converts, and those after the -/// fields are not read (`create_from_local`). +/// fields are not read (`create_from_local`). With `names`, the cached names of the schema that +/// differ from its fields (see [`cached_names`]), the row is taken apart by them, as `to_row` +/// reads `schema.fieldNames()`, and its values are converted with the fields of their positions. pub(crate) fn row_values<'py>( item: &Bound<'py, PyAny>, fields: &[StructField], + names: Option<&[Bound<'py, PyAny>]>, ) -> PyResult>> { let py = item.py(); + let len = names.map_or(fields.len(), <[_]>::len); if item.is_none() { - return Ok(vec![py.None().into_bound(py); fields.len()]); + return Ok(vec![py.None().into_bound(py); len]); } if let Ok(tuple) = item.cast::() { if !reads_as_stored(tuple, false)? { - return sequence_values(item, fields); + return sequence_values(item, len); } - return values_by_position(py, tuple.iter().collect(), fields); + return values_by_position(py, tuple.iter().collect(), len); } if item.is_instance_of::() { - return values_by_name(item, fields); + return values_by_name(item, fields, names); } if item.is_instance(variant_val_class(py)?)? { return Err(variant_row(py)); } if item.hasattr(intern!(py, "__dict__"))? { - return values_by_name(&item.getattr(intern!(py, "__dict__"))?, fields); + return values_by_name(&item.getattr(intern!(py, "__dict__"))?, fields, names); } - sequence_values(item, fields) + sequence_values(item, len) } -/// The values of `tuple(item)` of a row whose `len(item)` must be the number of `fields`. -fn sequence_values<'py>( - item: &Bound<'py, PyAny>, - fields: &[StructField], -) -> PyResult>> { - if item.len()? != fields.len() { +/// The values of `tuple(item)` of a row whose `len(item)` must be `len`, the number of the names +/// that the row is taken apart by. +fn sequence_values<'py>(item: &Bound<'py, PyAny>, len: usize) -> PyResult>> { + if item.len()? != len { // PySpark's message reads `len(item)` again. - return Err(axis_length_mismatch(item.py(), fields.len(), item.len()?)); + return Err(axis_length_mismatch(item.py(), len, item.len()?)); } Ok(sequence_tuple(item)?.iter().collect()) } diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs index e643f2bb..f7d7a7cb 100644 --- a/crates/pyspark-rs/src/session.rs +++ b/crates/pyspark-rs/src/session.rs @@ -1670,6 +1670,7 @@ impl PySparkSession { use crate::infer; let atomic = matches!(spec, Spec::Atomic(_)); + let given_struct = matches!(spec, Spec::Struct(_)); let (given, mut cols) = match spec { Spec::None => (None, None), Spec::Names(names) => (None, Some(names)), @@ -1773,17 +1774,24 @@ impl PySparkSession { } }; - // As PySpark: every row is taken apart first, then each column is converted. + // As PySpark: every row is taken apart first, then each column is converted. The rows are + // taken apart by `schema.fieldNames()` of the given `StructType`, which of a `fields` list + // changed in place are the names before the change (`infer::cached_names`). + let names = match (schema, given_struct) { + (Some(schema), true) => infer::cached_names(schema, &fields)?, + _ => None, + }; let mut parts: Vec>> = Vec::with_capacity(items.len()); for item in &items { budget.step(py)?; parts.push(if single { - if fields.len() != 1 { - return Err(infer::axis_length_mismatch(py, fields.len(), 1)); + let len = names.as_ref().map_or(fields.len(), Vec::len); + if len != 1 { + return Err(infer::axis_length_mismatch(py, len, 1)); } vec![item.clone()] } else { - infer::row_values(item, &fields)? + infer::row_values(item, &fields, names.as_deref())? }); } // As PySpark's converters, which assert the nullable flags of the schema when they are @@ -1851,6 +1859,11 @@ impl PySparkSession { } // The rows of a struct of no fields have no values. values.resize_with(row_count, Vec::new); + // As PySpark's Arrow schema of the fields, whose names of the structs below the top level + // are their cached names, of which fewer than the fields raise (`infer::dedup_names_error`). + if let Some(err) = infer::dedup_names_error(objects.as_ref(), &fields) { + return Err(err); + } // Then, as PySpark, the schema converts to Arrow, a field at a time and depth first, // which rejects a null type that is not nullable, and has no char, varchar, year-month // or calendar interval type, nor a type of its class only such as `AtomicType()` or a @@ -1975,16 +1988,21 @@ impl PySparkSession { .create_dataframe_from_arrow_with_options(batches, Some(spark_schema), &options) .map_err(Box::new) })?; - // As the other pandas paths, the explicit names are applied with toDF, when there are any. - Ok(Some(PyDataFrame::new(if fields.is_empty() { + // As the other pandas paths, the explicit names are applied with toDF, when there are any: + // PySpark's `toDF(*schema.fieldNames())`, which of a `fields` list changed in place are the + // names before the change, of which the server raises for another number of columns + // (`infer::cached_names`). + let names = match crate::infer::cached_names(schema, fields)? { + Some(names) => names + .iter() + .map(|name| Ok(name.str()?.to_cow()?.into_owned())) + .collect::>>()?, + None => fields.iter().map(|field| field.name.clone()).collect(), + }; + Ok(Some(PyDataFrame::new(if names.is_empty() { df } else { - df.to_df( - fields - .iter() - .map(|field| field.name.as_str()) - .collect::>(), - ) + df.to_df(names.iter().map(String::as_str).collect::>()) }))) } @@ -3108,10 +3126,16 @@ impl PySparkSession { // Preserve schema parsing, ignored-argument warnings, and pandas/Arrow routing. // Only a native StructType opts into this explicit integer optimization; // DDL, inferred schemas and unsupported values keep schema-driven conversion. - if schema.is_some_and(|s| s.extract::>().is_ok()) { + if let Some(struct_type) = + schema.filter(|s| s.extract::>().is_ok()) + { if let Spec::Struct(fields) = &spec { if let Ok(data) = data.cast::() { - if let Some(columns) = exact_long_tuple_columns(data, fields)? { + // Not of a `fields` list changed in place, whose rows are taken apart by the + // names before the change (`infer::cached_names`). + if let Some(columns) = exact_long_tuple_columns(data, fields)?.filter(|_| { + matches!(crate::infer::cached_names(struct_type, fields), Ok(None)) + }) { let schema = DataType::Struct { fields: fields.clone(), }; diff --git a/dev/parity/known-differences.md b/dev/parity/known-differences.md index a3429af3..ab6a95d1 100644 --- a/dev/parity/known-differences.md +++ b/dev/parity/known-differences.md @@ -120,6 +120,20 @@ difference is intentional or tracked for a fix. `timestamp_seconds(-62135596800)`, 0001-01-01 00:00:00 UTC, with the session time zone `America/Los_Angeles` on a host in UTC, this client returns `datetime(1, 1, 1, 0, 0)`. Both raise the `OverflowError` of a wall clock out of the range in the local time zone. +- **A `StructType` whose `fields` hold an element that is not a `StructField`** (inherent; the + uses of the native type lower its fields into the core's struct type, which holds fields only). + PySpark's `StructType` methods read the attributes of each element of `fields` as they go, so of + a list that a str or another object was appended to in place, each use raises the + `AttributeError` of the first attribute that it reads (`simpleString`, `jsonValue`, `name`, + `dataType`, `needConversion`), or uses an object that has them, and its pickles and copies keep + the element; `createDataFrame` of local rows takes them apart by the cached names first, so a + row of another number of values raises `AXIS_LENGTH_MISMATCH` before that. This client lowers the + list into the core's struct type at each use that needs the type (its JSON, simple string and + tree string, a lookup by name, `add`, its pickles and copies, the plans of `to`, `cast`, + `from_json` and a reader's schema, and `createDataFrame`), and raises + `TypeError: StructType fields must be StructField objects` there, and an assignment of `fields` + of such an element raises it at once. Of a list of `StructField`s changed in place, the cached + `names` and their uses are PySpark's (see `docs/compatibility.md`). - **The `__repr__` of a Column of a str that UTF-8 cannot encode** (intentional; the expressions of the core hold UTF-8). Such a str, of a name, a SQL string or a literal, fails the plans of the Column when they are built, as in PySpark, and its lone surrogates show escaped in the `__repr__` diff --git a/docs/compatibility.md b/docs/compatibility.md index 43820e3c..c50e32b9 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -2206,6 +2206,17 @@ PySpark's shares the list of its fields. The class's own types compare without t a user set, where PySpark's comparison of the `__dict__`s holds them: a `DecimalType(7, 3)` of an attribute equals `DecimalType(7, 3)`, where a type of a Python subclass compares them (see below). +A `StructType` caches its `names` as it is made and as `add` adds a field, as PySpark's, so an +in-place change of its `fields` list, such as a `StructField` appended, leaves `names` and +`fieldNames()` as they were. `createDataFrame` takes local rows, and the struct values below the +top level, apart by those names, as PySpark's conversion reads `fieldNames()`, and converts their +values by position with the fields as they are: a row of another number of values than the names +raises `AXIS_LENGTH_MISMATCH`, a dict is read by them, and a field past them has no value +(`IndexError`); below the top level, a value past the fields and fewer names than fields raise +`IndexError` too. A pandas DataFrame is named with `toDF(*schema.fieldNames())`, of which the server +raises for another number of names than columns. An element that is not a `StructField` is not +taken (see `dev/parity/known-differences.md`). + The parameters of a type are its attributes, as PySpark's: `t.scale = 3` makes a `DecimalType(10, 2)` a `DecimalType(10, 3)`, as its repr, JSON, `==` and hash show, also in the struct, array or map that holds it. A drop-in type holds only the values that its constructor diff --git a/python/pyspark/sql/_pandas_input.py b/python/pyspark/sql/_pandas_input.py index dc419c54..61e0a126 100644 --- a/python/pyspark/sql/_pandas_input.py +++ b/python/pyspark/sql/_pandas_input.py @@ -1513,7 +1513,9 @@ def prepare_pandas_table(frame, schema, timezone, safecheck, prefers_large_types ) if len(schema.fields) == 0: return table - return table.rename_columns(schema.names).cast(arrow_schema) + # The names of the fields, of a `fields` list changed in place the cached `names` of as many, + # as PySpark's names of `_deduplicate_field_names`, which raises of fewer. + return table.rename_columns(schema.names[: len(schema.fields)]).cast(arrow_schema) def arrow_decimal_column(values, dtype, name): diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py index c98f150a..bb5fae2e 100644 --- a/python/tests/test_dropin_e2e.py +++ b/python/tests/test_dropin_e2e.py @@ -5412,6 +5412,39 @@ def deserialize(self, datum): return datum // self.scale +def test_create_dataframe_of_a_struct_changed_in_place_reads_its_cached_names_as_pyspark(session): + # As official PySpark 4.3.0, whose StructType caches its `names` as it is made, which an + # in-place change of its `fields` list leaves as they were: createDataFrame takes local rows, + # and the struct values below the top level, apart by those names (`fieldNames()`), and + # converts their values by position with the fields as they are; a pandas DataFrame is + # named with `toDF(*schema.fieldNames())`. + pd = pytest.importorskip("pandas") + appended = StructType([StructField("a", LongType())]) + appended.fields.append(StructField("b", StringType())) + popped = StructType([StructField("a", LongType()), StructField("b", StringType())]) + popped.fields.pop() + replaced = StructType([StructField("a", LongType())]) + replaced.fields[0] = StructField("z", StringType()) + for data, schema in (([(1, "q")], appended), ([(1,)], popped), + ([((1, "q"),)], StructType([StructField("s", appended)]))): + with pytest.raises(PySparkValueError, match="AXIS_LENGTH_MISMATCH"): + session.createDataFrame(data, schema) + with pytest.raises(IndexError, match="tuple index out of range"): + session.createDataFrame([(1,)], appended) + for data, inner in (([((1,),)], appended), ([(None,)], appended), ([({"a": 1},)], popped)): + with pytest.raises(IndexError, match="list index out of range"): + session.createDataFrame(data, StructType([StructField("s", inner)])) + assert session.createDataFrame([(1, "q")], popped).collect() == [Row(a=1)] + assert session.createDataFrame([{"a": 1}], replaced).collect() == [Row(z="1")] + nested = StructType([StructField("s", replaced)]) + assert session.createDataFrame([({"a": 1},)], nested).collect() == [Row(s=Row(z="1"))] + string = StructType([StructField("a", StringType())]) + string.fields[0] = StructField("z", StringType()) + assert session.createDataFrame(pd.DataFrame({"a": ["x"]}), string).collect() == [Row(a="x")] + with pytest.raises(AnalysisException, match="ASSIGNMENT_ARITY_MISMATCH"): + session.createDataFrame(pd.DataFrame({"a": [1]}), popped).collect() + + def test_local_rows_are_serialized_by_the_udt_objects_of_the_schema_as_pyspark(session): # As official PySpark 4.3.0, which serializes each value of local rows with the UDT object # that the schema holds: two UDTs of one class and other parameters keep their own, in a diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 11a97c69..60b08d8c 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -6848,7 +6848,9 @@ def test_variant_spatial_and_user_defined_values_are_sent_as_pyspark(mock_connec # As PySpark's LocalDataToArrowConversion: a VariantVal is the struct of its value and # metadata, a Geometry or a Geography that of its srid and WKB, whose metadata tags the # struct as of its type, and a value of a user-defined type is serialized by the UDT and - # sent as its SQL type; a value of another type for these is malformed. + # sent as its SQL type; a value of another type for these is malformed, of the condition + # alone, whose message parameters are None, as PySpark's `PySparkValueError(errorClass=...)` + # of its converters, also nested. import pyarrow as pa from pyspark.errors import PySparkValueError from pyspark.testing.objects import ExamplePoint @@ -6889,12 +6891,15 @@ def local_relation(df): "_2": {"srid": 4326, "wkb": wkb}, "_3": {"srid": 4326, "wkb": wkb}, "_4": [1.0, 2.0]}, {"_1": None, "_2": None, "_3": None, "_4": None}, ] - for ddl, condition in (("v variant", "MALFORMED_VARIANT"), - ("g geometry(4326)", "MALFORMED_GEOMETRY"), - ("g geography(4326)", "MALFORMED_GEOGRAPHY")): + for ddl, value, condition in (("v variant", "x", "MALFORMED_VARIANT"), + ("g geometry(4326)", "x", "MALFORMED_GEOMETRY"), + ("g geography(4326)", 1, "MALFORMED_GEOGRAPHY"), + ("v array", ["x"], "MALFORMED_VARIANT"), + ("s struct", ("x",), "MALFORMED_GEOMETRY")): with pytest.raises(PySparkValueError) as error: - spark.createDataFrame([("x",)], ddl) + spark.createDataFrame([(value,)], ddl) assert error.value.getCondition() == condition + assert error.value.getMessageParameters() is None # The attributes of PySpark's spatial values. geometry = T.Geometry.fromWKB(wkb, 4326) assert (geometry.srid, geometry.wkb) == (4326, wkb) @@ -23135,6 +23140,98 @@ def test_local_rows_are_serialized_by_the_udt_objects_of_the_schema_as_pysparks( assert table.to_pylist() == [{"a": 6, "b": 40}] +def _struct_changed_in_place(change): + # A StructType whose `fields` list was changed in place, whose cached `names` stay as they + # were made: a field appended, the last one popped, or the first one replaced by another. + if change == "appended": + st = T.StructType([T.StructField("a", T.LongType())]) + st.fields.append(T.StructField("b", T.StringType())) + elif change == "popped": + st = T.StructType([T.StructField("a", T.LongType()), T.StructField("b", T.StringType())]) + st.fields.pop() + else: + st = T.StructType([T.StructField("a", T.LongType())]) + st.fields[0] = T.StructField("z", T.StringType()) + return st + + +def test_local_rows_of_a_struct_changed_in_place_are_taken_apart_by_its_cached_names(mock_connect): + # As PySpark's LocalDataToArrowConversion, whose `to_row`, and the converter of a struct + # below the top level, take each value apart by `fieldNames()` of the StructType, its cached + # `names`, which an in-place change of its `fields` leaves as they were, and convert the + # values by position with the fields as they are: the arity of a row is the number of the + # names, a dict is read by them, and a field past them has no value (`IndexError` of + # `row[i]`). Below the top level, a value past the fields has no converter, and the Arrow + # schema of fewer names than fields fails (`_deduplicate_field_names`), both `IndexError`. + # The integer fast path takes them so too. As official PySpark 4.3.0 (see + # test_dropin_e2e.py); the drop-in took them apart by the names of the fields as they are. + from pyspark.errors import PySparkValueError + + spark, service = mock_connect + appended, popped, replaced = ( + _struct_changed_in_place(change) for change in ("appended", "popped", "replaced")) + for data, schema in ( + ([(1, "q")], appended), ([Row(a=1, b="q")], appended), ([[1, "q"]], appended), + ([(1,)], popped), ([(1,), (2,)], popped), ([[1]], popped), ([None], popped), + ([1], popped), + ): + with pytest.raises(PySparkValueError) as error: + spark.createDataFrame(data, schema) + assert error.value.getCondition() == "AXIS_LENGTH_MISMATCH" + for data in ([(1,)], [{"a": 1, "b": "q"}], [None], [1]): + with pytest.raises(IndexError, match="tuple index out of range"): + spark.createDataFrame(data, appended) + table, _ = _local_rows_relation_of(service, spark.createDataFrame([(1, "q")], popped)) + assert table.to_pylist() == [{"a": 1}] + table, _ = _local_rows_relation_of( + service, spark.createDataFrame([{"a": 1, "z": 2}, {"z": 3}], replaced)) + assert table.to_pylist() == [{"z": "1"}, {"z": None}] + + def nested(inner): + return T.StructType([T.StructField("s", inner)]) + + for data, schema in (([((1, "q"),)], appended), ([((1,),)], popped)): + with pytest.raises(PySparkValueError) as error: + spark.createDataFrame(data, nested(schema)) + assert error.value.getCondition() == "AXIS_LENGTH_MISMATCH" + for data, schema in ( + ([((1,),)], appended), ([({"a": 1},)], appended), ([(None,)], appended), + ([((1, "q"),)], popped), ([({"a": 1},)], popped), ([(Row(a=1, b="q"),)], popped), + ): + with pytest.raises(IndexError, match="list index out of range"): + spark.createDataFrame(data, nested(schema)) + table, _ = _local_rows_relation_of( + service, spark.createDataFrame([({"a": 1, "z": 2},), ((3,),)], nested(replaced))) + assert table.to_pylist() == [{"s": {"z": "1"}}, {"s": {"z": "3"}}] + + +def test_pandas_frames_of_a_struct_changed_in_place_are_named_by_its_cached_names(mock_connect): + # As PySpark's createDataFrame of a pandas DataFrame, which names its columns with + # `toDF(*schema.fieldNames())`, of the cached `names` of a StructType whose `fields` list + # was changed in place, after the table of its fields as they are is renamed by as many of + # them, as official PySpark 4.3.0 (see test_dropin_e2e.py); the server raises of another + # number of names than columns. The drop-in named them by the fields as they are, and + # failed the rename of more names than columns. + pd = pytest.importorskip("pandas") + spark, service = mock_connect + + def column_names(df): + df.toArrow() + request = parse_message(service.execute_requests[-1]) + relation = parse_message(parse_message(request[3][0])[1][0]) + # Relation.to_df=18 -> ToDF.column_names=2. + return [name.decode() for name in parse_message(relation[18][0])[2]] + + replaced = T.StructType([T.StructField("a", T.StringType())]) + replaced.fields[0] = T.StructField("z", T.StringType()) + df = spark.createDataFrame(pd.DataFrame({"a": ["x"]}), replaced) + assert column_names(df) == ["a"] + table, schema = _local_rows_relation_of(service, df) + assert table.to_pylist() == [{"a": "x"}] and schema["fields"][0]["name"] == "z" + popped = _struct_changed_in_place("popped") + assert column_names(spark.createDataFrame(pd.DataFrame({"a": [1]}), popped)) == ["a", "b"] + + class _ScaledValueUDT(T.UserDefinedType): """A UserDefinedType of a scale, of the values of `_ScaledValue`.""" diff --git a/python/tests/test_dropin_types_columns_offline.py b/python/tests/test_dropin_types_columns_offline.py index f6f0df80..d144c16b 100644 --- a/python/tests/test_dropin_types_columns_offline.py +++ b/python/tests/test_dropin_types_columns_offline.py @@ -670,6 +670,65 @@ def __str__(self): pyspark_types_to_proto_types(UnparsedDataType(5)) +def test_proto_schema_conversion_raises_pysparks_errors_of_types_that_it_does_not_convert(): + # As official PySpark 4.3.0's `proto_schema_to_pyspark_data_type`, whose last branch raises + # UNSUPPORTED_OPERATION of `f"data type {schema}"`, of the proto text of the type, for a type + # of no kind, such as `pb2.DataType()` or an element type that an array does not set, and of + # a kind that it has no branch of, an unparsed type or a nanosecond-precision timestamp: in the + # order of its conversion, of which a spatial type of an SRID that it does not take raises + # ST_INVALID_SRID_VALUE first. A user-defined type is of its Python class, of no conversion of + # its SQL type. The drop-in raised a PySparkRuntimeError of its own for a type of no kind, + # and gave an UnparsedDataType. + from pyspark.errors import IllegalArgumentException + from pyspark.sql.connect import proto as pb2 + from pyspark.sql.connect.types import proto_schema_to_pyspark_data_type + + def struct(*builds): + dt = pb2.DataType() + dt.struct.SetInParent() + for i, build in enumerate(builds): + field = dt.struct.fields.add() + field.name, field.nullable = f"f{i}", True + if build is not None: + build(field.data_type) + return dt + + def unparsed(dt): + dt.unparsed.data_type_string = 'a "b"' + + def nanos(dt): + dt.timestamp_ntz_nanos.precision = 9 + + def srid(dt): + dt.geometry.srid = 2154 + + array, map_ = pb2.DataType(), pb2.DataType() + array.array.SetInParent() + map_.map.key_type.string.SetInParent() + for dt, text in [ + (pb2.DataType(), ""), + (struct(None), ""), + (struct(lambda dt: dt.long.SetInParent(), lambda dt: None), ""), + (array, ""), + (map_, ""), + (struct(unparsed), 'unparsed {\n data_type_string: "a \\"b\\""\n}\n'), + (struct(nanos, None), "timestamp_ntz_nanos {\n precision: 9\n}\n"), + (struct(None, nanos), ""), + ]: + with pytest.raises(PySparkValueError) as e: + proto_schema_to_pyspark_data_type(dt) + assert e.value.getCondition() == "UNSUPPORTED_OPERATION" + assert e.value.getMessageParameters() == {"operation": f"data type {text}"} + with pytest.raises(IllegalArgumentException) as e: + proto_schema_to_pyspark_data_type(struct(srid, None)) + assert e.value.getCondition() == "ST_INVALID_SRID_VALUE" + for sql_type in (nanos, unparsed, lambda dt: dt.SetInParent()): + udt = pb2.DataType() + udt.udt.type, udt.udt.python_class = "udt", "pyspark.ml.linalg.VectorUDT" + sql_type(udt.udt.sql_type) + assert type(proto_schema_to_pyspark_data_type(udt)).__name__ == "VectorUDT" + + def test_the_abstract_types_have_no_proto_nor_arrow_type_as_pysparks(): # PySpark's `pyspark_types_to_proto_types` has no proto of a type of its class only, of which # it raises UNSUPPORTED_OPERATION, also in a struct, an array or a map. `cast` and `try_cast`