diff --git a/crates/pyspark-rs/src/connect_bridge.rs b/crates/pyspark-rs/src/connect_bridge.rs index 6be449c8..48c61ce2 100644 --- a/crates/pyspark-rs/src/connect_bridge.rs +++ b/crates/pyspark-rs/src/connect_bridge.rs @@ -44,12 +44,134 @@ pub fn _data_type_proto<'py>( /// The type of the bytes of a `spark.connect.DataType`, as the client reads the types of the /// server, with the errors of PySpark's `proto_schema_to_pyspark_data_type` for a type that it -/// does not convert, such as a spatial type of an SRID that PySpark does not take. +/// does not convert (see [`unconverted`]), such as `pb2.DataType()` of no kind or a spatial type +/// of an SRID that PySpark does not take. PySpark makes a user-defined type of its Python class, +/// of no conversion of its SQL type, which is read only when the core can read it. #[pyfunction] pub fn _data_type_of_proto(py: Python<'_>, data_type: &[u8]) -> PyResult> { - let data_type = proto::DataType::decode(data_type) + let mut data_type = proto::DataType::decode(data_type) .map_err(|error| PyValueError::new_err(error.to_string()))?; + match unconverted(Some(&data_type)) { + Some(Unconverted::Kind(node)) => return Err(unsupported_data_type(py, &node)), + Some(Unconverted::Srid(srid)) => return Err(crate::types::invalid_schema_srid(py, srid)), + None => {} + } + drop_unread_sql_types(&mut data_type); let data_type = DataType::from_proto(&data_type).to_pyerr()?; - crate::dataframe::check_schema_conversion(py, Some(&data_type))?; crate::types::schema_to_py(py, &data_type) } + +/// Drops the SQL type of each user-defined type in `data_type` that holds a type of no kind, +/// which the core does not read and PySpark does not convert. +fn drop_unread_sql_types(data_type: &mut proto::DataType) { + use proto::data_type::Kind; + match &mut data_type.kind { + Some(Kind::Udt(udt)) => { + if udt.sql_type.as_deref().is_some_and(has_no_kind) { + udt.sql_type = None; + } + } + Some(Kind::Array(array)) => array + .element_type + .iter_mut() + .for_each(|t| drop_unread_sql_types(t)), + Some(Kind::Struct(st)) => st + .fields + .iter_mut() + .filter_map(|field| field.data_type.as_mut()) + .for_each(drop_unread_sql_types), + Some(Kind::Map(map)) => { + map.key_type + .iter_mut() + .for_each(|t| drop_unread_sql_types(t)); + map.value_type + .iter_mut() + .for_each(|t| drop_unread_sql_types(t)); + } + _ => {} + } +} + +/// Whether `data_type` is, or holds, a type of no kind, of which the core makes no type. +fn has_no_kind(data_type: &proto::DataType) -> bool { + use proto::data_type::Kind; + match &data_type.kind { + None => true, + Some(Kind::Array(array)) => array.element_type.as_deref().is_none_or(has_no_kind), + Some(Kind::Struct(st)) => st + .fields + .iter() + .any(|field| field.data_type.as_ref().is_none_or(has_no_kind)), + Some(Kind::Map(map)) => { + map.key_type.as_deref().is_none_or(has_no_kind) + || map.value_type.as_deref().is_none_or(has_no_kind) + } + Some(Kind::Udt(udt)) => udt.sql_type.as_deref().is_some_and(has_no_kind), + Some(_) => false, + } +} + +/// A type that PySpark's `proto_schema_to_pyspark_data_type` does not convert. +enum Unconverted { + /// A type of no kind, such as `pb2.DataType()` or the element type that an array does not + /// set, which PySpark reads as the empty message of the field, or of a kind that it has no + /// branch of, an unparsed type or a nanosecond-precision timestamp: the bytes of the type, of + /// whose proto text PySpark's last branch raises `UNSUPPORTED_OPERATION`. + Kind(Vec), + /// A spatial type of an SRID that PySpark's `GeometryType` or `GeographyType` does not take, + /// which raises `ST_INVALID_SRID_VALUE`. + Srid(i32), +} + +/// The first type of `data_type` that PySpark's `proto_schema_to_pyspark_data_type` does not +/// convert, in the order in which it converts them: the element of an array, the types of the +/// fields of a struct, and the key and then the value of a map. PySpark makes a user-defined type +/// of its Python class, of no conversion of its SQL type. +fn unconverted(data_type: Option<&proto::DataType>) -> Option { + use proto::data_type::Kind; + let Some(data_type) = data_type else { + return Some(Unconverted::Kind(Vec::new())); + }; + let spatial = |srid: i32, geographic: bool| { + (!crate::types::pyspark_converts_srid(srid, geographic)).then_some(Unconverted::Srid(srid)) + }; + match &data_type.kind { + None + | Some(Kind::Unparsed(_)) + | Some(Kind::TimestampNtzNanos(_)) + | Some(Kind::TimestampLtzNanos(_)) => Some(Unconverted::Kind(data_type.encode_to_vec())), + Some(Kind::Geometry(geometry)) => spatial(geometry.srid, false), + Some(Kind::Geography(geography)) => spatial(geography.srid, true), + Some(Kind::Array(array)) => unconverted(array.element_type.as_deref()), + Some(Kind::Struct(st)) => st + .fields + .iter() + .find_map(|field| unconverted(field.data_type.as_ref())), + Some(Kind::Map(map)) => { + unconverted(map.key_type.as_deref()).or_else(|| unconverted(map.value_type.as_deref())) + } + Some(_) => None, + } +} + +/// PySpark's `UNSUPPORTED_OPERATION` of `data type {schema}`, a `PySparkValueError`, of the proto +/// text of the type of the bytes `node`, as `str()` of its message gives it: empty of a type of no +/// kind. +fn unsupported_data_type(py: Python<'_>, node: &[u8]) -> PyErr { + let text = || -> PyResult { + let message = py + .import("pyspark.sql.connect.proto")? + .getattr("DataType")? + .call_method1("FromString", (PyBytes::new(py, node),))?; + Ok(message.str()?.to_cow()?.into_owned()) + }; + match text() { + Ok(text) => crate::errors::raise_pyspark( + py, + "PySparkValueError", + "UNSUPPORTED_OPERATION", + &[("operation", &format!("data type {text}"))], + ), + Err(err) => err, + } +} diff --git a/crates/pyspark-rs/src/errors.rs b/crates/pyspark-rs/src/errors.rs index 24ee8fda..310306b8 100644 --- a/crates/pyspark-rs/src/errors.rs +++ b/crates/pyspark-rs/src/errors.rs @@ -28,6 +28,24 @@ pub(crate) fn raise_pyspark( build().unwrap_or_else(|e| e) } +/// Raise a `pyspark.errors` exception (`class_name`) of the `error_class` condition alone, as +/// PySpark raises some of them without `messageParameters`, such as `MALFORMED_VARIANT` of the +/// conversion of local rows: its `getMessageParameters()` is None, where that of +/// [`raise_pyspark`] of no parameters is `{}`. +pub(crate) fn raise_pyspark_condition( + py: Python<'_>, + class_name: &str, + error_class: &str, +) -> PyErr { + let build = || -> PyResult { + let cls = py.import("pyspark.errors")?.getattr(class_name)?; + let kwargs = PyDict::new(py); + kwargs.set_item("errorClass", error_class)?; + Ok(PyErr::from_value(cls.call((), Some(&kwargs))?)) + }; + build().unwrap_or_else(|e| e) +} + /// As [`raise_pyspark`], of parameters that are Python objects, as PySpark formats them into its /// message: a str that UTF-8 cannot encode, such as one of a lone surrogate, shows as it is. pub(crate) fn raise_pyspark_of( diff --git a/crates/pyspark-rs/src/infer.rs b/crates/pyspark-rs/src/infer.rs index f3b4ee10..e648f2bb 100644 --- a/crates/pyspark-rs/src/infer.rs +++ b/crates/pyspark-rs/src/infer.rs @@ -7,7 +7,8 @@ use std::collections::{HashMap, HashSet}; use pyo3::exceptions::{ - PyAssertionError, PyNotImplementedError, PyOverflowError, PyTypeError, PyValueError, + PyAssertionError, PyIndexError, PyNotImplementedError, PyOverflowError, PyTypeError, + PyValueError, }; use pyo3::intern; use pyo3::prelude::*; @@ -1529,6 +1530,10 @@ pub(crate) struct TypeObjects<'py> { /// The objects of the fields of a struct, in their order; of the element of an array; of the /// key and the value of a map; or of the `sqlType()` of a user-defined type. children: Vec>, + /// The names by which PySpark's converter of a struct below the top of the schema takes its + /// values apart, `fieldNames()` of its type object, when they are not those of its fields as + /// they are (see [`cached_names`]). + names: Option>>, } impl<'py> TypeObjects<'py> { @@ -1536,41 +1541,55 @@ impl<'py> TypeObjects<'py> { /// makes its converters of them: of the `dataType` of each of its `fields`, its `elementType`, /// its `keyType` and `valueType`, and of the `sqlType()` of a user-defined type. pub(crate) fn of(obj: &Bound<'py, PyAny>, dt: &DataType) -> PyResult { + Self::of_place(obj, dt, false) + } + + /// As [`Self::of`], `nested` below the top of the schema, where PySpark's converter of a + /// struct reads its `fieldNames()`, which the top level takes of the rows (see + /// [`cached_names`]). + fn of_place(obj: &Bound<'py, PyAny>, dt: &DataType, nested: bool) -> PyResult { let py = obj.py(); + let mut names = None; let children = match dt { DataType::Udt { sql_type: Some(sql_type), .. - } => vec![Self::of( + } => vec![Self::of_place( &obj.call_method0(intern!(py, "sqlType"))?, sql_type, + true, )?], DataType::Struct { fields } => { + if nested { + names = cached_names(obj, fields)?; + } let objects = obj.getattr(intern!(py, "fields"))?; let mut children = Vec::with_capacity(fields.len()); for (field, object) in fields.iter().zip(objects.try_iter()?) { let data_type = object?.getattr(intern!(py, "dataType"))?; - children.push(Self::of(&data_type, &field.data_type)?); + children.push(Self::of_place(&data_type, &field.data_type, true)?); } children } - DataType::Array { element_type, .. } => vec![Self::of( + DataType::Array { element_type, .. } => vec![Self::of_place( &obj.getattr(intern!(py, "elementType"))?, element_type, + true, )?], DataType::Map { key_type, value_type, .. } => vec![ - Self::of(&obj.getattr(intern!(py, "keyType"))?, key_type)?, - Self::of(&obj.getattr(intern!(py, "valueType"))?, value_type)?, + Self::of_place(&obj.getattr(intern!(py, "keyType"))?, key_type, true)?, + Self::of_place(&obj.getattr(intern!(py, "valueType"))?, value_type, true)?, ], _ => Vec::new(), }; Ok(Self { object: Some(obj.clone()), children, + names, }) } @@ -1580,6 +1599,7 @@ impl<'py> TypeObjects<'py> { Self { object: Some(udt), children: Vec::new(), + names: None, } } @@ -1589,6 +1609,7 @@ impl<'py> TypeObjects<'py> { Self { object: None, children: fields, + names: None, } } @@ -1609,6 +1630,7 @@ impl<'py> TypeObjects<'py> { Self { object: None, children, + names: None, } } @@ -1626,7 +1648,7 @@ impl<'py> TypeObjects<'py> { } => { if let Some(udt) = &self.object { let sql = udt.call_method0(intern!(udt.py(), "sqlType"))?; - self.children = vec![Self::of(&sql, sql_type)?]; + self.children = vec![Self::of_place(&sql, sql_type, true)?]; } } DataType::Struct { fields } => { @@ -1658,6 +1680,12 @@ impl<'py> TypeObjects<'py> { self.object.as_ref() } + /// The names by which a value of the struct of this place is taken apart, when they are not + /// those of its fields as they are (see [`cached_names`]). + fn names(&self) -> Option<&[Bound<'py, PyAny>]> { + self.names.as_deref() + } + /// The objects of the field `index` of a struct. pub(crate) fn field(&self, index: usize) -> Option<&Self> { self.children.get(index) @@ -1708,6 +1736,7 @@ impl<'py> FieldObjects<'py> { TypeObjects { object: None, children: self.0, + names: None, } } } @@ -1879,6 +1908,8 @@ pub(crate) fn convert<'py>( // `metadata` attributes of a VariantVal, and the `srid` and `wkb` of a Geometry or a // Geography, which PyArrow converts as the fields of the structs of their Arrow types. A // spatial type of the schema of a Python subclass converts with its own (`spatial_value`). + // Another value raises the `MALFORMED_*` condition of no message parameters, as PySpark's + // `PySparkValueError(errorClass=...)` of its converter. DataType::Variant => { if obj.is_instance(variant_val_class(py)?)? { let value = obj.getattr(intern!(py, "value"))?; @@ -1890,11 +1921,10 @@ pub(crate) fn convert<'py>( }) })) } else { - Err(raise_pyspark( + Err(crate::errors::raise_pyspark_condition( py, "PySparkValueError", "MALFORMED_VARIANT", - &[], )) } } @@ -1903,11 +1933,10 @@ pub(crate) fn convert<'py>( let native = py.get_type::(); spatial_value(obj, objects.and_then(TypeObjects::object), &native, late) } else { - Err(raise_pyspark( + Err(crate::errors::raise_pyspark_condition( py, "PySparkValueError", "MALFORMED_GEOMETRY", - &[], )) } } @@ -1916,11 +1945,10 @@ pub(crate) fn convert<'py>( let native = py.get_type::(); spatial_value(obj, objects.and_then(TypeObjects::object), &native, late) } else { - Err(raise_pyspark( + Err(crate::errors::raise_pyspark_condition( py, "PySparkValueError", "MALFORMED_GEOGRAPHY", - &[], )) } } @@ -2383,39 +2411,119 @@ fn convert_integral(obj: &Bound<'_, PyAny>, dt: &DataType) -> PyResult { }) } -/// The values of `fields` from a mapping, by their names, as `mapping.get(name)`. +/// The values of a struct value from a mapping, as `mapping.get(name)` of each of `names`, or of +/// the names of `fields` without them. fn values_by_name<'py>( mapping: &Bound<'py, PyAny>, fields: &[StructField], + names: Option<&[Bound<'py, PyAny>]>, ) -> PyResult>> { let py = mapping.py(); + let Some(names) = names else { + let names = fields + .iter() + .map(|f| PyString::new(py, &f.name).into_any()) + .collect::>(); + return values_by_name(mapping, fields, Some(&names)); + }; if let Ok(dict) = mapping.cast::() { - return fields + return names .iter() - .map(|f| { + .map(|name| { Ok(dict - .get_item(&f.name)? + .get_item(name)? .unwrap_or_else(|| py.None().into_bound(py))) }) .collect(); } - fields + names .iter() - .map(|f| mapping.call_method1(intern!(py, "get"), (&f.name,))) + .map(|name| mapping.call_method1(intern!(py, "get"), (name,))) .collect() } -/// The values of a tuple or a Row, which must be as many as `fields`. +/// The values of a tuple or a Row, which must be as many as the names that the struct value is +/// taken apart by, `len`. fn values_by_position<'py>( py: Python<'py>, values: Vec>, - fields: &[StructField], + len: usize, ) -> PyResult>> { - if values.len() == fields.len() { + if values.len() == len { Ok(values) } else { - Err(axis_length_mismatch(py, fields.len(), values.len())) + Err(axis_length_mismatch(py, len, values.len())) + } +} + +/// The names by which PySpark's converters take a value of the struct `obj` of the given schema +/// apart, `obj.fieldNames()`, which `to_row` reads of the schema and the converter of a struct of +/// its type object, when they are not the names of `fields`, the fields of `obj` as they are: +/// PySpark's `StructType` caches its `names` as it is made (and as `add` adds a field), so of a +/// `fields` list changed in place, such as of a `StructField` appended, they are the names before +/// the change, while the converters are of the fields as they are. None of the type of another +/// class, or of names that are those of the fields. +pub(crate) fn cached_names<'py>( + obj: &Bound<'py, PyAny>, + fields: &[StructField], +) -> PyResult>>> { + if !obj.is_instance_of::() { + return Ok(None); + } + let names = obj + .call_method0(intern!(obj.py(), "fieldNames"))? + .try_iter()? + .collect::>>()?; + let same = names.len() == fields.len() + && names.iter().zip(fields).all(|(name, field)| { + name.cast::() + .is_ok_and(|name| name.to_cow().is_ok_and(|name| name == field.name)) + }); + Ok((!same).then_some(names)) +} + +/// The `IndexError` that PySpark's conversion of local rows raises once their values are +/// converted, as it makes the Arrow schema of the fields of the schema, whose +/// `_deduplicate_field_names` names the fields of each struct below the top level, in an array +/// or a map too, by the cached names of its type object (see [`cached_names`]): of fewer names +/// than fields, of a `fields` list that a field was added to in place. Its converters stop at a +/// user-defined type, of whose SQL type the schema has no struct. +pub(crate) fn dedup_names_error( + objects: Option<&TypeObjects<'_>>, + fields: &[StructField], +) -> Option { + fn short<'py>(objects: Option<&TypeObjects<'py>>, dt: &DataType) -> bool { + let Some(objects) = objects else { + return false; + }; + match dt { + DataType::Struct { fields } => { + objects + .names() + .is_some_and(|names| names.len() < fields.len()) + || fields + .iter() + .enumerate() + .any(|(i, field)| short(objects.field(i), &field.data_type)) + } + DataType::Array { element_type, .. } => short(objects.element(), element_type), + DataType::Map { + key_type, + value_type, + .. + } => { + let (key, value) = objects.key_and_value(); + short(key, key_type) || short(value, value_type) + } + _ => false, + } } + let objects = objects?; + fields + .iter() + .enumerate() + .any(|(i, field)| short(objects.field(i), &field.data_type)) + .then(|| PyIndexError::new_err("list index out of range")) } /// A struct value, as PySpark converts one: a tuple or a Row by position, a dict by name, @@ -2429,15 +2537,19 @@ fn convert_struct<'py>( budget: &ConversionBudget, ) -> PyResult { let py = obj.py(); + // Of the cached names of a `fields` list changed in place, if they differ (see + // `cached_names`). + let names = objects.and_then(TypeObjects::names); + let len = names.map_or(fields.len(), <[_]>::len); let values = if let Ok(tuple) = obj.cast::() { if !reads_as_stored(tuple, true)? { - return struct_by_index(obj, fields, objects, late, budget); + return struct_by_index(obj, fields, len, objects, late, budget); } - values_by_position(py, tuple.iter().collect(), fields)? + values_by_position(py, tuple.iter().collect(), len)? } else if obj.is_instance_of::() { - values_by_name(obj, fields)? + values_by_name(obj, fields, names)? } else if let Ok(attributes) = obj.getattr(intern!(py, "__dict__")) { - values_by_name(&attributes, fields)? + values_by_name(&attributes, fields, names)? } else { // PySpark asserts that another value has a `__dict__`. return Err(PyAssertionError::new_err(format!( @@ -2457,36 +2569,54 @@ fn struct_of<'py>( budget: &ConversionBudget, ) -> PyResult { let names: std::sync::Arc<[String]> = fields.iter().map(|f| f.name.clone()).collect(); - let values = values - .iter() - .zip(fields) - .enumerate() - .map(|(index, (value, f))| { - let field_objects = objects.and_then(|objects| objects.field(index)); - convert(value, &f.data_type, field_objects, f.nullable, late, budget) - }) - .collect::>>()?; - Ok(Value::Struct(StructValue::new(names, values))) + // Each value is converted with the field of its position, as PySpark's `field_convs[i]` of + // the values of the cached names of a `fields` list changed in place (see `cached_names`): + // past the fields, its `IndexError`, once those before are converted; short of them, the + // fields after are null, and the Arrow schema of the conversion raises once every value is + // converted (`dedup_names_error`). + let mut converted = Vec::with_capacity(fields.len()); + for (index, value) in values.iter().enumerate() { + let Some(f) = fields.get(index) else { + return Err(PyIndexError::new_err("list index out of range")); + }; + let field_objects = objects.and_then(|objects| objects.field(index)); + converted.push(convert( + value, + &f.data_type, + field_objects, + f.nullable, + late, + budget, + )?); + } + converted.resize_with(fields.len(), || Value::Null); + Ok(Value::Struct(StructValue::new(names, converted))) } /// A struct value of a tuple of a subclass of its own `__len__` or `__getitem__`, as PySpark's -/// converter reads one: `len(value)` must be the number of fields, and `value[i]` of each field -/// is read as the value before it is converted. +/// converter reads one: `len(value)` must be `len`, the number of the names that it is taken +/// apart by (see `cached_names`), and `value[i]` of each is read as the value before it is +/// converted with the field of its position, of which PySpark reads the converter first: past +/// the fields, its `IndexError`, and short of them, the fields after are null. fn struct_by_index<'py>( obj: &Bound<'py, PyAny>, fields: &[StructField], + len: usize, objects: Option<&TypeObjects<'py>>, late: &mut Late, budget: &ConversionBudget, ) -> PyResult { let py = obj.py(); - if obj.len()? != fields.len() { + if obj.len()? != len { // PySpark's message reads `len(value)` again. - return Err(axis_length_mismatch(py, fields.len(), obj.len()?)); + return Err(axis_length_mismatch(py, len, obj.len()?)); } let names: std::sync::Arc<[String]> = fields.iter().map(|f| f.name.clone()).collect(); let mut values = Vec::with_capacity(fields.len()); - for (index, f) in fields.iter().enumerate() { + for index in 0..len { + let Some(f) = fields.get(index) else { + return Err(PyIndexError::new_err("list index out of range")); + }; let field_objects = objects.and_then(|objects| objects.field(index)); let value = obj.get_item(index)?; values.push(convert( @@ -2498,6 +2628,7 @@ fn struct_by_index<'py>( budget, )?); } + values.resize_with(fields.len(), || Value::Null); Ok(Value::Struct(StructValue::new(names, values))) } @@ -2571,41 +2702,43 @@ pub(crate) fn variant_row(py: Python<'_>) -> PyErr { /// by `len(item)`, which must be the number of fields, and `tuple(item)`, of the `__len__` and /// `__iter__` of a subclass too, which may give fewer or more values than the fields: PySpark's /// `row[i]` of a missing one raises `IndexError` as its column converts, and those after the -/// fields are not read (`create_from_local`). +/// fields are not read (`create_from_local`). With `names`, the cached names of the schema that +/// differ from its fields (see [`cached_names`]), the row is taken apart by them, as `to_row` +/// reads `schema.fieldNames()`, and its values are converted with the fields of their positions. pub(crate) fn row_values<'py>( item: &Bound<'py, PyAny>, fields: &[StructField], + names: Option<&[Bound<'py, PyAny>]>, ) -> PyResult>> { let py = item.py(); + let len = names.map_or(fields.len(), <[_]>::len); if item.is_none() { - return Ok(vec![py.None().into_bound(py); fields.len()]); + return Ok(vec![py.None().into_bound(py); len]); } if let Ok(tuple) = item.cast::() { if !reads_as_stored(tuple, false)? { - return sequence_values(item, fields); + return sequence_values(item, len); } - return values_by_position(py, tuple.iter().collect(), fields); + return values_by_position(py, tuple.iter().collect(), len); } if item.is_instance_of::() { - return values_by_name(item, fields); + return values_by_name(item, fields, names); } if item.is_instance(variant_val_class(py)?)? { return Err(variant_row(py)); } if item.hasattr(intern!(py, "__dict__"))? { - return values_by_name(&item.getattr(intern!(py, "__dict__"))?, fields); + return values_by_name(&item.getattr(intern!(py, "__dict__"))?, fields, names); } - sequence_values(item, fields) + sequence_values(item, len) } -/// The values of `tuple(item)` of a row whose `len(item)` must be the number of `fields`. -fn sequence_values<'py>( - item: &Bound<'py, PyAny>, - fields: &[StructField], -) -> PyResult>> { - if item.len()? != fields.len() { +/// The values of `tuple(item)` of a row whose `len(item)` must be `len`, the number of the names +/// that the row is taken apart by. +fn sequence_values<'py>(item: &Bound<'py, PyAny>, len: usize) -> PyResult>> { + if item.len()? != len { // PySpark's message reads `len(item)` again. - return Err(axis_length_mismatch(item.py(), fields.len(), item.len()?)); + return Err(axis_length_mismatch(item.py(), len, item.len()?)); } Ok(sequence_tuple(item)?.iter().collect()) } diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs index e643f2bb..f7d7a7cb 100644 --- a/crates/pyspark-rs/src/session.rs +++ b/crates/pyspark-rs/src/session.rs @@ -1670,6 +1670,7 @@ impl PySparkSession { use crate::infer; let atomic = matches!(spec, Spec::Atomic(_)); + let given_struct = matches!(spec, Spec::Struct(_)); let (given, mut cols) = match spec { Spec::None => (None, None), Spec::Names(names) => (None, Some(names)), @@ -1773,17 +1774,24 @@ impl PySparkSession { } }; - // As PySpark: every row is taken apart first, then each column is converted. + // As PySpark: every row is taken apart first, then each column is converted. The rows are + // taken apart by `schema.fieldNames()` of the given `StructType`, which of a `fields` list + // changed in place are the names before the change (`infer::cached_names`). + let names = match (schema, given_struct) { + (Some(schema), true) => infer::cached_names(schema, &fields)?, + _ => None, + }; let mut parts: Vec>> = Vec::with_capacity(items.len()); for item in &items { budget.step(py)?; parts.push(if single { - if fields.len() != 1 { - return Err(infer::axis_length_mismatch(py, fields.len(), 1)); + let len = names.as_ref().map_or(fields.len(), Vec::len); + if len != 1 { + return Err(infer::axis_length_mismatch(py, len, 1)); } vec![item.clone()] } else { - infer::row_values(item, &fields)? + infer::row_values(item, &fields, names.as_deref())? }); } // As PySpark's converters, which assert the nullable flags of the schema when they are @@ -1851,6 +1859,11 @@ impl PySparkSession { } // The rows of a struct of no fields have no values. values.resize_with(row_count, Vec::new); + // As PySpark's Arrow schema of the fields, whose names of the structs below the top level + // are their cached names, of which fewer than the fields raise (`infer::dedup_names_error`). + if let Some(err) = infer::dedup_names_error(objects.as_ref(), &fields) { + return Err(err); + } // Then, as PySpark, the schema converts to Arrow, a field at a time and depth first, // which rejects a null type that is not nullable, and has no char, varchar, year-month // or calendar interval type, nor a type of its class only such as `AtomicType()` or a @@ -1975,16 +1988,21 @@ impl PySparkSession { .create_dataframe_from_arrow_with_options(batches, Some(spark_schema), &options) .map_err(Box::new) })?; - // As the other pandas paths, the explicit names are applied with toDF, when there are any. - Ok(Some(PyDataFrame::new(if fields.is_empty() { + // As the other pandas paths, the explicit names are applied with toDF, when there are any: + // PySpark's `toDF(*schema.fieldNames())`, which of a `fields` list changed in place are the + // names before the change, of which the server raises for another number of columns + // (`infer::cached_names`). + let names = match crate::infer::cached_names(schema, fields)? { + Some(names) => names + .iter() + .map(|name| Ok(name.str()?.to_cow()?.into_owned())) + .collect::>>()?, + None => fields.iter().map(|field| field.name.clone()).collect(), + }; + Ok(Some(PyDataFrame::new(if names.is_empty() { df } else { - df.to_df( - fields - .iter() - .map(|field| field.name.as_str()) - .collect::>(), - ) + df.to_df(names.iter().map(String::as_str).collect::>()) }))) } @@ -3108,10 +3126,16 @@ impl PySparkSession { // Preserve schema parsing, ignored-argument warnings, and pandas/Arrow routing. // Only a native StructType opts into this explicit integer optimization; // DDL, inferred schemas and unsupported values keep schema-driven conversion. - if schema.is_some_and(|s| s.extract::>().is_ok()) { + if let Some(struct_type) = + schema.filter(|s| s.extract::>().is_ok()) + { if let Spec::Struct(fields) = &spec { if let Ok(data) = data.cast::() { - if let Some(columns) = exact_long_tuple_columns(data, fields)? { + // Not of a `fields` list changed in place, whose rows are taken apart by the + // names before the change (`infer::cached_names`). + if let Some(columns) = exact_long_tuple_columns(data, fields)?.filter(|_| { + matches!(crate::infer::cached_names(struct_type, fields), Ok(None)) + }) { let schema = DataType::Struct { fields: fields.clone(), }; diff --git a/dev/parity/known-differences.md b/dev/parity/known-differences.md index a3429af3..ab6a95d1 100644 --- a/dev/parity/known-differences.md +++ b/dev/parity/known-differences.md @@ -120,6 +120,20 @@ difference is intentional or tracked for a fix. `timestamp_seconds(-62135596800)`, 0001-01-01 00:00:00 UTC, with the session time zone `America/Los_Angeles` on a host in UTC, this client returns `datetime(1, 1, 1, 0, 0)`. Both raise the `OverflowError` of a wall clock out of the range in the local time zone. +- **A `StructType` whose `fields` hold an element that is not a `StructField`** (inherent; the + uses of the native type lower its fields into the core's struct type, which holds fields only). + PySpark's `StructType` methods read the attributes of each element of `fields` as they go, so of + a list that a str or another object was appended to in place, each use raises the + `AttributeError` of the first attribute that it reads (`simpleString`, `jsonValue`, `name`, + `dataType`, `needConversion`), or uses an object that has them, and its pickles and copies keep + the element; `createDataFrame` of local rows takes them apart by the cached names first, so a + row of another number of values raises `AXIS_LENGTH_MISMATCH` before that. This client lowers the + list into the core's struct type at each use that needs the type (its JSON, simple string and + tree string, a lookup by name, `add`, its pickles and copies, the plans of `to`, `cast`, + `from_json` and a reader's schema, and `createDataFrame`), and raises + `TypeError: StructType fields must be StructField objects` there, and an assignment of `fields` + of such an element raises it at once. Of a list of `StructField`s changed in place, the cached + `names` and their uses are PySpark's (see `docs/compatibility.md`). - **The `__repr__` of a Column of a str that UTF-8 cannot encode** (intentional; the expressions of the core hold UTF-8). Such a str, of a name, a SQL string or a literal, fails the plans of the Column when they are built, as in PySpark, and its lone surrogates show escaped in the `__repr__` diff --git a/docs/compatibility.md b/docs/compatibility.md index 43820e3c..c50e32b9 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -2206,6 +2206,17 @@ PySpark's shares the list of its fields. The class's own types compare without t a user set, where PySpark's comparison of the `__dict__`s holds them: a `DecimalType(7, 3)` of an attribute equals `DecimalType(7, 3)`, where a type of a Python subclass compares them (see below). +A `StructType` caches its `names` as it is made and as `add` adds a field, as PySpark's, so an +in-place change of its `fields` list, such as a `StructField` appended, leaves `names` and +`fieldNames()` as they were. `createDataFrame` takes local rows, and the struct values below the +top level, apart by those names, as PySpark's conversion reads `fieldNames()`, and converts their +values by position with the fields as they are: a row of another number of values than the names +raises `AXIS_LENGTH_MISMATCH`, a dict is read by them, and a field past them has no value +(`IndexError`); below the top level, a value past the fields and fewer names than fields raise +`IndexError` too. A pandas DataFrame is named with `toDF(*schema.fieldNames())`, of which the server +raises for another number of names than columns. An element that is not a `StructField` is not +taken (see `dev/parity/known-differences.md`). + The parameters of a type are its attributes, as PySpark's: `t.scale = 3` makes a `DecimalType(10, 2)` a `DecimalType(10, 3)`, as its repr, JSON, `==` and hash show, also in the struct, array or map that holds it. A drop-in type holds only the values that its constructor diff --git a/python/pyspark/sql/_pandas_input.py b/python/pyspark/sql/_pandas_input.py index dc419c54..61e0a126 100644 --- a/python/pyspark/sql/_pandas_input.py +++ b/python/pyspark/sql/_pandas_input.py @@ -1513,7 +1513,9 @@ def prepare_pandas_table(frame, schema, timezone, safecheck, prefers_large_types ) if len(schema.fields) == 0: return table - return table.rename_columns(schema.names).cast(arrow_schema) + # The names of the fields, of a `fields` list changed in place the cached `names` of as many, + # as PySpark's names of `_deduplicate_field_names`, which raises of fewer. + return table.rename_columns(schema.names[: len(schema.fields)]).cast(arrow_schema) def arrow_decimal_column(values, dtype, name): diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py index c98f150a..bb5fae2e 100644 --- a/python/tests/test_dropin_e2e.py +++ b/python/tests/test_dropin_e2e.py @@ -5412,6 +5412,39 @@ def deserialize(self, datum): return datum // self.scale +def test_create_dataframe_of_a_struct_changed_in_place_reads_its_cached_names_as_pyspark(session): + # As official PySpark 4.3.0, whose StructType caches its `names` as it is made, which an + # in-place change of its `fields` list leaves as they were: createDataFrame takes local rows, + # and the struct values below the top level, apart by those names (`fieldNames()`), and + # converts their values by position with the fields as they are; a pandas DataFrame is + # named with `toDF(*schema.fieldNames())`. + pd = pytest.importorskip("pandas") + appended = StructType([StructField("a", LongType())]) + appended.fields.append(StructField("b", StringType())) + popped = StructType([StructField("a", LongType()), StructField("b", StringType())]) + popped.fields.pop() + replaced = StructType([StructField("a", LongType())]) + replaced.fields[0] = StructField("z", StringType()) + for data, schema in (([(1, "q")], appended), ([(1,)], popped), + ([((1, "q"),)], StructType([StructField("s", appended)]))): + with pytest.raises(PySparkValueError, match="AXIS_LENGTH_MISMATCH"): + session.createDataFrame(data, schema) + with pytest.raises(IndexError, match="tuple index out of range"): + session.createDataFrame([(1,)], appended) + for data, inner in (([((1,),)], appended), ([(None,)], appended), ([({"a": 1},)], popped)): + with pytest.raises(IndexError, match="list index out of range"): + session.createDataFrame(data, StructType([StructField("s", inner)])) + assert session.createDataFrame([(1, "q")], popped).collect() == [Row(a=1)] + assert session.createDataFrame([{"a": 1}], replaced).collect() == [Row(z="1")] + nested = StructType([StructField("s", replaced)]) + assert session.createDataFrame([({"a": 1},)], nested).collect() == [Row(s=Row(z="1"))] + string = StructType([StructField("a", StringType())]) + string.fields[0] = StructField("z", StringType()) + assert session.createDataFrame(pd.DataFrame({"a": ["x"]}), string).collect() == [Row(a="x")] + with pytest.raises(AnalysisException, match="ASSIGNMENT_ARITY_MISMATCH"): + session.createDataFrame(pd.DataFrame({"a": [1]}), popped).collect() + + def test_local_rows_are_serialized_by_the_udt_objects_of_the_schema_as_pyspark(session): # As official PySpark 4.3.0, which serializes each value of local rows with the UDT object # that the schema holds: two UDTs of one class and other parameters keep their own, in a diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 11a97c69..60b08d8c 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -6848,7 +6848,9 @@ def test_variant_spatial_and_user_defined_values_are_sent_as_pyspark(mock_connec # As PySpark's LocalDataToArrowConversion: a VariantVal is the struct of its value and # metadata, a Geometry or a Geography that of its srid and WKB, whose metadata tags the # struct as of its type, and a value of a user-defined type is serialized by the UDT and - # sent as its SQL type; a value of another type for these is malformed. + # sent as its SQL type; a value of another type for these is malformed, of the condition + # alone, whose message parameters are None, as PySpark's `PySparkValueError(errorClass=...)` + # of its converters, also nested. import pyarrow as pa from pyspark.errors import PySparkValueError from pyspark.testing.objects import ExamplePoint @@ -6889,12 +6891,15 @@ def local_relation(df): "_2": {"srid": 4326, "wkb": wkb}, "_3": {"srid": 4326, "wkb": wkb}, "_4": [1.0, 2.0]}, {"_1": None, "_2": None, "_3": None, "_4": None}, ] - for ddl, condition in (("v variant", "MALFORMED_VARIANT"), - ("g geometry(4326)", "MALFORMED_GEOMETRY"), - ("g geography(4326)", "MALFORMED_GEOGRAPHY")): + for ddl, value, condition in (("v variant", "x", "MALFORMED_VARIANT"), + ("g geometry(4326)", "x", "MALFORMED_GEOMETRY"), + ("g geography(4326)", 1, "MALFORMED_GEOGRAPHY"), + ("v array", ["x"], "MALFORMED_VARIANT"), + ("s struct", ("x",), "MALFORMED_GEOMETRY")): with pytest.raises(PySparkValueError) as error: - spark.createDataFrame([("x",)], ddl) + spark.createDataFrame([(value,)], ddl) assert error.value.getCondition() == condition + assert error.value.getMessageParameters() is None # The attributes of PySpark's spatial values. geometry = T.Geometry.fromWKB(wkb, 4326) assert (geometry.srid, geometry.wkb) == (4326, wkb) @@ -23135,6 +23140,98 @@ def test_local_rows_are_serialized_by_the_udt_objects_of_the_schema_as_pysparks( assert table.to_pylist() == [{"a": 6, "b": 40}] +def _struct_changed_in_place(change): + # A StructType whose `fields` list was changed in place, whose cached `names` stay as they + # were made: a field appended, the last one popped, or the first one replaced by another. + if change == "appended": + st = T.StructType([T.StructField("a", T.LongType())]) + st.fields.append(T.StructField("b", T.StringType())) + elif change == "popped": + st = T.StructType([T.StructField("a", T.LongType()), T.StructField("b", T.StringType())]) + st.fields.pop() + else: + st = T.StructType([T.StructField("a", T.LongType())]) + st.fields[0] = T.StructField("z", T.StringType()) + return st + + +def test_local_rows_of_a_struct_changed_in_place_are_taken_apart_by_its_cached_names(mock_connect): + # As PySpark's LocalDataToArrowConversion, whose `to_row`, and the converter of a struct + # below the top level, take each value apart by `fieldNames()` of the StructType, its cached + # `names`, which an in-place change of its `fields` leaves as they were, and convert the + # values by position with the fields as they are: the arity of a row is the number of the + # names, a dict is read by them, and a field past them has no value (`IndexError` of + # `row[i]`). Below the top level, a value past the fields has no converter, and the Arrow + # schema of fewer names than fields fails (`_deduplicate_field_names`), both `IndexError`. + # The integer fast path takes them so too. As official PySpark 4.3.0 (see + # test_dropin_e2e.py); the drop-in took them apart by the names of the fields as they are. + from pyspark.errors import PySparkValueError + + spark, service = mock_connect + appended, popped, replaced = ( + _struct_changed_in_place(change) for change in ("appended", "popped", "replaced")) + for data, schema in ( + ([(1, "q")], appended), ([Row(a=1, b="q")], appended), ([[1, "q"]], appended), + ([(1,)], popped), ([(1,), (2,)], popped), ([[1]], popped), ([None], popped), + ([1], popped), + ): + with pytest.raises(PySparkValueError) as error: + spark.createDataFrame(data, schema) + assert error.value.getCondition() == "AXIS_LENGTH_MISMATCH" + for data in ([(1,)], [{"a": 1, "b": "q"}], [None], [1]): + with pytest.raises(IndexError, match="tuple index out of range"): + spark.createDataFrame(data, appended) + table, _ = _local_rows_relation_of(service, spark.createDataFrame([(1, "q")], popped)) + assert table.to_pylist() == [{"a": 1}] + table, _ = _local_rows_relation_of( + service, spark.createDataFrame([{"a": 1, "z": 2}, {"z": 3}], replaced)) + assert table.to_pylist() == [{"z": "1"}, {"z": None}] + + def nested(inner): + return T.StructType([T.StructField("s", inner)]) + + for data, schema in (([((1, "q"),)], appended), ([((1,),)], popped)): + with pytest.raises(PySparkValueError) as error: + spark.createDataFrame(data, nested(schema)) + assert error.value.getCondition() == "AXIS_LENGTH_MISMATCH" + for data, schema in ( + ([((1,),)], appended), ([({"a": 1},)], appended), ([(None,)], appended), + ([((1, "q"),)], popped), ([({"a": 1},)], popped), ([(Row(a=1, b="q"),)], popped), + ): + with pytest.raises(IndexError, match="list index out of range"): + spark.createDataFrame(data, nested(schema)) + table, _ = _local_rows_relation_of( + service, spark.createDataFrame([({"a": 1, "z": 2},), ((3,),)], nested(replaced))) + assert table.to_pylist() == [{"s": {"z": "1"}}, {"s": {"z": "3"}}] + + +def test_pandas_frames_of_a_struct_changed_in_place_are_named_by_its_cached_names(mock_connect): + # As PySpark's createDataFrame of a pandas DataFrame, which names its columns with + # `toDF(*schema.fieldNames())`, of the cached `names` of a StructType whose `fields` list + # was changed in place, after the table of its fields as they are is renamed by as many of + # them, as official PySpark 4.3.0 (see test_dropin_e2e.py); the server raises of another + # number of names than columns. The drop-in named them by the fields as they are, and + # failed the rename of more names than columns. + pd = pytest.importorskip("pandas") + spark, service = mock_connect + + def column_names(df): + df.toArrow() + request = parse_message(service.execute_requests[-1]) + relation = parse_message(parse_message(request[3][0])[1][0]) + # Relation.to_df=18 -> ToDF.column_names=2. + return [name.decode() for name in parse_message(relation[18][0])[2]] + + replaced = T.StructType([T.StructField("a", T.StringType())]) + replaced.fields[0] = T.StructField("z", T.StringType()) + df = spark.createDataFrame(pd.DataFrame({"a": ["x"]}), replaced) + assert column_names(df) == ["a"] + table, schema = _local_rows_relation_of(service, df) + assert table.to_pylist() == [{"a": "x"}] and schema["fields"][0]["name"] == "z" + popped = _struct_changed_in_place("popped") + assert column_names(spark.createDataFrame(pd.DataFrame({"a": [1]}), popped)) == ["a", "b"] + + class _ScaledValueUDT(T.UserDefinedType): """A UserDefinedType of a scale, of the values of `_ScaledValue`.""" diff --git a/python/tests/test_dropin_types_columns_offline.py b/python/tests/test_dropin_types_columns_offline.py index f6f0df80..d144c16b 100644 --- a/python/tests/test_dropin_types_columns_offline.py +++ b/python/tests/test_dropin_types_columns_offline.py @@ -670,6 +670,65 @@ def __str__(self): pyspark_types_to_proto_types(UnparsedDataType(5)) +def test_proto_schema_conversion_raises_pysparks_errors_of_types_that_it_does_not_convert(): + # As official PySpark 4.3.0's `proto_schema_to_pyspark_data_type`, whose last branch raises + # UNSUPPORTED_OPERATION of `f"data type {schema}"`, of the proto text of the type, for a type + # of no kind, such as `pb2.DataType()` or an element type that an array does not set, and of + # a kind that it has no branch of, an unparsed type or a nanosecond-precision timestamp: in the + # order of its conversion, of which a spatial type of an SRID that it does not take raises + # ST_INVALID_SRID_VALUE first. A user-defined type is of its Python class, of no conversion of + # its SQL type. The drop-in raised a PySparkRuntimeError of its own for a type of no kind, + # and gave an UnparsedDataType. + from pyspark.errors import IllegalArgumentException + from pyspark.sql.connect import proto as pb2 + from pyspark.sql.connect.types import proto_schema_to_pyspark_data_type + + def struct(*builds): + dt = pb2.DataType() + dt.struct.SetInParent() + for i, build in enumerate(builds): + field = dt.struct.fields.add() + field.name, field.nullable = f"f{i}", True + if build is not None: + build(field.data_type) + return dt + + def unparsed(dt): + dt.unparsed.data_type_string = 'a "b"' + + def nanos(dt): + dt.timestamp_ntz_nanos.precision = 9 + + def srid(dt): + dt.geometry.srid = 2154 + + array, map_ = pb2.DataType(), pb2.DataType() + array.array.SetInParent() + map_.map.key_type.string.SetInParent() + for dt, text in [ + (pb2.DataType(), ""), + (struct(None), ""), + (struct(lambda dt: dt.long.SetInParent(), lambda dt: None), ""), + (array, ""), + (map_, ""), + (struct(unparsed), 'unparsed {\n data_type_string: "a \\"b\\""\n}\n'), + (struct(nanos, None), "timestamp_ntz_nanos {\n precision: 9\n}\n"), + (struct(None, nanos), ""), + ]: + with pytest.raises(PySparkValueError) as e: + proto_schema_to_pyspark_data_type(dt) + assert e.value.getCondition() == "UNSUPPORTED_OPERATION" + assert e.value.getMessageParameters() == {"operation": f"data type {text}"} + with pytest.raises(IllegalArgumentException) as e: + proto_schema_to_pyspark_data_type(struct(srid, None)) + assert e.value.getCondition() == "ST_INVALID_SRID_VALUE" + for sql_type in (nanos, unparsed, lambda dt: dt.SetInParent()): + udt = pb2.DataType() + udt.udt.type, udt.udt.python_class = "udt", "pyspark.ml.linalg.VectorUDT" + sql_type(udt.udt.sql_type) + assert type(proto_schema_to_pyspark_data_type(udt)).__name__ == "VectorUDT" + + def test_the_abstract_types_have_no_proto_nor_arrow_type_as_pysparks(): # PySpark's `pyspark_types_to_proto_types` has no proto of a type of its class only, of which # it raises UNSUPPORTED_OPERATION, also in a struct, an array or a map. `cast` and `try_cast`