From b1cb34cf103bd16c60c2b5fbfb06379c87c1a4be Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Thu, 19 Nov 2026 02:40:16 +0000 Subject: [PATCH] [SPARK-62065][CONNECT][PYTHON] Hold PySpark's user-defined type objects in the cached schemas of set operations and pickles, raise the conversion errors of the schema of the server where PySpark reads it, and make DataType.simpleString the typeName PySpark's _cached_schema of a DataFrame is the object of to() or of a map function, or the type objects of its conversion of the schema of the server, of a new object of each user-defined type (UserDefinedType.fromJson). The drop-in still gave the objects of its registry of lowered UDTs, by their JSON, in three places, or did not convert the schema: - set_operation_result compared a cached schema of the server's with data_type_to_py, the UDT that was lowered last, often the user's own; PySpark's _merge_cached_schema compares the new objects with ==, of their __dict__, so a union of to() of a marked UDT and a DataFrame of the server caches none. Now the known schema converts with schema_to_py (known_schema_object), and one that PySpark could not convert counts as none. - __reduce__ pickled the registry's objects, and the unpickle (_of_relation) lowered the pickled StructType and dropped it. Now the state carries the new objects of the server's schema, and the unpickled DataFrame keeps the pickled StructType as its schema object, which dtypes, columns and filter() read, while schema is the one that its unpickle requested. - The reads of the schema that need only the names (columns, printSchema, df[0], dir(), a lookup of a name of two underscores, the columns of mapInPandas and applyInPandas, the copies and unpickles, and the numeric aggregations of grouped data) did not convert it, so a UDT whose __init__ needs arguments, a UDT of no Python class or a nanosecond timestamp did not raise there. Now check_python_schema raises PySpark's errors in own_schema, fresh_schema_request, _of_relation and GroupedData's numeric aggregation. DataType.simpleString is self.typeName(), as PySpark's base method, which the types of a string of their own override; YearMonthIntervalType and DayTimeIntervalType have their own, PySpark's _str_repr. The drop-in's gave the simple string of the type, of the registry's UDTs. A schema that PySpark cannot convert is still requested once (dev/parity/known-differences.md). Co-authored-by: Isaac --- crates/pyspark-rs/src/dataframe.rs | 97 +++++++++++++---- crates/pyspark-rs/src/group.rs | 4 + crates/pyspark-rs/src/types.rs | 66 +++++------- dev/parity/known-differences.md | 8 ++ docs/compatibility.md | 25 +++-- python/tests/test_dropin_e2e.py | 78 +++++++++++++- python/tests/test_dropin_offline.py | 157 ++++++++++++++++++++++++++-- 7 files changed, 354 insertions(+), 81 deletions(-) diff --git a/crates/pyspark-rs/src/dataframe.rs b/crates/pyspark-rs/src/dataframe.rs index 1d4500c3..90b3101c 100644 --- a/crates/pyspark-rs/src/dataframe.rs +++ b/crates/pyspark-rs/src/dataframe.rs @@ -99,7 +99,8 @@ impl PyDataFrame { /// The schema of this DataFrame, as PySpark's `_cached_schema` is: the object of `to` that is /// not a DataType, which it keeps (see `schema_object`), or else the schema of the server, as - /// PySpark's client converts it. The schema is cached, so it is asked of the server once. + /// PySpark's client converts it, of the errors of that conversion (see `check_python_schema`). + /// The schema is cached, so it is asked of the server once. fn own_schema<'py>(&self, py: Python<'py>) -> PyResult> { if let Some(schema) = &self.schema_object { return Ok(PropertySchema::Object(schema.bind(py).clone())); @@ -107,10 +108,21 @@ impl PyDataFrame { let schema = interruptible(py, &self.dataframe.spark_session(), || { self.dataframe.schema() })?; - check_schema_conversion(py, Some(&schema))?; + check_python_schema(py, &schema)?; Ok(PropertySchema::Core(schema)) } + /// Raises as PySpark's `schema` of this DataFrame does, which its grouped data reads, of the + /// conversion of the schema that it caches (see `own_schema`): of a copy or an unpickle, + /// whose `schema` is the schema that it requested and converted then (see `copy_schema`), + /// nothing. + pub(crate) fn check_schema_read(&self, py: Python<'_>) -> PyResult<()> { + if self.copy_schema.is_none() { + self.own_schema(py)?; + } + Ok(()) + } + /// The schema that the properties `schema`, `columns` and `dtypes` read, as PySpark's read /// `self._schema`: `own_schema`, or, when the plan fails to build with an `AttributeError`, as /// of a join on a tuple, the object that Python then gives of `self._schema`, as PySpark's @@ -358,7 +370,10 @@ impl PyDataFrame { .to_pyerr() .map_err(|err| columns_read_error(py, err))?; let fresh = DataFrame::from_sent_relation(session.clone(), relation); - interruptible(py, &session, || fresh.schema()) + let schema = interruptible(py, &session, || fresh.schema())?; + // As the read of the columns, which converts the schema (see `check_python_schema`). + check_python_schema(py, &schema)?; + Ok(schema) } /// The number of rows and the truncation of the eager evaluation, as PySpark's `int` of @@ -1009,8 +1024,9 @@ impl PyDataFrame { }, }; // The schema that this DataFrame cached, of the types of PySpark, when they hold it: the - // `StructType` that it keeps as an object, of `to` or of a map function, as it is now, as - // PySpark's state carries the object. + // `StructType` that it keeps as an object, of `to`, of a map function or of an unpickle, + // as it is now, as PySpark's state carries the object; else that of the server, of new + // objects of its user-defined types, as PySpark's `_schema` made them. let cached_schema = match &this.schema_object { Some(schema) if schema @@ -1019,12 +1035,7 @@ impl PyDataFrame { { Some(schema.clone_ref(py)) } - _ => this - .dataframe - .known_schema() - .filter(|schema| check_schema_conversion(py, Some(schema)).is_ok()) - .map(|schema| crate::types::data_type_to_py(py, &schema)) - .transpose()?, + _ => known_schema_object(py, &this.dataframe)?, }; py.import("pyspark.sql.connect.dataframe")? .getattr("_reduce_dataframe")? @@ -1179,9 +1190,10 @@ impl PyDataFrame { /// The DataFrame of `relation`, the bytes of a relation of `levels` levels of `session`, as a /// pickled DataFrame unpickles, with the `_support_repr_html` and the `_cached_schema` of its /// state. As PySpark's unpickle, whose lookup of the `__setstate__` of the new DataFrame reads - /// its columns, it requests the schema of the relation, whose error it raises, and keeps it - /// (see `copy_schema`), and then the DataFrame has the cached schema of the pickled one, - /// none when it had none, so that its `schema` requests it again then. Without + /// its columns, it requests the schema of the relation, whose error it raises, also that of + /// its conversion (see `check_python_schema`), and keeps it (see `copy_schema`), and then the + /// DataFrame has the cached schema of the pickled one, the objects that it is of, or none + /// when it had none, so that its `schema` requests it again then. Without /// `request_schema`, of an object that PySpark pickles of the plan of the DataFrame, as a /// DataStreamWriter, the schema is not requested. #[staticmethod] @@ -1214,12 +1226,18 @@ impl PyDataFrame { .to_pyerr()?; let requested = if request_schema { let schema = interruptible(py, &dataframe.spark_session(), || dataframe.schema())?; - check_schema_conversion(py, Some(&schema))?; + check_python_schema(py, &schema)?; Some(schema) } else { None }; let mut dataframe = PyDataFrame::new(dataframe.with_known_schema(known_schema)); + // The `StructType` of the state as the object that it is, as PySpark's `_cached_schema`, + // which `columns`, `dtypes` and the methods that copy the cached schema read: of the + // objects of `to`, of a map function or of the server that the pickled DataFrame held. + dataframe.schema_object = cached_schema + .filter(|schema| schema.is_instance_of::()) + .map(|schema| schema.clone().unbind()); if let Some(requested) = requested { dataframe.keep_copy_schema(requested); } @@ -7289,13 +7307,51 @@ pub(crate) fn check_schema_conversion( } } +/// Raises as PySpark's conversion of `schema`, a schema of the server, raises where its +/// `_schema` makes the type objects of it: the errors of [`check_schema_conversion`], and those of +/// its user-defined types, of which `UserDefinedType.fromJson` makes `UDT()` of the class that +/// the `pyClass` names (see `types::schema_to_py`), so that a class whose `__init__` takes +/// arguments raises its `TypeError`. The properties that read the schema, the column lookups, +/// the copies and the unpickles raise it so. +pub(crate) fn check_python_schema( + py: Python<'_>, + schema: &spark_connect::types::DataType, +) -> PyResult<()> { + check_schema_conversion(py, Some(schema))?; + if crate::types::has_udt(schema) { + crate::types::schema_to_py(py, schema)?; + } + Ok(()) +} + +/// The `_cached_schema` of PySpark's DataFrame of `dataframe`, of the schema that the core caches, +/// which is of the server: its type objects, as PySpark converts the schema of the server, of a +/// new object of each user-defined type (see `types::schema_to_py`), and not of the objects that +/// the client was given. None when it has none, or when PySpark would not have one, as its +/// conversion of the schema raised (see [`check_python_schema`]). +fn known_schema_object(py: Python<'_>, dataframe: &DataFrame) -> PyResult>> { + let Some(schema) = dataframe.known_schema() else { + return Ok(None); + }; + if check_schema_conversion(py, Some(&schema)).is_err() { + return Ok(None); + } + match crate::types::schema_to_py(py, &schema) { + Ok(schema) => Ok(Some(schema)), + Err(err) if err.is_instance_of::(py) => Ok(None), + Err(err) => Err(err), + } +} + /// The result of a set operation of `this` and `other`, `dataframe`, which keeps the schema that /// PySpark's `_merge_cached_schema` keeps: `this.schema` when the cached schemas of both are /// equal, else none. The core keeps that of two cached structs; of a schema that a DataFrame /// keeps as a Python object, the schema of `to` that is not a DataType or the `StructType` of a /// map function, the objects are compared as they are now, as PySpark compares them with `==`, -/// and the result keeps a deep copy of that of `this`, as `this.schema` gives. Of a copy or an -/// unpickle, `this.schema` is the schema that it requested (see `copy_schema`). +/// with the type objects of the cached schema of the other, which of the server holds new objects +/// of its user-defined types (see `known_schema_object`), and the result keeps a deep copy of +/// that of `this`, as `this.schema` gives. Of a copy or an unpickle, `this.schema` is the schema +/// that it requested (see `copy_schema`). fn set_operation_result( py: Python<'_>, this: &PyDataFrame, @@ -7313,12 +7369,7 @@ fn set_operation_result( let cached = |df: &PyDataFrame| -> PyResult>> { match &df.schema_object { Some(schema) => Ok(Some(schema.clone_ref(py))), - None => df - .dataframe - .known_schema() - .filter(|schema| check_schema_conversion(py, Some(schema)).is_ok()) - .map(|schema| crate::types::data_type_to_py(py, &schema)) - .transpose(), + None => known_schema_object(py, &df.dataframe), } }; if let (Some(left), Some(right)) = (cached(this)?, cached(other)?) { diff --git a/crates/pyspark-rs/src/group.rs b/crates/pyspark-rs/src/group.rs index 55c80762..63c62f2f 100644 --- a/crates/pyspark-rs/src/group.rs +++ b/crates/pyspark-rs/src/group.rs @@ -226,6 +226,10 @@ impl PyGroupedData { }; names.push(name.to_cow().ok().map(|name| name.into_owned())); } + // As PySpark's `self._df.schema`, which raises the errors of its conversion of the schema + // of the server, such as the `TypeError` of a user-defined type whose `__init__` takes + // arguments. + self.frame.bind(py).borrow().check_schema_read(py)?; let requested = self.frame.bind(py).borrow().copy_schema(); let of_requested; let grouped = match requested { diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs index f7dab459..7336fd04 100644 --- a/crates/pyspark-rs/src/types.rs +++ b/crates/pyspark-rs/src/types.rs @@ -2058,46 +2058,12 @@ impl PyDataType { class_repr(slf.as_any()) } - /// As pyspark's `simpleString()`, the `typeName()` of a type of its class only. An array, map - /// or struct of a user-defined type names it by the `simpleString()` of its Python class, - /// such as `array`. - fn simpleString(slf: &Bound<'_, Self>) -> PyResult { - let py = slf.py(); - // A struct, array or map reads its live type, as its own `simpleString` does: the base - // method is also called on them directly, as `DataType.simpleString(st)`. - if slf.is_instance_of::() - || slf.is_instance_of::() - || slf.is_instance_of::() - { - return match py_to_data_type(slf.as_any()) { - Ok(data_type) => simple_string_with_udts(py, &data_type), - // Of a type that holds one that does not lower now, of the type objects that it - // holds, as PySpark's reads them (see `lenient_type`). - Err(error) if is_exception(py, &error) => { - if let Ok(array) = slf.cast::() { - return array.borrow().simple_string(py); - } - if let Ok(map) = slf.cast::() { - return map.borrow().simple_string(py); - } - slf.cast::()?.borrow().simple_string(py) - } - Err(error) => Err(error), - }; - } - let this = slf.borrow(); - let Some(inner) = &this.inner else { - drop(this); - return slf.call_method0(pyo3::intern!(py, "typeName"))?.extract(); - }; - match inner { - DataType::Array { .. } | DataType::Map { .. } | DataType::Struct { .. } - if has_udt(inner) => - { - simple_string_with_udts(py, inner) - } - _ => Ok(inner.simple_string()), - } + /// As PySpark's `DataType.simpleString`, `self.typeName()`. The types of a string of their + /// own, such as `bigint`, `decimal(10,2)` or `struct`, override it, as PySpark's + /// do, so the base method called on one of them directly, as `DataType.simpleString(st)`, is + /// its `typeName()`, `struct`, as in PySpark. + fn simpleString<'py>(slf: &Bound<'py, Self>) -> PyResult> { + type_name_of(slf.as_any()) } // Equality/hashing are defined once here and inherited by every concrete type class @@ -5945,6 +5911,16 @@ impl PyYearMonthIntervalType { const YEAR: i32 = 0; #[classattr] const MONTH: i32 = 1; + /// As PySpark's `simpleString`, its `_str_repr` of the fields: `interval year to month`, or + /// `interval year` of one field, also of a Python subclass, as it reads no `typeName()`. + #[pyo3(name = "simpleString")] + fn simple_string(&self) -> String { + DataType::YearMonthInterval { + start_field: self.start_field, + end_field: self.end_field, + } + .simple_string() + } // --- DataType object-model methods (v4.2.0 parity) --- /// As PySpark's `DataType.json`, of the `jsonValue` of the type, of a Python subclass its own. #[pyo3(name = "json")] @@ -6113,6 +6089,16 @@ impl PyDayTimeIntervalType { const MINUTE: i32 = 2; #[classattr] const SECOND: i32 = 3; + /// As PySpark's `simpleString`, its `_str_repr` of the fields: `interval day to second`, or + /// `interval day` of one field, also of a Python subclass, as it reads no `typeName()`. + #[pyo3(name = "simpleString")] + fn simple_string(&self) -> String { + DataType::DayTimeInterval { + start_field: self.start_field, + end_field: self.end_field, + } + .simple_string() + } // --- DataType object-model methods (v4.2.0 parity) --- /// As PySpark's `DataType.json`, of the `jsonValue` of the type, of a Python subclass its own. #[pyo3(name = "json")] diff --git a/dev/parity/known-differences.md b/dev/parity/known-differences.md index bfbbcba8..6db72892 100644 --- a/dev/parity/known-differences.md +++ b/dev/parity/known-differences.md @@ -134,6 +134,14 @@ difference is intentional or tracked for a fix. `TypeError: StructType fields must be StructField objects` there, and an assignment of `fields` of such an element raises it at once. Of a list of `StructField`s changed in place, the cached `names` and their uses are PySpark's (see `docs/compatibility.md`). +- **A schema that PySpark does not convert is requested once** (intentional; the errors are the + same). PySpark's `_schema` keeps the schema that it requested only once its conversion of it + succeeded, so of a schema that it does not convert, such as of a user-defined type whose + `__init__` takes arguments or of no Python class, of a nanosecond-precision timestamp, or of a + spatial type of an SRID that it does not take, each read of the schema (`schema`, `columns`, + `dtypes`, `printSchema` and the lookups that read the columns) requests it again, and raises. + This client keeps the schema of the server once it requested it, as of any other schema, and + raises the same error of its conversion at each read, of no other request. - **The `__repr__` of a Column of a str that UTF-8 cannot encode** (intentional; the expressions of the core hold UTF-8). Such a str, of a name, a SQL string or a literal, fails the plans of the Column when they are built, as in PySpark, and its lone surrogates show escaped in the `__repr__` diff --git a/docs/compatibility.md b/docs/compatibility.md index caf6041b..44ec8c65 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -2274,9 +2274,16 @@ that hold them, as `UserDefinedType`s of other parameters do, and `toNullable()` are. The schema of a DataFrame and of its rows, as the server gives it, holds a new object of each user-defined type, as PySpark's `UserDefinedType.fromJson` makes it of its class, and so does `fromJson`: not the object that the client was given, nor of its attributes, and a class whose -`__init__` takes arguments raises its `TypeError` there, as in PySpark. A struct, array or map that -the client makes of core types, without the objects of a user, holds the user-defined type of its -JSON that a type was last made of, if one was. `IntegralType()` is one instance, as PySpark's +`__init__` takes arguments raises its `TypeError` wherever PySpark converts that schema, as in +PySpark: the schema, `columns`, `dtypes`, `printSchema`, the rows, the lookups that read the +columns (`df[0]`, `dir(df)`, names of two underscores), the copies and the unpickles, the map +functions and the numeric aggregations of grouped data. The cached schema of a DataFrame holds the +objects that PySpark's `_cached_schema` holds, those of `to` or of a map function, else the new +objects of the schema of the server: a set operation compares them as PySpark's does, by the +`__dict__` of their user-defined types, and a pickle carries them, which the unpickled DataFrame +keeps as its cached schema. `DataType.simpleString(t)`, the method of the base class, is +`t.typeName()`, as PySpark's, which the types of a string of their own, such as `bigint` or +`struct`, override. `IntegralType()` is one instance, as PySpark's `DataTypeSingleton` makes it, and so is each Python subclass of it, of an instance of its own, and a Python subclass of the other classes, such as a `UserDefinedType` of parameters, takes the arguments of its `__init__`. @@ -2881,13 +2888,13 @@ session fails the copy with `SparkConnectException`, as PySpark's does. One diff PySpark's `DataFrame.__getattr__` answers `getattr(df, "__deepcopy__")` and `getattr(df, "__copy__")` by requesting the schema, to read the columns, and raises `ATTRIBUTE_NOT_SUPPORTED`, so `hasattr(df, "__copy__")` is false; the drop-in defines both -itself, which it finds without a request. An unpickled DataFrame requests its -schema once, at the unpickle, which raises the error of the request, as PySpark's pickle looks the +itself, which it finds without a request. An unpickled DataFrame requests its schema once, at the +unpickle, which raises the error of the request or of its conversion, as PySpark's pickle looks the `__setstate__` of the new DataFrame up through its `__getattr__`. Its cached schema is then that of -the pickled DataFrame, as the state of PySpark's pickle sets it: of none, its `schema` requests it -again. So does the unpickle of an object that PySpark pickles with its DataFrame: a writer, of its -callback, grouped data, `na` and `stat`; a stream writer, which PySpark's pickles of the plan of -the DataFrame, requests nothing. +the pickled DataFrame, as the state of PySpark's pickle sets it, of the objects that it carries: of +none, its `schema` requests it again. So does the unpickle of an object that PySpark pickles with +its DataFrame: a writer, of its callback, grouped data, `na` and `stat`; a stream writer, which +PySpark's pickles of the plan of the DataFrame, requests nothing. A copy, a deep copy or an unpickle of a DataFrame keeps the schema that it requests apart from the cached schema of the state, as PySpark's keeps it as the `_cached_schema_serialized` that its diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py index 703a3c4c..9b9dd0c3 100644 --- a/python/tests/test_dropin_e2e.py +++ b/python/tests/test_dropin_e2e.py @@ -936,8 +936,10 @@ def test_nanosecond_timestamps_are_not_collected_as_pyspark(session): # As PySpark's conversion of the schema from its proto: a TIMESTAMP_NTZ(p) or TIMESTAMP_LTZ(p) # with p > 6, also nested, raises UNSUPPORTED_OPERATION from every collecting API, toPandas # and toArrow, and from every schema surface (the schema, columns, dtypes, printSchema, repr - # and json, and the DDL strings of the types), while count, show and the microsecond - # timestamps keep working. + # and json, the copies, whose lookups read the columns of the copy, the numeric aggregations + # of grouped data, which read `self._df.schema`, and the DDL strings of the types), while + # count, show and the microsecond timestamps keep working. + import copy import datetime import re @@ -988,6 +990,9 @@ def message(proto_text): lambda d: d.printSchema(), lambda d: repr(d), lambda d: d.schema.json(), + copy.copy, + copy.deepcopy, + lambda d: d.groupBy().sum(), ): with pytest.raises(PySparkValueError, match=message(proto_text)): surface(df) @@ -4922,6 +4927,75 @@ def test_schemas_of_the_server_make_new_user_defined_type_objects_as_pyspark(ses assert own["p"].dataType is marked and ArrayType(marked).elementType is marked +def test_cached_schemas_of_set_operations_and_pickles_hold_the_objects_of_pyspark(session): + # As official PySpark 4.3.0, whose cached schema of a DataFrame (`_cached_schema`) is the + # object of `to`, or the type objects of its conversion of the schema of the server, of new + # user-defined type objects. A set operation keeps the cached schema of equal ones + # (`_merge_cached_schema`), which `==` compares by the `__dict__` of their user-defined types: + # that of `to` of a marked object differs from the server's, so their union caches none, and + # its schema is the server's. A pickle carries the cached schema as its objects, which the + # methods that copy the cached schema of the unpickled DataFrame keep, such as `filter`, while + # its own `schema` is the one that its unpickle requested. The client gave instead the objects + # of their JSON that it lowered last. + from pyspark import cloudpickle + + marked = _MarkedPointUDT() + marked.marker = "made" + server = session.createDataFrame( + [((1.0, 2.0),)], StructType([StructField("p", _MarkedPointUDT())]) + ) + assert server.columns == ["p"] + cast = server.to(StructType([StructField("p", marked)])) + for union in (cast.union(server), server.union(cast), cast.intersect(server)): + assert not hasattr(union.schema["p"].dataType, "marker") + assert cast.union(cast).schema["p"].dataType.marker == "made" + unpickled = cloudpickle.loads(cloudpickle.dumps(cast)) + assert not hasattr(unpickled.schema["p"].dataType, "marker") + assert unpickled.filter("true").schema["p"].dataType.marker == "made" + assert unpickled.columns == ["p"] and unpickled.dtypes == [("p", "udt")] + unpickled = cloudpickle.loads(cloudpickle.dumps(server)) + assert not hasattr(unpickled.filter("true").schema["p"].dataType, "marker") + assert unpickled.collect() == [Row(p=(1.0, 2.0))] + + +def test_reads_of_a_schema_of_a_user_defined_type_of_arguments_raise_as_pyspark(session): + # As official PySpark 4.3.0, whose reads of the schema of a DataFrame convert the schema of the + # server (`_schema`) with `UserDefinedType.fromJson`, which calls the class of no arguments: + # of a class whose __init__ takes arguments, `columns` and what reads them (printSchema, the + # index of a column, `dir`, the lookups of names of two underscores, the copies, of which + # pickle and copy look `__setstate__` up so, and the map functions), and the numeric + # aggregations of grouped data, which read `self._df.schema`, raise its TypeError. The uses + # of the plan that do not read the schema work. + import copy + + from pyspark import cloudpickle + + df = session.createDataFrame( + [((1.0, 2.0), 1)], + StructType([StructField("p", _ArgPointUDT(2)), StructField("i", LongType())]), + ) + for use in ( + lambda: df.columns, + df.printSchema, + lambda: df[0], + lambda: dir(df), + lambda: hasattr(df, "__array__"), + lambda: df.select("*").columns, + lambda: copy.copy(df), + lambda: copy.deepcopy(df), + lambda: cloudpickle.loads(cloudpickle.dumps(df)), + lambda: df.groupBy("i").sum(), + lambda: df.groupBy("i").avg("i"), + lambda: df.groupBy("i").applyInPandas(lambda pdf: pdf, "i long"), + lambda: df.mapInPandas(lambda batches: batches, "i long"), + ): + with pytest.raises(TypeError, match="missing 1 required positional argument"): + use() + assert df.count() == 1 + assert df.select("i").columns == ["i"] + assert df.groupBy("i").count().collect() == [Row(i=1, count=1)] + + def test_uses_of_the_json_of_a_type_of_a_user_defined_type_of_no_json_raise_its_error(session): # As official PySpark 4.3.0, whose struct, array and map of a UserDefinedType whose module() # is not implemented are made (see test_dropin_types_columns_offline.py): the uses that read diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 4d5cf660..269a1343 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -5618,17 +5618,65 @@ def test_structtype_add_reflected_in_json(): def test_structtype_add_reflected_in_the_base_data_type(): # Regression: add() must update the base DataType too, which the schemas of - # mapInArrow/mapInPandas/applyInPandas/... read (via resolve_datatype). The base - # implementation of simpleString reads it. + # mapInArrow/mapInPandas/applyInPandas/... read (via resolve_datatype), and so its JSON. built = T.StructType([T.StructField("a", T.LongType()), T.StructField("b", T.LongType())]) chained = T.StructType().add("a", T.LongType()).add("b", T.LongType()) st = T.StructType([T.StructField("a", T.LongType())]) st.add("b", T.LongType()) for added in (chained, st): - assert T.DataType.simpleString(added) == "struct" + assert added.simpleString() == "struct" + assert added.json() == built.json() assert str(added) == str(built) == repr(built) +@pytest.mark.parametrize("data_type,type_name", [ + (T.ByteType(), "byte"), + (T.IntegerType(), "integer"), + (T.LongType(), "long"), + (T.DecimalType(10, 2), "decimal"), + (T.StringType("UTF8_LCASE"), "string"), + (T.CharType(3), "char"), + (T.VarcharType(3), "varchar"), + (T.TimeType(6), "time"), + (T.YearMonthIntervalType(), "yearmonthinterval"), + (T.DayTimeIntervalType(0, 3), "daytimeinterval"), + (T.GeometryType(4326), "geometry"), + (T.ArrayType(T.IntegerType()), "array"), + (T.MapType(T.StringType(), T.LongType()), "map"), + (T.StructType([T.StructField("a", T.LongType())]), "struct"), + (T.StructType([T.StructField("a", _DayPointUDT())]), "struct"), + (T.CalendarIntervalType(), "interval"), + (T.TimestampNTZType(), "timestamp_ntz"), + (T.NullType(), "void"), + (T.AtomicType(), "atomic"), +]) +def test_the_base_simple_string_of_a_type_is_its_type_name_as_pysparks(data_type, type_name): + # As PySpark's DataType.simpleString, `self.typeName()`, which the types of a string of + # their own override: the base method called on one of them directly is its typeName(), as + # official PySpark 4.3.0's. The drop-in's gave the simple string of the type, of the + # user-defined type objects of its JSON that it had lowered last. + assert T.DataType.simpleString(data_type) == type_name == data_type.typeName() + + +@pytest.mark.parametrize("base,simple", [ + (T.DoubleType, "renamed"), + (T.LongType, "bigint"), + (T.DecimalType, "decimal(10,0)"), + (T.YearMonthIntervalType, "interval year to month"), + (T.DayTimeIntervalType, "interval day to second"), +]) +def test_the_simple_strings_of_subclasses_of_their_own_type_name_are_pysparks(base, simple): + # As official PySpark 4.3.0: of a Python subclass of another typeName(), the base + # simpleString is that name, and the simpleString of its class is that of PySpark's class, + # which reads the typeName() of the type of its class only, as DoubleType's, and not of the + # interval types, whose own simpleString is of their fields. + subclass = type("Renamed" + base.__name__, (base,), { + "typeName": classmethod(lambda cls: "renamed")}) + data_type = subclass() + assert T.DataType.simpleString(data_type) == "renamed" + assert data_type.simpleString() == simple + + @pytest.mark.parametrize("data_type,expected", [ (T.LongType(), "LongType()"), (T.TimestampNTZType(), "TimestampNTZType()"), @@ -23348,6 +23396,91 @@ def column_names(df): assert column_names(spark.createDataFrame(pd.DataFrame({"a": [1]}), popped)) == ["a", "b"] +def _server_schema_of(*fields): + # The schema that the server sends of the fields (name, type), of LongType or of a + # user-defined type of a LongType SQL type: DataType.struct=21, Struct.fields=1, + # StructField: name=1, data_type=2, nullable=3; DataType.long=7, and DataType.udt=23 -> + # UDT: type=1, python_class=3, sql_type=5. + def proto(data_type): + if isinstance(data_type, T.UserDefinedType): + python_class = f"{data_type.module()}.{type(data_type).__name__}" + return message_field(23, message_field(1, "udt") + message_field(3, python_class) + + message_field(5, message_field(7, b""))) + return message_field(7, b"") + + return message_field(21, b"".join( + message_field(1, message_field(1, name) + message_field(2, proto(data_type)) + + varint_field(3, 1)) + for name, data_type in fields)) + + +def test_reads_of_a_schema_of_a_udt_of_constructor_arguments_raise_as_pysparks(mock_connect): + # As PySpark's reads of the schema of a DataFrame, which convert the schema of the server + # (`_schema`) with UserDefinedType.fromJson, which calls the class of no arguments: of a + # class whose __init__ needs one, `columns` and what reads them (printSchema, the index of a + # column, `dir`, the lookups of names of two underscores, and the copies and an unpickle, + # which look `__setstate__` up so), and the numeric aggregations of grouped data, which read + # `self._df.schema`, raise its TypeError, as official PySpark 4.3.0 does (see + # test_dropin_e2e.py). The drop-in read the names of the columns of its core type. + import copy + + from pyspark import cloudpickle + + spark, service = mock_connect + service.data_type = _server_schema_of(("p", _OffsetUDT(1)), ("i", T.LongType())) + df = spark.range(1) + for use in ( + lambda: df.columns, + df.printSchema, + lambda: df[0], + lambda: dir(df), + lambda: hasattr(df, "__array__"), + lambda: df._ipython_key_completions_(), + lambda: copy.copy(df), + lambda: copy.deepcopy(df), + lambda: cloudpickle.loads(cloudpickle.dumps(df)), + lambda: df.groupBy("i").sum(), + lambda: df.groupBy("i").max("i"), + ): + with pytest.raises(TypeError, match="missing 1 required positional argument"): + use() + # Of a class of a default, a new object of it. + service.data_type = _server_schema_of(("p", _ScaleUDT(3)), ("i", T.LongType())) + df = spark.range(1) + assert df.columns == ["p", "i"] and df.schema["p"].dataType.scale == 1 + assert hasattr(copy.copy(df), "columns") and df.groupBy("i").sum() is not None + + +def test_cached_schemas_of_set_operations_and_pickles_are_pysparks_objects(mock_connect): + # As PySpark's `_cached_schema`, the object of `to`, or the type objects of its conversion of + # the schema of the server, of new user-defined type objects: a set operation keeps the cached + # schema of `this` when both are equal by `==` (`_merge_cached_schema`), which compares the + # `__dict__` of user-defined types, so that of `to` of another scale and the server's differ, + # and their union has none, whose schema is the server's. A pickle carries the cached schema + # as its objects, which `filter` of the unpickled DataFrame keeps, while its own `schema` is + # the one that its unpickle requested; that of a DataFrame of the server's schema is of new + # objects. As official PySpark 4.3.0 (see test_dropin_e2e.py); the drop-in gave the objects + # of their JSON that it lowered last. + from pyspark import cloudpickle + + spark, service = mock_connect + service.data_type = _server_schema_of(("p", _ScaleUDT())) + server = spark.range(1) + assert server.columns == ["p"] + cast = server.to(T.StructType([T.StructField("p", _ScaleUDT(2))])) + for union in (cast.union(server), server.union(cast), cast.exceptAll(server)): + assert union.schema["p"].dataType.scale == 1 + assert cast.union(cast).schema["p"].dataType.scale == 2 + unpickled = cloudpickle.loads(cloudpickle.dumps(cast)) + assert unpickled.schema["p"].dataType.scale == 1 + assert unpickled.filter("true").schema["p"].dataType.scale == 2 + assert unpickled.union(cast).schema["p"].dataType.scale == 1 + unpickled = cloudpickle.loads(cloudpickle.dumps(server)) + service.data_type = _server_schema_of(("q", T.LongType())) + assert unpickled.columns == ["p"] + assert unpickled.filter("true").schema["p"].dataType.scale == 1 + + class _ScaledValueUDT(T.UserDefinedType): """A UserDefinedType of a scale, of the values of `_ScaledValue`.""" @@ -34339,10 +34472,13 @@ def test_schema_tree_type_names_and_escaped_field_names(): "python_class, expected", [ ("pyspark.ml.linalg.VectorUDT", "vectorudt"), - (None, "udt"), + (None, None), ], ) def test_print_schema_udt_class_name(mock_connect, capsys, python_class, expected): + # As PySpark's printSchema, of the schema that its `_schema` converts, whose + # UserDefinedType.fromJson makes the user-defined type of its Python class: of a type of no + # Python class, it raises the KeyError of `pyClass`, as `columns` does. spark, service = mock_connect udt = message_field(1, "udt") if python_class is not None: @@ -34350,9 +34486,16 @@ def test_print_schema_udt_class_name(mock_connect, capsys, python_class, expecte service.data_type = message_field( 21, message_field(1, _print_schema_field("features", message_field(23, udt))) ) - assert spark.range(10).printSchema() is None - assert capsys.readouterr().out == f"root\n |-- features: {expected} (nullable = true)\n\n" - assert service.calls == 1 + if expected is None: + df = spark.range(10) + for read in (df.printSchema, lambda: df.columns): + with pytest.raises(KeyError, match="pyClass"): + read() + assert capsys.readouterr().out == "" + else: + assert spark.range(10).printSchema() is None + assert capsys.readouterr().out == f"root\n |-- features: {expected} (nullable = true)\n\n" + assert service.calls == 1 assert service.execute_calls == 0