diff --git a/crates/pyspark-rs/src/dataframe.rs b/crates/pyspark-rs/src/dataframe.rs index 1d4500c3..90b3101c 100644 --- a/crates/pyspark-rs/src/dataframe.rs +++ b/crates/pyspark-rs/src/dataframe.rs @@ -99,7 +99,8 @@ impl PyDataFrame { /// The schema of this DataFrame, as PySpark's `_cached_schema` is: the object of `to` that is /// not a DataType, which it keeps (see `schema_object`), or else the schema of the server, as - /// PySpark's client converts it. The schema is cached, so it is asked of the server once. + /// PySpark's client converts it, of the errors of that conversion (see `check_python_schema`). + /// The schema is cached, so it is asked of the server once. fn own_schema<'py>(&self, py: Python<'py>) -> PyResult> { if let Some(schema) = &self.schema_object { return Ok(PropertySchema::Object(schema.bind(py).clone())); @@ -107,10 +108,21 @@ impl PyDataFrame { let schema = interruptible(py, &self.dataframe.spark_session(), || { self.dataframe.schema() })?; - check_schema_conversion(py, Some(&schema))?; + check_python_schema(py, &schema)?; Ok(PropertySchema::Core(schema)) } + /// Raises as PySpark's `schema` of this DataFrame does, which its grouped data reads, of the + /// conversion of the schema that it caches (see `own_schema`): of a copy or an unpickle, + /// whose `schema` is the schema that it requested and converted then (see `copy_schema`), + /// nothing. + pub(crate) fn check_schema_read(&self, py: Python<'_>) -> PyResult<()> { + if self.copy_schema.is_none() { + self.own_schema(py)?; + } + Ok(()) + } + /// The schema that the properties `schema`, `columns` and `dtypes` read, as PySpark's read /// `self._schema`: `own_schema`, or, when the plan fails to build with an `AttributeError`, as /// of a join on a tuple, the object that Python then gives of `self._schema`, as PySpark's @@ -358,7 +370,10 @@ impl PyDataFrame { .to_pyerr() .map_err(|err| columns_read_error(py, err))?; let fresh = DataFrame::from_sent_relation(session.clone(), relation); - interruptible(py, &session, || fresh.schema()) + let schema = interruptible(py, &session, || fresh.schema())?; + // As the read of the columns, which converts the schema (see `check_python_schema`). + check_python_schema(py, &schema)?; + Ok(schema) } /// The number of rows and the truncation of the eager evaluation, as PySpark's `int` of @@ -1009,8 +1024,9 @@ impl PyDataFrame { }, }; // The schema that this DataFrame cached, of the types of PySpark, when they hold it: the - // `StructType` that it keeps as an object, of `to` or of a map function, as it is now, as - // PySpark's state carries the object. + // `StructType` that it keeps as an object, of `to`, of a map function or of an unpickle, + // as it is now, as PySpark's state carries the object; else that of the server, of new + // objects of its user-defined types, as PySpark's `_schema` made them. let cached_schema = match &this.schema_object { Some(schema) if schema @@ -1019,12 +1035,7 @@ impl PyDataFrame { { Some(schema.clone_ref(py)) } - _ => this - .dataframe - .known_schema() - .filter(|schema| check_schema_conversion(py, Some(schema)).is_ok()) - .map(|schema| crate::types::data_type_to_py(py, &schema)) - .transpose()?, + _ => known_schema_object(py, &this.dataframe)?, }; py.import("pyspark.sql.connect.dataframe")? .getattr("_reduce_dataframe")? @@ -1179,9 +1190,10 @@ impl PyDataFrame { /// The DataFrame of `relation`, the bytes of a relation of `levels` levels of `session`, as a /// pickled DataFrame unpickles, with the `_support_repr_html` and the `_cached_schema` of its /// state. As PySpark's unpickle, whose lookup of the `__setstate__` of the new DataFrame reads - /// its columns, it requests the schema of the relation, whose error it raises, and keeps it - /// (see `copy_schema`), and then the DataFrame has the cached schema of the pickled one, - /// none when it had none, so that its `schema` requests it again then. Without + /// its columns, it requests the schema of the relation, whose error it raises, also that of + /// its conversion (see `check_python_schema`), and keeps it (see `copy_schema`), and then the + /// DataFrame has the cached schema of the pickled one, the objects that it is of, or none + /// when it had none, so that its `schema` requests it again then. Without /// `request_schema`, of an object that PySpark pickles of the plan of the DataFrame, as a /// DataStreamWriter, the schema is not requested. #[staticmethod] @@ -1214,12 +1226,18 @@ impl PyDataFrame { .to_pyerr()?; let requested = if request_schema { let schema = interruptible(py, &dataframe.spark_session(), || dataframe.schema())?; - check_schema_conversion(py, Some(&schema))?; + check_python_schema(py, &schema)?; Some(schema) } else { None }; let mut dataframe = PyDataFrame::new(dataframe.with_known_schema(known_schema)); + // The `StructType` of the state as the object that it is, as PySpark's `_cached_schema`, + // which `columns`, `dtypes` and the methods that copy the cached schema read: of the + // objects of `to`, of a map function or of the server that the pickled DataFrame held. + dataframe.schema_object = cached_schema + .filter(|schema| schema.is_instance_of::()) + .map(|schema| schema.clone().unbind()); if let Some(requested) = requested { dataframe.keep_copy_schema(requested); } @@ -7289,13 +7307,51 @@ pub(crate) fn check_schema_conversion( } } +/// Raises as PySpark's conversion of `schema`, a schema of the server, raises where its +/// `_schema` makes the type objects of it: the errors of [`check_schema_conversion`], and those of +/// its user-defined types, of which `UserDefinedType.fromJson` makes `UDT()` of the class that +/// the `pyClass` names (see `types::schema_to_py`), so that a class whose `__init__` takes +/// arguments raises its `TypeError`. The properties that read the schema, the column lookups, +/// the copies and the unpickles raise it so. +pub(crate) fn check_python_schema( + py: Python<'_>, + schema: &spark_connect::types::DataType, +) -> PyResult<()> { + check_schema_conversion(py, Some(schema))?; + if crate::types::has_udt(schema) { + crate::types::schema_to_py(py, schema)?; + } + Ok(()) +} + +/// The `_cached_schema` of PySpark's DataFrame of `dataframe`, of the schema that the core caches, +/// which is of the server: its type objects, as PySpark converts the schema of the server, of a +/// new object of each user-defined type (see `types::schema_to_py`), and not of the objects that +/// the client was given. None when it has none, or when PySpark would not have one, as its +/// conversion of the schema raised (see [`check_python_schema`]). +fn known_schema_object(py: Python<'_>, dataframe: &DataFrame) -> PyResult>> { + let Some(schema) = dataframe.known_schema() else { + return Ok(None); + }; + if check_schema_conversion(py, Some(&schema)).is_err() { + return Ok(None); + } + match crate::types::schema_to_py(py, &schema) { + Ok(schema) => Ok(Some(schema)), + Err(err) if err.is_instance_of::(py) => Ok(None), + Err(err) => Err(err), + } +} + /// The result of a set operation of `this` and `other`, `dataframe`, which keeps the schema that /// PySpark's `_merge_cached_schema` keeps: `this.schema` when the cached schemas of both are /// equal, else none. The core keeps that of two cached structs; of a schema that a DataFrame /// keeps as a Python object, the schema of `to` that is not a DataType or the `StructType` of a /// map function, the objects are compared as they are now, as PySpark compares them with `==`, -/// and the result keeps a deep copy of that of `this`, as `this.schema` gives. Of a copy or an -/// unpickle, `this.schema` is the schema that it requested (see `copy_schema`). +/// with the type objects of the cached schema of the other, which of the server holds new objects +/// of its user-defined types (see `known_schema_object`), and the result keeps a deep copy of +/// that of `this`, as `this.schema` gives. Of a copy or an unpickle, `this.schema` is the schema +/// that it requested (see `copy_schema`). fn set_operation_result( py: Python<'_>, this: &PyDataFrame, @@ -7313,12 +7369,7 @@ fn set_operation_result( let cached = |df: &PyDataFrame| -> PyResult>> { match &df.schema_object { Some(schema) => Ok(Some(schema.clone_ref(py))), - None => df - .dataframe - .known_schema() - .filter(|schema| check_schema_conversion(py, Some(schema)).is_ok()) - .map(|schema| crate::types::data_type_to_py(py, &schema)) - .transpose(), + None => known_schema_object(py, &df.dataframe), } }; if let (Some(left), Some(right)) = (cached(this)?, cached(other)?) { diff --git a/crates/pyspark-rs/src/group.rs b/crates/pyspark-rs/src/group.rs index 55c80762..63c62f2f 100644 --- a/crates/pyspark-rs/src/group.rs +++ b/crates/pyspark-rs/src/group.rs @@ -226,6 +226,10 @@ impl PyGroupedData { }; names.push(name.to_cow().ok().map(|name| name.into_owned())); } + // As PySpark's `self._df.schema`, which raises the errors of its conversion of the schema + // of the server, such as the `TypeError` of a user-defined type whose `__init__` takes + // arguments. + self.frame.bind(py).borrow().check_schema_read(py)?; let requested = self.frame.bind(py).borrow().copy_schema(); let of_requested; let grouped = match requested { diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs index f7dab459..7336fd04 100644 --- a/crates/pyspark-rs/src/types.rs +++ b/crates/pyspark-rs/src/types.rs @@ -2058,46 +2058,12 @@ impl PyDataType { class_repr(slf.as_any()) } - /// As pyspark's `simpleString()`, the `typeName()` of a type of its class only. An array, map - /// or struct of a user-defined type names it by the `simpleString()` of its Python class, - /// such as `array`. - fn simpleString(slf: &Bound<'_, Self>) -> PyResult { - let py = slf.py(); - // A struct, array or map reads its live type, as its own `simpleString` does: the base - // method is also called on them directly, as `DataType.simpleString(st)`. - if slf.is_instance_of::() - || slf.is_instance_of::() - || slf.is_instance_of::() - { - return match py_to_data_type(slf.as_any()) { - Ok(data_type) => simple_string_with_udts(py, &data_type), - // Of a type that holds one that does not lower now, of the type objects that it - // holds, as PySpark's reads them (see `lenient_type`). - Err(error) if is_exception(py, &error) => { - if let Ok(array) = slf.cast::() { - return array.borrow().simple_string(py); - } - if let Ok(map) = slf.cast::() { - return map.borrow().simple_string(py); - } - slf.cast::()?.borrow().simple_string(py) - } - Err(error) => Err(error), - }; - } - let this = slf.borrow(); - let Some(inner) = &this.inner else { - drop(this); - return slf.call_method0(pyo3::intern!(py, "typeName"))?.extract(); - }; - match inner { - DataType::Array { .. } | DataType::Map { .. } | DataType::Struct { .. } - if has_udt(inner) => - { - simple_string_with_udts(py, inner) - } - _ => Ok(inner.simple_string()), - } + /// As PySpark's `DataType.simpleString`, `self.typeName()`. The types of a string of their + /// own, such as `bigint`, `decimal(10,2)` or `struct`, override it, as PySpark's + /// do, so the base method called on one of them directly, as `DataType.simpleString(st)`, is + /// its `typeName()`, `struct`, as in PySpark. + fn simpleString<'py>(slf: &Bound<'py, Self>) -> PyResult> { + type_name_of(slf.as_any()) } // Equality/hashing are defined once here and inherited by every concrete type class @@ -5945,6 +5911,16 @@ impl PyYearMonthIntervalType { const YEAR: i32 = 0; #[classattr] const MONTH: i32 = 1; + /// As PySpark's `simpleString`, its `_str_repr` of the fields: `interval year to month`, or + /// `interval year` of one field, also of a Python subclass, as it reads no `typeName()`. + #[pyo3(name = "simpleString")] + fn simple_string(&self) -> String { + DataType::YearMonthInterval { + start_field: self.start_field, + end_field: self.end_field, + } + .simple_string() + } // --- DataType object-model methods (v4.2.0 parity) --- /// As PySpark's `DataType.json`, of the `jsonValue` of the type, of a Python subclass its own. #[pyo3(name = "json")] @@ -6113,6 +6089,16 @@ impl PyDayTimeIntervalType { const MINUTE: i32 = 2; #[classattr] const SECOND: i32 = 3; + /// As PySpark's `simpleString`, its `_str_repr` of the fields: `interval day to second`, or + /// `interval day` of one field, also of a Python subclass, as it reads no `typeName()`. + #[pyo3(name = "simpleString")] + fn simple_string(&self) -> String { + DataType::DayTimeInterval { + start_field: self.start_field, + end_field: self.end_field, + } + .simple_string() + } // --- DataType object-model methods (v4.2.0 parity) --- /// As PySpark's `DataType.json`, of the `jsonValue` of the type, of a Python subclass its own. #[pyo3(name = "json")] diff --git a/dev/parity/known-differences.md b/dev/parity/known-differences.md index bfbbcba8..6db72892 100644 --- a/dev/parity/known-differences.md +++ b/dev/parity/known-differences.md @@ -134,6 +134,14 @@ difference is intentional or tracked for a fix. `TypeError: StructType fields must be StructField objects` there, and an assignment of `fields` of such an element raises it at once. Of a list of `StructField`s changed in place, the cached `names` and their uses are PySpark's (see `docs/compatibility.md`). +- **A schema that PySpark does not convert is requested once** (intentional; the errors are the + same). PySpark's `_schema` keeps the schema that it requested only once its conversion of it + succeeded, so of a schema that it does not convert, such as of a user-defined type whose + `__init__` takes arguments or of no Python class, of a nanosecond-precision timestamp, or of a + spatial type of an SRID that it does not take, each read of the schema (`schema`, `columns`, + `dtypes`, `printSchema` and the lookups that read the columns) requests it again, and raises. + This client keeps the schema of the server once it requested it, as of any other schema, and + raises the same error of its conversion at each read, of no other request. - **The `__repr__` of a Column of a str that UTF-8 cannot encode** (intentional; the expressions of the core hold UTF-8). Such a str, of a name, a SQL string or a literal, fails the plans of the Column when they are built, as in PySpark, and its lone surrogates show escaped in the `__repr__` diff --git a/docs/compatibility.md b/docs/compatibility.md index caf6041b..44ec8c65 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -2274,9 +2274,16 @@ that hold them, as `UserDefinedType`s of other parameters do, and `toNullable()` are. The schema of a DataFrame and of its rows, as the server gives it, holds a new object of each user-defined type, as PySpark's `UserDefinedType.fromJson` makes it of its class, and so does `fromJson`: not the object that the client was given, nor of its attributes, and a class whose -`__init__` takes arguments raises its `TypeError` there, as in PySpark. A struct, array or map that -the client makes of core types, without the objects of a user, holds the user-defined type of its -JSON that a type was last made of, if one was. `IntegralType()` is one instance, as PySpark's +`__init__` takes arguments raises its `TypeError` wherever PySpark converts that schema, as in +PySpark: the schema, `columns`, `dtypes`, `printSchema`, the rows, the lookups that read the +columns (`df[0]`, `dir(df)`, names of two underscores), the copies and the unpickles, the map +functions and the numeric aggregations of grouped data. The cached schema of a DataFrame holds the +objects that PySpark's `_cached_schema` holds, those of `to` or of a map function, else the new +objects of the schema of the server: a set operation compares them as PySpark's does, by the +`__dict__` of their user-defined types, and a pickle carries them, which the unpickled DataFrame +keeps as its cached schema. `DataType.simpleString(t)`, the method of the base class, is +`t.typeName()`, as PySpark's, which the types of a string of their own, such as `bigint` or +`struct`, override. `IntegralType()` is one instance, as PySpark's `DataTypeSingleton` makes it, and so is each Python subclass of it, of an instance of its own, and a Python subclass of the other classes, such as a `UserDefinedType` of parameters, takes the arguments of its `__init__`. @@ -2881,13 +2888,13 @@ session fails the copy with `SparkConnectException`, as PySpark's does. One diff PySpark's `DataFrame.__getattr__` answers `getattr(df, "__deepcopy__")` and `getattr(df, "__copy__")` by requesting the schema, to read the columns, and raises `ATTRIBUTE_NOT_SUPPORTED`, so `hasattr(df, "__copy__")` is false; the drop-in defines both -itself, which it finds without a request. An unpickled DataFrame requests its -schema once, at the unpickle, which raises the error of the request, as PySpark's pickle looks the +itself, which it finds without a request. An unpickled DataFrame requests its schema once, at the +unpickle, which raises the error of the request or of its conversion, as PySpark's pickle looks the `__setstate__` of the new DataFrame up through its `__getattr__`. Its cached schema is then that of -the pickled DataFrame, as the state of PySpark's pickle sets it: of none, its `schema` requests it -again. So does the unpickle of an object that PySpark pickles with its DataFrame: a writer, of its -callback, grouped data, `na` and `stat`; a stream writer, which PySpark's pickles of the plan of -the DataFrame, requests nothing. +the pickled DataFrame, as the state of PySpark's pickle sets it, of the objects that it carries: of +none, its `schema` requests it again. So does the unpickle of an object that PySpark pickles with +its DataFrame: a writer, of its callback, grouped data, `na` and `stat`; a stream writer, which +PySpark's pickles of the plan of the DataFrame, requests nothing. A copy, a deep copy or an unpickle of a DataFrame keeps the schema that it requests apart from the cached schema of the state, as PySpark's keeps it as the `_cached_schema_serialized` that its diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py index 703a3c4c..9b9dd0c3 100644 --- a/python/tests/test_dropin_e2e.py +++ b/python/tests/test_dropin_e2e.py @@ -936,8 +936,10 @@ def test_nanosecond_timestamps_are_not_collected_as_pyspark(session): # As PySpark's conversion of the schema from its proto: a TIMESTAMP_NTZ(p) or TIMESTAMP_LTZ(p) # with p > 6, also nested, raises UNSUPPORTED_OPERATION from every collecting API, toPandas # and toArrow, and from every schema surface (the schema, columns, dtypes, printSchema, repr - # and json, and the DDL strings of the types), while count, show and the microsecond - # timestamps keep working. + # and json, the copies, whose lookups read the columns of the copy, the numeric aggregations + # of grouped data, which read `self._df.schema`, and the DDL strings of the types), while + # count, show and the microsecond timestamps keep working. + import copy import datetime import re @@ -988,6 +990,9 @@ def message(proto_text): lambda d: d.printSchema(), lambda d: repr(d), lambda d: d.schema.json(), + copy.copy, + copy.deepcopy, + lambda d: d.groupBy().sum(), ): with pytest.raises(PySparkValueError, match=message(proto_text)): surface(df) @@ -4922,6 +4927,75 @@ def test_schemas_of_the_server_make_new_user_defined_type_objects_as_pyspark(ses assert own["p"].dataType is marked and ArrayType(marked).elementType is marked +def test_cached_schemas_of_set_operations_and_pickles_hold_the_objects_of_pyspark(session): + # As official PySpark 4.3.0, whose cached schema of a DataFrame (`_cached_schema`) is the + # object of `to`, or the type objects of its conversion of the schema of the server, of new + # user-defined type objects. A set operation keeps the cached schema of equal ones + # (`_merge_cached_schema`), which `==` compares by the `__dict__` of their user-defined types: + # that of `to` of a marked object differs from the server's, so their union caches none, and + # its schema is the server's. A pickle carries the cached schema as its objects, which the + # methods that copy the cached schema of the unpickled DataFrame keep, such as `filter`, while + # its own `schema` is the one that its unpickle requested. The client gave instead the objects + # of their JSON that it lowered last. + from pyspark import cloudpickle + + marked = _MarkedPointUDT() + marked.marker = "made" + server = session.createDataFrame( + [((1.0, 2.0),)], StructType([StructField("p", _MarkedPointUDT())]) + ) + assert server.columns == ["p"] + cast = server.to(StructType([StructField("p", marked)])) + for union in (cast.union(server), server.union(cast), cast.intersect(server)): + assert not hasattr(union.schema["p"].dataType, "marker") + assert cast.union(cast).schema["p"].dataType.marker == "made" + unpickled = cloudpickle.loads(cloudpickle.dumps(cast)) + assert not hasattr(unpickled.schema["p"].dataType, "marker") + assert unpickled.filter("true").schema["p"].dataType.marker == "made" + assert unpickled.columns == ["p"] and unpickled.dtypes == [("p", "udt")] + unpickled = cloudpickle.loads(cloudpickle.dumps(server)) + assert not hasattr(unpickled.filter("true").schema["p"].dataType, "marker") + assert unpickled.collect() == [Row(p=(1.0, 2.0))] + + +def test_reads_of_a_schema_of_a_user_defined_type_of_arguments_raise_as_pyspark(session): + # As official PySpark 4.3.0, whose reads of the schema of a DataFrame convert the schema of the + # server (`_schema`) with `UserDefinedType.fromJson`, which calls the class of no arguments: + # of a class whose __init__ takes arguments, `columns` and what reads them (printSchema, the + # index of a column, `dir`, the lookups of names of two underscores, the copies, of which + # pickle and copy look `__setstate__` up so, and the map functions), and the numeric + # aggregations of grouped data, which read `self._df.schema`, raise its TypeError. The uses + # of the plan that do not read the schema work. + import copy + + from pyspark import cloudpickle + + df = session.createDataFrame( + [((1.0, 2.0), 1)], + StructType([StructField("p", _ArgPointUDT(2)), StructField("i", LongType())]), + ) + for use in ( + lambda: df.columns, + df.printSchema, + lambda: df[0], + lambda: dir(df), + lambda: hasattr(df, "__array__"), + lambda: df.select("*").columns, + lambda: copy.copy(df), + lambda: copy.deepcopy(df), + lambda: cloudpickle.loads(cloudpickle.dumps(df)), + lambda: df.groupBy("i").sum(), + lambda: df.groupBy("i").avg("i"), + lambda: df.groupBy("i").applyInPandas(lambda pdf: pdf, "i long"), + lambda: df.mapInPandas(lambda batches: batches, "i long"), + ): + with pytest.raises(TypeError, match="missing 1 required positional argument"): + use() + assert df.count() == 1 + assert df.select("i").columns == ["i"] + assert df.groupBy("i").count().collect() == [Row(i=1, count=1)] + + def test_uses_of_the_json_of_a_type_of_a_user_defined_type_of_no_json_raise_its_error(session): # As official PySpark 4.3.0, whose struct, array and map of a UserDefinedType whose module() # is not implemented are made (see test_dropin_types_columns_offline.py): the uses that read diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 4d5cf660..269a1343 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -5618,17 +5618,65 @@ def test_structtype_add_reflected_in_json(): def test_structtype_add_reflected_in_the_base_data_type(): # Regression: add() must update the base DataType too, which the schemas of - # mapInArrow/mapInPandas/applyInPandas/... read (via resolve_datatype). The base - # implementation of simpleString reads it. + # mapInArrow/mapInPandas/applyInPandas/... read (via resolve_datatype), and so its JSON. built = T.StructType([T.StructField("a", T.LongType()), T.StructField("b", T.LongType())]) chained = T.StructType().add("a", T.LongType()).add("b", T.LongType()) st = T.StructType([T.StructField("a", T.LongType())]) st.add("b", T.LongType()) for added in (chained, st): - assert T.DataType.simpleString(added) == "struct" + assert added.simpleString() == "struct" + assert added.json() == built.json() assert str(added) == str(built) == repr(built) +@pytest.mark.parametrize("data_type,type_name", [ + (T.ByteType(), "byte"), + (T.IntegerType(), "integer"), + (T.LongType(), "long"), + (T.DecimalType(10, 2), "decimal"), + (T.StringType("UTF8_LCASE"), "string"), + (T.CharType(3), "char"), + (T.VarcharType(3), "varchar"), + (T.TimeType(6), "time"), + (T.YearMonthIntervalType(), "yearmonthinterval"), + (T.DayTimeIntervalType(0, 3), "daytimeinterval"), + (T.GeometryType(4326), "geometry"), + (T.ArrayType(T.IntegerType()), "array"), + (T.MapType(T.StringType(), T.LongType()), "map"), + (T.StructType([T.StructField("a", T.LongType())]), "struct"), + (T.StructType([T.StructField("a", _DayPointUDT())]), "struct"), + (T.CalendarIntervalType(), "interval"), + (T.TimestampNTZType(), "timestamp_ntz"), + (T.NullType(), "void"), + (T.AtomicType(), "atomic"), +]) +def test_the_base_simple_string_of_a_type_is_its_type_name_as_pysparks(data_type, type_name): + # As PySpark's DataType.simpleString, `self.typeName()`, which the types of a string of + # their own override: the base method called on one of them directly is its typeName(), as + # official PySpark 4.3.0's. The drop-in's gave the simple string of the type, of the + # user-defined type objects of its JSON that it had lowered last. + assert T.DataType.simpleString(data_type) == type_name == data_type.typeName() + + +@pytest.mark.parametrize("base,simple", [ + (T.DoubleType, "renamed"), + (T.LongType, "bigint"), + (T.DecimalType, "decimal(10,0)"), + (T.YearMonthIntervalType, "interval year to month"), + (T.DayTimeIntervalType, "interval day to second"), +]) +def test_the_simple_strings_of_subclasses_of_their_own_type_name_are_pysparks(base, simple): + # As official PySpark 4.3.0: of a Python subclass of another typeName(), the base + # simpleString is that name, and the simpleString of its class is that of PySpark's class, + # which reads the typeName() of the type of its class only, as DoubleType's, and not of the + # interval types, whose own simpleString is of their fields. + subclass = type("Renamed" + base.__name__, (base,), { + "typeName": classmethod(lambda cls: "renamed")}) + data_type = subclass() + assert T.DataType.simpleString(data_type) == "renamed" + assert data_type.simpleString() == simple + + @pytest.mark.parametrize("data_type,expected", [ (T.LongType(), "LongType()"), (T.TimestampNTZType(), "TimestampNTZType()"), @@ -23348,6 +23396,91 @@ def column_names(df): assert column_names(spark.createDataFrame(pd.DataFrame({"a": [1]}), popped)) == ["a", "b"] +def _server_schema_of(*fields): + # The schema that the server sends of the fields (name, type), of LongType or of a + # user-defined type of a LongType SQL type: DataType.struct=21, Struct.fields=1, + # StructField: name=1, data_type=2, nullable=3; DataType.long=7, and DataType.udt=23 -> + # UDT: type=1, python_class=3, sql_type=5. + def proto(data_type): + if isinstance(data_type, T.UserDefinedType): + python_class = f"{data_type.module()}.{type(data_type).__name__}" + return message_field(23, message_field(1, "udt") + message_field(3, python_class) + + message_field(5, message_field(7, b""))) + return message_field(7, b"") + + return message_field(21, b"".join( + message_field(1, message_field(1, name) + message_field(2, proto(data_type)) + + varint_field(3, 1)) + for name, data_type in fields)) + + +def test_reads_of_a_schema_of_a_udt_of_constructor_arguments_raise_as_pysparks(mock_connect): + # As PySpark's reads of the schema of a DataFrame, which convert the schema of the server + # (`_schema`) with UserDefinedType.fromJson, which calls the class of no arguments: of a + # class whose __init__ needs one, `columns` and what reads them (printSchema, the index of a + # column, `dir`, the lookups of names of two underscores, and the copies and an unpickle, + # which look `__setstate__` up so), and the numeric aggregations of grouped data, which read + # `self._df.schema`, raise its TypeError, as official PySpark 4.3.0 does (see + # test_dropin_e2e.py). The drop-in read the names of the columns of its core type. + import copy + + from pyspark import cloudpickle + + spark, service = mock_connect + service.data_type = _server_schema_of(("p", _OffsetUDT(1)), ("i", T.LongType())) + df = spark.range(1) + for use in ( + lambda: df.columns, + df.printSchema, + lambda: df[0], + lambda: dir(df), + lambda: hasattr(df, "__array__"), + lambda: df._ipython_key_completions_(), + lambda: copy.copy(df), + lambda: copy.deepcopy(df), + lambda: cloudpickle.loads(cloudpickle.dumps(df)), + lambda: df.groupBy("i").sum(), + lambda: df.groupBy("i").max("i"), + ): + with pytest.raises(TypeError, match="missing 1 required positional argument"): + use() + # Of a class of a default, a new object of it. + service.data_type = _server_schema_of(("p", _ScaleUDT(3)), ("i", T.LongType())) + df = spark.range(1) + assert df.columns == ["p", "i"] and df.schema["p"].dataType.scale == 1 + assert hasattr(copy.copy(df), "columns") and df.groupBy("i").sum() is not None + + +def test_cached_schemas_of_set_operations_and_pickles_are_pysparks_objects(mock_connect): + # As PySpark's `_cached_schema`, the object of `to`, or the type objects of its conversion of + # the schema of the server, of new user-defined type objects: a set operation keeps the cached + # schema of `this` when both are equal by `==` (`_merge_cached_schema`), which compares the + # `__dict__` of user-defined types, so that of `to` of another scale and the server's differ, + # and their union has none, whose schema is the server's. A pickle carries the cached schema + # as its objects, which `filter` of the unpickled DataFrame keeps, while its own `schema` is + # the one that its unpickle requested; that of a DataFrame of the server's schema is of new + # objects. As official PySpark 4.3.0 (see test_dropin_e2e.py); the drop-in gave the objects + # of their JSON that it lowered last. + from pyspark import cloudpickle + + spark, service = mock_connect + service.data_type = _server_schema_of(("p", _ScaleUDT())) + server = spark.range(1) + assert server.columns == ["p"] + cast = server.to(T.StructType([T.StructField("p", _ScaleUDT(2))])) + for union in (cast.union(server), server.union(cast), cast.exceptAll(server)): + assert union.schema["p"].dataType.scale == 1 + assert cast.union(cast).schema["p"].dataType.scale == 2 + unpickled = cloudpickle.loads(cloudpickle.dumps(cast)) + assert unpickled.schema["p"].dataType.scale == 1 + assert unpickled.filter("true").schema["p"].dataType.scale == 2 + assert unpickled.union(cast).schema["p"].dataType.scale == 1 + unpickled = cloudpickle.loads(cloudpickle.dumps(server)) + service.data_type = _server_schema_of(("q", T.LongType())) + assert unpickled.columns == ["p"] + assert unpickled.filter("true").schema["p"].dataType.scale == 1 + + class _ScaledValueUDT(T.UserDefinedType): """A UserDefinedType of a scale, of the values of `_ScaledValue`.""" @@ -34339,10 +34472,13 @@ def test_schema_tree_type_names_and_escaped_field_names(): "python_class, expected", [ ("pyspark.ml.linalg.VectorUDT", "vectorudt"), - (None, "udt"), + (None, None), ], ) def test_print_schema_udt_class_name(mock_connect, capsys, python_class, expected): + # As PySpark's printSchema, of the schema that its `_schema` converts, whose + # UserDefinedType.fromJson makes the user-defined type of its Python class: of a type of no + # Python class, it raises the KeyError of `pyClass`, as `columns` does. spark, service = mock_connect udt = message_field(1, "udt") if python_class is not None: @@ -34350,9 +34486,16 @@ def test_print_schema_udt_class_name(mock_connect, capsys, python_class, expecte service.data_type = message_field( 21, message_field(1, _print_schema_field("features", message_field(23, udt))) ) - assert spark.range(10).printSchema() is None - assert capsys.readouterr().out == f"root\n |-- features: {expected} (nullable = true)\n\n" - assert service.calls == 1 + if expected is None: + df = spark.range(10) + for read in (df.printSchema, lambda: df.columns): + with pytest.raises(KeyError, match="pyClass"): + read() + assert capsys.readouterr().out == "" + else: + assert spark.range(10).printSchema() is None + assert capsys.readouterr().out == f"root\n |-- features: {expected} (nullable = true)\n\n" + assert service.calls == 1 assert service.execute_calls == 0