diff --git a/crates/pyspark-rs/src/column.rs b/crates/pyspark-rs/src/column.rs index f29bcc63..b1fc9a77 100644 --- a/crates/pyspark-rs/src/column.rs +++ b/crates/pyspark-rs/src/column.rs @@ -191,6 +191,27 @@ impl PyColumnExpression { self.column = Column::new(spark_connect::expression::Expression::column("")); } } + + /// An expression pickles, as PySpark's expression, a Python object, pickles its fields: as the + /// `_expr` of its Column, `getattr(column, "_expr")`, of the pickle of the Column (see + /// `PyColumn::__reduce__`), so that the unpickled expression is that of the unpickled Column, + /// also on a Python worker of the server, where it is the expression of PySpark Connect's + /// Column. + fn __reduce__<'py>(&self, py: Python<'py>) -> PyResult> { + let column = Py::new(py, PyColumn::new(self.column.clone()))?; + let args = PyTuple::new( + py, + [ + column.into_any().into_bound(py), + pyo3::types::PyString::new(py, "_expr").into_any(), + ], + )?; + PyTuple::new( + py, + [py.import("builtins")?.getattr("getattr")?, args.into_any()], + ) + } + /// As PySpark's `__repr__` of the expression, which raises its error of an expression that it /// does not render (a `ColumnAlias` of names that are not all `str`). fn __repr__(&self) -> PyResult { @@ -930,7 +951,10 @@ impl PyColumn { /// that `pickle` keeps it, as it does of PySpark's Column. The function of a UDF of the /// Column is pickled now, into the command of the expression, as PySpark's pickle of the /// Column pickles the function with it; one that does not pickle raises the error of its - /// pickling. + /// pickling. A Column of subqueries pickles the relations of the plans that they refer to + /// too, of their plan IDs, after the expression, as PySpark's pickle keeps the plan objects of + /// its subqueries: the unpickled Column sends them in the `WithRelations` of the relation that + /// holds it, as PySpark's plan sends the plans of its subqueries. fn __reduce__<'py>(slf: &Bound<'py, Self>, py: Python<'py>) -> PyResult> { let this = slf.borrow(); // Of an expression that a request fails to build, as PySpark's Column of an alias of a @@ -948,10 +972,34 @@ impl PyColumn { { return Err(exception); } - let build_error = unbuildable + let mut build_error = unbuildable .as_ref() .map(|error| error_description(py, error)) .transpose()?; + // A plan of a subquery that does not build pickles of its error, as a DataFrame does, + // which the requests of the unpickled Column raise; one that does not pickle, or that is + // too deep, raises its error. A CASE WHEN and a sort order are given back of their parts, + // which pickle their subqueries. + let mut subqueries = Vec::new(); + let of_parts = matches!( + this.column.expression().node(), + ExpressionNode::CaseWhen(_) | ExpressionNode::SortOrder(_) + ); + if build_error.is_none() && !of_parts { + match crate::origin::pickling(|| this.column.expression().subquery_relations()) { + Ok(relations) => { + for (relation, levels) in relations { + let relation = + pyo3::types::PyBytes::new(py, &crate::encode_on_grown_stack(relation)); + subqueries.push((relation, levels)); + } + } + Err(error) => match described_error(py, &error)? { + Some(description) => build_error = Some(description), + None => return Err(crate::errors::spark_error_to_py_exception(error)), + }, + } + } let repr_error = this .column .expression() @@ -999,14 +1047,17 @@ impl PyColumn { levels, build_error, repr_error, + subqueries, ))?; crate::group::reduce_with_state(&reduce, slf.as_any()) } /// The Column of `expression`, the bytes of a `spark.connect.Expression`, and of its /// `rendered` rendering, of its `levels`, as a pickled Column unpickles: it sends the - /// expression as it is. The CASE WHEN of `branches` and `else_value`, the Columns of its - /// parts, is built as it was. + /// expression as it is, and the relations of `subqueries`, the bytes of the relation of each + /// plan that its subqueries refer to and its levels, in the `WithRelations` of the relation + /// that holds it. The CASE WHEN of `branches` and `else_value`, the Columns of its parts, is + /// built as it was. #[staticmethod] #[pyo3(signature = ( expression, @@ -1015,7 +1066,8 @@ impl PyColumn { else_value=None, levels=1, build_error=None, - repr_error=None + repr_error=None, + subqueries=None ))] #[allow(clippy::too_many_arguments)] fn _of_expression( @@ -1026,6 +1078,7 @@ impl PyColumn { levels: usize, build_error: Option<&Bound<'_, PyAny>>, repr_error: Option<&Bound<'_, PyAny>>, + subqueries: Option, usize)>>, ) -> PyResult { if let Some(branches) = branches { let branches = branches @@ -1048,12 +1101,25 @@ impl PyColumn { ))); } use prost::Message; + let subqueries: Vec<(&[u8], usize)> = subqueries + .iter() + .flatten() + .map(|(relation, levels)| (relation.as_bytes(), *levels)) + .collect(); // Decoded, and then built or dropped, on a stack that holds its levels. // The errors become Python exceptions on the caller's stack, which runs no Python code // on the grown one. let column = crate::on_grown_stack(|| { - spark_connect_proto::Expression::decode(expression) - .map(|expression| Column::sent(expression, rendered, levels)) + let relations = subqueries + .iter() + .map(|(relation, levels)| { + spark_connect_proto::Relation::decode(*relation) + .map(|relation| (relation, *levels)) + }) + .collect::, _>>()?; + spark_connect_proto::Expression::decode(expression).map(|expression| { + Column::sent_with_subqueries(expression, rendered, levels, relations) + }) }) .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))? .to_pyerr()?; diff --git a/crates/pyspark-rs/src/dataframe.rs b/crates/pyspark-rs/src/dataframe.rs index ecbb9775..3cc7c2f1 100644 --- a/crates/pyspark-rs/src/dataframe.rs +++ b/crates/pyspark-rs/src/dataframe.rs @@ -155,7 +155,10 @@ impl PyDataFrame { /// first plan of the session reads the plan compression settings then; then /// `SparkConnectClient.to_pandas` reads the settings of the conversion, in one request, /// and sends the plan; the Arrow result with the schema that the server sends is converted - /// by `pyspark.sql.connect.client._to_pandas`. + /// by `pyspark.sql.connect.client._to_pandas`. The DataFrame keeps the information of the + /// execution once the result is converted, as `_to_pandas` sets `_execution_info` after + /// `SparkConnectClient.to_pandas` returns: a conversion that fails leaves `executionInfo` as + /// it was. pub(crate) fn to_pandas( &self, py: Python<'_>, @@ -172,9 +175,9 @@ impl PyDataFrame { "spark.sql.execution.arrow.pyspark.selfDestruct.enabled", ]) })?; - let (batches, schema) = interruptible(py, &session, || { + let (batches, schema, info) = interruptible(py, &session, || { execution - .collect_record_batches_checked(check_schema_at_receipt) + .collect_record_batches_and_info(check_schema_at_receipt) .map_err(Box::new) })?; check_schema_conversion(py, schema.as_ref())?; @@ -191,13 +194,7 @@ impl PyDataFrame { Some(schema) => crate::types::schema_to_py(py, schema)?, None => py.None(), }; - let (metrics, observed) = match self.dataframe.execution_info() { - Ok(info) => execution_metrics(py, &info)?, - Err(_) => ( - pyo3::types::PyList::empty(py), - pyo3::types::PyList::empty(py), - ), - }; + let (metrics, observed) = execution_metrics(py, &info)?; let kwargs = pyo3::types::PyDict::new(py); kwargs.set_item("timezone", &settings[0])?; match struct_handling_mode { @@ -210,6 +207,7 @@ impl PyDataFrame { (table, schema, metrics, observed), Some(&kwargs), )?; + self.dataframe.keep_execution_info(info); Ok(pdf.unbind()) } @@ -4638,8 +4636,11 @@ type MergeClause = ( /// which asserts a str and sets it in a string field, and then its value, whose `to_plan` an /// object that is not a Column lacks; and then the condition. A Column that does not build raises /// its error there, such as a str that UTF-8 cannot encode, or the `RecursionError` of PySpark's -/// serializer of one too deep to be sent, and the clause is not added. The merge builds them again -/// when it runs. The assignments keep their order, in which the merge sends them. +/// serializer of one too deep to be sent, and the clause is not added. The clause keeps them as +/// they were built then ([`built_column`]), which the merge sends, as PySpark's merge sends the +/// protos that its clause built: so the functions of their UDFs are pickled once, when the clause +/// is added, and the warning of a window of no partitions is given then. The assignments keep +/// their order, in which the merge sends them. fn merge_clause( py: Python<'_>, condition: &ClauseCondition, @@ -4660,25 +4661,30 @@ fn merge_clause( return Err(not_column(&value, "value")); } }; - value.try_to_proto().to_pyerr()?; - built.push((key, value)); + built.push((key, built_column(&value)?)); } } let condition = match condition { ClauseCondition::None => None, - ClauseCondition::Column(column) => Some(column.clone()), + ClauseCondition::Column(column) => Some(built_column(column)?), ClauseCondition::Object(object) => { let object = object.bind(py); object.getattr("to_plan")?; return Err(not_column(object, "condition")); } }; - if let Some(condition) = &condition { - condition.try_to_proto().to_pyerr()?; - } Ok((condition, built)) } +/// The Column of the `spark.connect.Expression` of `column` as a request builds it now, of its +/// rendering and its levels, which sends that expression as it is (`Column::sent`), as the +/// `MergeAction` that PySpark's `_build_merge_action` builds holds the protos of its Columns; or +/// the error of the build. +fn built_column(column: &spark_connect::column::Column) -> PyResult { + let (expression, levels) = column.try_to_proto_with_levels().to_pyerr()?; + spark_connect::column::Column::sent(expression, column.expression().render(), levels).to_pyerr() +} + #[pymethods] impl PyMergeIntoWriter { /// The shallow copy of a merge builder, as PySpark's `copy.copy`: the copy shares the state diff --git a/crates/pyspark-rs/src/origin.rs b/crates/pyspark-rs/src/origin.rs index 4aa5343b..414cd9b7 100644 --- a/crates/pyspark-rs/src/origin.rs +++ b/crates/pyspark-rs/src/origin.rs @@ -67,7 +67,7 @@ impl spark_connect::plan_build::PlanBuildListener for PlanBuild { } fn window_without_partitions(&self) -> spark_connect::Result<()> { - if PICKLING.with(std::cell::Cell::get) { + if is_pickling() { return Ok(()); } Python::attach(|py| { @@ -95,6 +95,12 @@ thread_local! { static PICKLING: std::cell::Cell = const { std::cell::Cell::new(false) }; } +/// Whether this thread builds the plan of a DataFrame, a Column or a table argument to pickle it +/// ([`pickling`]). +pub(crate) fn is_pickling() -> bool { + PICKLING.with(std::cell::Cell::get) +} + /// Run `pickle`, which builds the plan of a DataFrame or a Column to pickle it, without the /// warning of a window of no partitions: PySpark's pickle keeps its plan objects, and builds no /// plan. diff --git a/crates/pyspark-rs/src/udf_command.rs b/crates/pyspark-rs/src/udf_command.rs index 225871ed..639d0b13 100644 --- a/crates/pyspark-rs/src/udf_command.rs +++ b/crates/pyspark-rs/src/udf_command.rs @@ -71,34 +71,55 @@ impl PythonCommand { /// Of a UDTF, the `PythonUdtf` message that the function returns; of a UDF, the `PythonUdf` /// message of its output type, converted first, and of the pickle of the object, as PySpark's /// `PythonUDF.to_plan` converts `pyspark_types_to_proto_types(output_type)` before it pickles - /// `(func, output_type)`, of the type as it is then; of a data type, its message. + /// `(func, output_type)`, of the type as it is then; of a data type, its message. Its error is + /// that of [`pickle_error`]. fn pickle(&self) -> spark_connect_core::error::Result> { - Python::attach(|py| { - let object = self.object.bind(py); - match self.kind { - CommandKind::Udtf => command_of(&object.call0()?), - CommandKind::Udf => { - use prost::Message; - let output_type = - crate::types::py_to_proto_data_type(&object.get_item(1)?)?.to_proto(); - let command = crate::dataframe::py_cloudpickle(py, object)?; - Ok(spark_connect_proto::PythonUdf { - output_type: Some(output_type), - command: command.into(), - ..Default::default() - } - .encode_to_vec()) - } - CommandKind::DataType => { - use prost::Message; - Ok(crate::types::py_to_proto_data_type(object)? - .to_proto() - .encode_to_vec()) + Python::attach(|py| self.pickled(py).map_err(|error| pickle_error(py, error))) + } + + fn pickled(&self, py: Python<'_>) -> PyResult> { + let object = self.object.bind(py); + match self.kind { + CommandKind::Udtf => command_of(&object.call0()?), + CommandKind::Udf => { + use prost::Message; + let output_type = + crate::types::py_to_proto_data_type(&object.get_item(1)?)?.to_proto(); + let command = crate::dataframe::py_cloudpickle(py, object)?; + Ok(spark_connect_proto::PythonUdf { + output_type: Some(output_type), + command: command.into(), + ..Default::default() } + .encode_to_vec()) + } + CommandKind::DataType => { + use prost::Message; + Ok(crate::types::py_to_proto_data_type(object)? + .to_proto() + .encode_to_vec()) } - }) - .map_err(crate::errors::callback_error) + } + } +} + +/// The error of a request of `exception`, which the pickling of a command raised: the exception +/// itself ([`crate::errors::callback_error`]). But while a DataFrame, a Column or a table argument +/// is pickled ([`crate::origin::pickling`]), the pickle keeps an error of one of PySpark's error +/// classes other than a `PySparkPicklingError`, which the conversion of a type raises before the +/// function is pickled, such as the `UNSUPPORTED_OPERATION` of a type that has no proto, as the +/// error of its plan (see `crate::column::described_error`): PySpark's pickle keeps the type +/// object as it is, and the conversion fails at the requests of the unpickled object, or at the +/// unpickle of a DataFrame, which builds its plan. The pickle raises the error of a pickling. +fn pickle_error(py: Python<'_>, exception: PyErr) -> spark_connect_core::error::SparkError { + if crate::origin::is_pickling() { + if let Ok(Some(error)) = crate::errors::pyspark_class_error(py, &exception) { + if error.kind != spark_connect_core::error::SparkErrorKind::PicklingError { + return error; + } + } } + crate::errors::callback_error(exception) } /// The pickler of `udtf`, a Python function of no arguments that returns the `PythonUDTF` diff --git a/crates/spark-connect/src/column.rs b/crates/spark-connect/src/column.rs index 41fdf7cc..ed6d0994 100644 --- a/crates/spark-connect/src/column.rs +++ b/crates/spark-connect/src/column.rs @@ -458,6 +458,22 @@ impl Column { Ok(Column::new(Expression::sent(expression, rendered, levels)?)) } + /// As [`Column::sent`], of an expression whose subqueries refer to the plans of the relations + /// of `subqueries`, each of its levels, which a pickled Column keeps + /// ([`Expression::subquery_relations`]): the relation that holds the Column sends them (see + /// [`Expression::sent_with_subqueries`]). Not a stable API. + #[doc(hidden)] + pub fn sent_with_subqueries( + expression: proto::Expression, + rendered: String, + levels: usize, + subqueries: Vec<(proto::Relation, usize)>, + ) -> spark_connect_core::error::Result { + Ok(Column::new(Expression::sent_with_subqueries( + expression, rendered, levels, subqueries, + )?)) + } + /// Mirrors `pyspark.sql.column.Column.over` - window function application. pub fn over(self, window_spec: WindowSpec) -> Column { // Convert window frame spec if present diff --git a/crates/spark-connect/src/dataframe.rs b/crates/spark-connect/src/dataframe.rs index f1307cfb..f2ec88e5 100644 --- a/crates/spark-connect/src/dataframe.rs +++ b/crates/spark-connect/src/dataframe.rs @@ -127,12 +127,21 @@ pub struct PreparedExecution { impl PreparedExecution { /// Send the execution and collect its Arrow batches, as - /// [`DataFrame::collect_record_batches_with_schema_checked`] does. - pub fn collect_record_batches_checked( + /// [`DataFrame::collect_record_batches_with_schema_checked`] does, and the information of the + /// execution, which the session records as its most recent one, and which the DataFrame keeps + /// once [`DataFrame::keep_execution_info`] is given it: PySpark's `toPandas` sets the + /// `_execution_info` of the DataFrame once it has converted the result, so a conversion that + /// fails leaves it as it was. + pub fn collect_record_batches_and_info( self, check: impl Fn(&DataType) -> Result<()>, - ) -> Result<(Vec, Option)> { - self.dataframe.record_batches_of(self.request, check) + ) -> Result<( + Vec, + Option, + ExecutionInfo, + )> { + self.dataframe + .record_batches_of(self.request, check, KeptBy::Caller) } /// Send the execution and stream its rows, as [`DataFrame::to_local_iterator`] does. @@ -142,6 +151,14 @@ impl PreparedExecution { } } +/// Who keeps the information of an execution as that of the last execution of its DataFrame: the +/// DataFrame, when the execution ends, or the caller, with [`DataFrame::keep_execution_info`]. +#[derive(Clone, Copy)] +enum KeptBy { + DataFrame, + Caller, +} + /// Returned by `DataFrame::to_local_iterator()`, this iterator consumes the ExecutePlan /// response stream incrementally and yields Row objects without buffering the entire result. pub struct LocalRowIterator { @@ -1296,16 +1313,23 @@ impl DataFrame { check: impl Fn(&DataType) -> Result<()>, ) -> Result<(Vec, Option)> { let request = self.build_execute_request()?; - self.record_batches_of(request, check) + self.record_batches_of(request, check, KeptBy::DataFrame) + .map(|(batches, schema, _)| (batches, schema)) } /// The Arrow batches of the execution of `request`, a request of this DataFrame, as - /// [`DataFrame::collect_record_batches_with_schema_checked`] collects them. + /// [`DataFrame::collect_record_batches_with_schema_checked`] collects them, and the + /// information of the execution, which the session records and `kept_by` keeps. fn record_batches_of( &self, request: proto::ExecutePlanRequest, check: impl Fn(&DataType) -> Result<()>, - ) -> Result<(Vec, Option)> { + kept_by: KeptBy, + ) -> Result<( + Vec, + Option, + ExecutionInfo, + )> { let observations = self.plan.observations(); let mut stream = self.session.execute_plan(request)?; @@ -1334,8 +1358,19 @@ impl DataFrame { relaxed = read.or(relaxed); } } - self.record_execution(info); - Ok((share_schema(batches, relaxed)?, schema)) + match kept_by { + KeptBy::DataFrame => self.record_execution(info.clone()), + KeptBy::Caller => self.session.record_execution(info.clone()), + } + Ok((share_schema(batches, relaxed)?, schema, info)) + } + + /// Keep `info`, the information of an execution of this DataFrame, as the information of its + /// last execution ([`DataFrame::execution_info`]), of + /// [`PreparedExecution::collect_record_batches_and_info`]. Not a stable API. + #[doc(hidden)] + pub fn keep_execution_info(&self, info: ExecutionInfo) { + *self.execution.lock().unwrap() = Some(info); } /// Get the count of rows. diff --git a/crates/spark-connect/src/expression.rs b/crates/spark-connect/src/expression.rs index b6cc33f8..178e5123 100644 --- a/crates/spark-connect/src/expression.rs +++ b/crates/spark-connect/src/expression.rs @@ -77,10 +77,19 @@ thread_local! { /// recursion of the build, so the Python code of a pickling runs where PySpark's does; and when /// none fails and no expression failed after them, the second build takes their commands, in the /// same order. A build is deterministic, so the two meet the same UDFs. +/// +/// PySpark's `WindowExpression.to_plan` warns of a window of no partitions as it builds it, in +/// the same order: the first build warns of one that comes before the first pickler as it meets +/// it, and of one after a pickler in its turn among the picklings, so a pickling that fails before +/// it raises its error without the warning, and the error of the warning (of an `"error"` filter +/// of the warnings) is raised before the picklings after it. The second build warns no more. #[derive(Default)] struct UdfCommands { /// The picklers that the first build met, in the order of the build. met: Vec, + /// The windows of no partitions that the first build met after a pickler, each the number of + /// picklers that it met before it, of which [`pickle_in_turn`] warns in their turn. + windows: Vec, /// The commands that they gave, which the second build takes in turn. pickled: std::collections::VecDeque>, /// Whether this is the second build, which takes `pickled`. @@ -91,6 +100,7 @@ impl UdfCommands { const fn new() -> Self { UdfCommands { met: Vec::new(), + windows: Vec::new(), pickled: std::collections::VecDeque::new(), pickled_build: false, } @@ -119,22 +129,64 @@ pub(crate) fn pickled_command(pickler: &crate::udf::CommandPickler) -> Vec { }) } -/// Whether this thread is in the second build of the plan of a request, which takes the commands -/// that the first build's picklers gave (see [`UdfCommands`]): what PySpark's client does once as -/// it builds the plan, the first build did. -fn in_pickled_build() -> bool { - UDF_COMMANDS.with(|commands| commands.borrow().pickled_build) +/// Warn of a window of no partitions that this thread builds, as PySpark's +/// `WindowExpression.to_plan` warns once it has built its window function: now, unless the first +/// build of a request met a pickler before it, after which it is warned of in its turn among the +/// picklings, unless an expression before it failed to build; and not in the second build, which +/// warns no more (see [`UdfCommands`]). +fn warn_of_a_window_without_partitions() { + let now = UDF_COMMANDS.with(|commands| { + let mut commands = commands.borrow_mut(); + if commands.pickled_build { + return false; + } + if commands.met.is_empty() { + return true; + } + if UNBUILDABLE_EXPRESSION_ERROR.with(|slot| slot.borrow().is_none()) { + let picklers_before = commands.met.len(); + commands.windows.push(picklers_before); + } + false + }); + if now { + tell_plan_build_listener(|listener| listener.window_without_partitions()); + } +} + +/// Tell the listener of the plans of the process ([`crate::plan_build`]), if any, of a window of +/// no partitions that the first build of a request met after a pickler: its error is the +/// request's. +fn warn_of_a_met_window() -> Result<()> { + match crate::plan_build::listener() { + Some(listener) => listener.window_without_partitions(), + None => Ok(()), + } } -/// The commands of `picklers`, each pickled in turn, or the error of the first that fails, after -/// which none is pickled, as PySpark's build stops at it. +/// The commands of `picklers`, each pickled in turn, each after the warnings of the `windows` of +/// no partitions that the build met before it (each window is the number of picklers met before +/// it), and then the warnings of the windows after them; or the error of the first pickling or +/// warning that fails, after which none is pickled or warned of, as PySpark's build stops at it. fn pickle_in_turn( picklers: Vec, + windows: &[usize], ) -> Result>> { - picklers - .iter() - .map(crate::udf::CommandPickler::pickle) - .collect() + let mut windows = windows.iter().peekable(); + let mut pickled = std::collections::VecDeque::with_capacity(picklers.len()); + for (picklers_before, pickler) in picklers.iter().enumerate() { + while windows + .next_if(|&&window| window <= picklers_before) + .is_some() + { + warn_of_a_met_window()?; + } + pickled.push_back(pickler.pickle()?); + } + for _ in windows { + warn_of_a_met_window()?; + } + Ok(pickled) } impl ExpressionNode { @@ -325,9 +377,16 @@ impl DepthBoundScope { DEEPEST_EXPRESSION_LEVEL.with(Cell::get) } - /// The picklers that the build met since the scope was entered. - fn met_picklers(&self) -> Vec { - UDF_COMMANDS.with(|commands| std::mem::take(&mut commands.borrow_mut().met)) + /// The picklers that the build met since the scope was entered, and the windows of no + /// partitions that it met after them (see [`UdfCommands`]). + fn met_picklers_and_windows(&self) -> (Vec, Vec) { + UDF_COMMANDS.with(|commands| { + let mut commands = commands.borrow_mut(); + ( + std::mem::take(&mut commands.met), + std::mem::take(&mut commands.windows), + ) + }) } /// Make the next build take `pickled`, the commands of the picklers that the first met. @@ -389,17 +448,26 @@ fn with_expression_levels_and_unbuildable( let mut value = serialize(); let levels = scope.deepest(); let mut unbuildable = scope.unbuildable(); + let (picklers, windows) = scope.met_picklers_and_windows(); if levels > MAX_EXPRESSION_DEPTH { + // The request fails with its `RecursionError`, which stands for the errors of the build, + // and no function is pickled: the windows after a pickler warn, as those before it did in + // the build, up to the first that fails. + for _ in &windows { + if warn_of_a_met_window().is_err() { + break; + } + } return Err(SparkError::recursion_msg(format!( "maximum recursion depth exceeded while serializing an expression; an expression \ may be at most {MAX_EXPRESSION_DEPTH} levels deep" ))); } - let picklers = scope.met_picklers(); if !picklers.is_empty() { - // The picklers come before the first expression that failed to build, if any: the first - // of them that fails is the error, else that expression's. - match pickle_in_turn(picklers) { + // The picklers, and the windows after the first of them, come before the first + // expression that failed to build, if any: the first of them that fails is the error, + // else that expression's. + match pickle_in_turn(picklers, &windows) { Err(error) => unbuildable = Some(error), Ok(pickled) if unbuildable.is_none() => { drop(value); @@ -641,12 +709,65 @@ fn origin_proto(origin: &crate::origin::Origin) -> proto::ExpressionCommon { /// An expression of [`ExpressionNode::Sent`]: the `spark.connect.Expression` of a Column as it /// was sent, its rendering, as `Column<'...'>` shows it, and its levels, as -/// [`Expression::try_to_proto_with_levels`] counts them when the Column was pickled. +/// [`Expression::try_to_proto_with_levels`] counts them when the Column was pickled, and the +/// plans that its subqueries refer to. #[derive(Debug, Clone, PartialEq)] pub struct SentExpression { pub expression: proto::Expression, pub rendered: String, pub levels: usize, + pub subqueries: SentSubqueries, +} + +/// The plans that the subqueries of a [`SentExpression`] refer to by their plan IDs, which the +/// relation that holds the expression sends in its `WithRelations`, as it sends the plan of a +/// [`SubqueryExpression`]: each the relation of a plan as it was sent +/// ([`Expression::subquery_relations`]), of its levels, which keeps its plan ID in plan IDs of its +/// own, as PySpark's unpickled Column holds the plan objects of its subqueries, which keep their +/// `_plan_id`. Two are equal when they are of the same plans. +#[derive(Debug, Clone, Default)] +pub struct SentSubqueries(Vec<(Arc, Arc)>); + +impl SentSubqueries { + /// The plans of `relations`, the relations of the plans that the subqueries of an expression + /// refer to, each of its levels. A relation of more levels than + /// [`crate::plan::MAX_PLAN_DEPTH`] is a `RecursionError`, as the API raises it when it sends + /// one. + fn of(relations: Vec<(proto::Relation, usize)>) -> Result { + relations + .into_iter() + .map(|(relation, levels)| { + if levels > crate::plan::MAX_PLAN_DEPTH { + return Err(crate::plan::too_deep_a_plan(levels)); + } + let plan_id = relation.common.as_ref().and_then(|common| common.plan_id); + let plan = Arc::new(LogicalPlan::Raw { + relation: Box::new(relation), + owner: None, + depth: levels, + observations: Vec::new(), + }); + let ids = Arc::new(crate::plan::PlanIds::new()); + if let Some(plan_id) = plan_id { + ids.assign(&plan, plan_id); + } + Ok((plan, ids)) + }) + .collect::>() + .map(SentSubqueries) + } +} + +impl PartialEq for SentSubqueries { + fn eq(&self, other: &Self) -> bool { + // Plans have no equality: the subqueries of the same plans are equal. + self.0.len() == other.0.len() + && self + .0 + .iter() + .zip(&other.0) + .all(|((plan, _), (other, _))| Arc::ptr_eq(plan, other)) + } } /// The type of a [`SubqueryExpression`]. Mirrors the `subquery_type` of @@ -1167,6 +1288,22 @@ impl Expression { expression: proto::Expression, rendered: String, levels: usize, + ) -> Result { + Expression::sent_with_subqueries(expression, rendered, levels, Vec::new()) + } + + /// As [`Expression::sent`], of an expression whose subqueries refer to the plans of + /// `subqueries`, the relations that [`Expression::subquery_relations`] gave when the Column + /// was pickled, each of its levels: the relation that holds the expression sends them in its + /// `WithRelations`, of their plan IDs, as PySpark's unpickled Column holds the plans of its + /// subqueries, and its plan sends them. A relation of more levels than + /// [`crate::plan::MAX_PLAN_DEPTH`] is a `RecursionError`. Not a stable API. + #[doc(hidden)] + pub fn sent_with_subqueries( + expression: proto::Expression, + rendered: String, + levels: usize, + subqueries: Vec<(proto::Relation, usize)>, ) -> Result { if levels > MAX_EXPRESSION_DEPTH { return Err(SparkError::recursion_msg(format!( @@ -1179,10 +1316,70 @@ impl Expression { expression, rendered, levels, + subqueries: SentSubqueries::of(subqueries)?, }, )))) } + /// The relations of the plans that the subqueries of this expression refer to, each once, in + /// the order of the expression, of the plan IDs of the session of the DataFrame of each, as a + /// relation that holds the expression sends them in its `WithRelations`, and the levels of + /// each, which a pickled Column keeps for [`Expression::sent_with_subqueries`]: those of its + /// subqueries, of an IN subquery and of a table argument too, of the expressions of a + /// subquery, and of an expression that was sent. They are built as a request builds them, so + /// the functions of their UDFs are pickled, and the first error of a build is the error. Not a + /// stable API. + #[doc(hidden)] + pub fn subquery_relations(&self) -> Result> { + use crate::udf::PicklerHolder; + let mut plans: Vec<(&Arc, Option<&Arc>)> = Vec::new(); + let mut add = |plan, ids| { + if !plans.iter().any(|(known, _)| Arc::ptr_eq(known, plan)) { + plans.push((plan, ids)); + } + }; + let mut pending = vec![PicklerHolder::Expression(self)]; + while let Some(holder) = pending.pop() { + let mut below = Vec::new(); + match holder { + PicklerHolder::Expression(expression) => match expression.node() { + ExpressionNode::Sent(sent) => { + for (plan, ids) in &sent.subqueries.0 { + add(plan, Some(ids)); + } + } + node => node.push_pickler_holders(&mut below), + }, + PicklerHolder::Subquery(subquery) => { + add(&subquery.plan, subquery.plan_ids.as_ref()); + below.extend( + subquery + .partition_spec + .iter() + .chain(&subquery.order_spec) + .chain(&subquery.in_subquery_values) + .map(PicklerHolder::Expression), + ); + } + PicklerHolder::Udf(udf) => { + below.extend(udf.arguments.iter().map(PicklerHolder::Expression)); + } + PicklerHolder::Plan(_) | PicklerHolder::Pickler(_) => {} + } + // The first is visited first. + pending.extend(below.into_iter().rev()); + } + plans + .into_iter() + .map(|(plan, ids)| { + let ids = ids + .cloned() + .unwrap_or_else(|| Arc::new(crate::plan::PlanIds::new())); + Ok((crate::plan::relation_with_ids(plan, &ids)?, plan.depth())) + }) + .collect() + } + fn to_proto_with_origin(&self) -> proto::Expression { let mut expr = self.node_to_proto(); // PySpark's `CaseWhen.to_plan` sends a new `UnresolvedFunction("when")` of its branches, @@ -1266,6 +1463,11 @@ impl Expression { }, ExpressionNode::Subquery(subquery) => subquery.to_proto(), ExpressionNode::Sent(sent) => { + // The relation that holds the expression sends the plans of its subqueries, as + // those of a subquery expression. + for (plan, ids) in &sent.subqueries.0 { + crate::plan::refer_to(plan, Some(ids)); + } // The clone recurses for each level of the expression: grow the stack for them // all, as the clone of a raw relation does. let stack = sent_expression_stack(sent.levels); @@ -2082,10 +2284,10 @@ impl WindowExpressionWrapper { // Set the window function window.window_function = Some(Box::new(self.window_function.to_proto())); - // Set partition spec. PySpark's `to_plan` warns of a window of no partitions here, once a - // build: the second build of a plan of UDFs (see [`UdfCommands`]) warns no more. - if self.partition_spec.is_empty() && !in_pickled_build() { - tell_plan_build_listener(|listener| listener.window_without_partitions()); + // Set partition spec. PySpark's `to_plan` warns of a window of no partitions here, in the + // order of the build, among the picklings of the UDFs of the plan (see [`UdfCommands`]). + if self.partition_spec.is_empty() { + warn_of_a_window_without_partitions(); } window.partition_spec = self.partition_spec.iter().map(|e| e.to_proto()).collect(); @@ -2770,6 +2972,7 @@ mod tests { expression: original.to_proto(), rendered: original.render(), levels: 3, + subqueries: SentSubqueries::default(), }))); assert_eq!(sent.to_proto(), original.to_proto()); assert_eq!(sent.render(), "+(a, 1) AS b"); diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs index cb83bab1..966c92f5 100644 --- a/crates/spark-connect/src/plan.rs +++ b/crates/spark-connect/src/plan.rs @@ -5519,6 +5519,80 @@ mod deep_plan_tests { assert!(error.message().contains("data type plan"), "{error}"); } + /// As PySpark's pickle of a Column of a subquery, which keeps the plan objects of its + /// subqueries, of their plan IDs: the expression that the pickle sends, with the relations of + /// the plans of its subqueries (`subquery_relations`), is sent in a relation as the expression + /// was, with the same relations in its `WithRelations`, also from another session's plan IDs, + /// under another expression and pickled again. + #[test] + fn a_sent_expression_of_subqueries_sends_their_plans_as_the_expression_did() { + use crate::expression::{SubqueryExpression, SubqueryType}; + + let ids = Arc::new(PlanIds::new()); + let inner = Arc::new(LogicalPlan::Project { + input: Arc::new(range(0, 5, 1)), + columns: vec![crate::column::Column::new(Expression::function( + "max", + vec![Expression::column("id")], + ))], + }); + let subquery = SubqueryExpression { + plan_ids: Some(Arc::clone(&ids)), + ..SubqueryExpression::new(Arc::clone(&inner), ids.id_of(&inner), SubqueryType::Scalar) + }; + let expression = Expression::function( + "+", + vec![ + Expression::column("id"), + Expression::new(ExpressionNode::Subquery(Box::new(subquery))), + ], + ); + let project = |expression: &Expression| { + Arc::new(LogicalPlan::Project { + input: Arc::new(range(0, 3, 1)), + columns: vec![crate::column::Column::new(expression.clone())], + }) + }; + // The expressions of the root of the `WithRelations` of `relation`, and its references. + let sent = |relation: proto::Relation| { + let Some(proto::relation::RelType::WithRelations(with)) = relation.rel_type else { + panic!("expected WithRelations: {relation:?}"); + }; + let Some(proto::relation::RelType::Project(root)) = with.root.and_then(|r| r.rel_type) + else { + panic!("expected a projection"); + }; + (root.expressions, with.references) + }; + let original = sent(relation_with_ids(&project(&expression), &ids).unwrap()); + let relations = expression.subquery_relations().unwrap(); + assert_eq!(relations.len(), 1); + assert_eq!(relations[0].0, original.1[0]); + let (built, levels) = expression.try_to_proto_with_levels().unwrap(); + let unpickled = + Expression::sent_with_subqueries(built, expression.render(), levels, relations.clone()) + .unwrap(); + assert_eq!(unpickled.render(), expression.render()); + assert_eq!( + sent(relation_with_ids(&project(&unpickled), &PlanIds::new()).unwrap()), + original + ); + assert_eq!(unpickled.subquery_relations().unwrap(), relations); + let over = Expression::function("abs", vec![unpickled]); + assert_eq!(over.subquery_relations().unwrap(), relations); + let (_, references) = sent(relation_with_ids(&project(&over), &PlanIds::new()).unwrap()); + assert_eq!(references, original.1); + // A relation deeper than a plan may be is the `RecursionError` of a pickled DataFrame's. + let error = Expression::sent_with_subqueries( + proto::Expression::default(), + String::new(), + 1, + vec![(relations[0].0.clone(), MAX_PLAN_DEPTH + 1)], + ) + .unwrap_err(); + assert_eq!(error.kind, SparkErrorKind::RecursionError, "{error}"); + } + /// The plan of a subquery that a request fails to build fails the request of the plan /// that refers to it, as PySpark builds the plans of its references with the plan. #[test] diff --git a/crates/spark-connect/src/plan_build.rs b/crates/spark-connect/src/plan_build.rs index 5029d0f9..4b46d51e 100644 --- a/crates/spark-connect/src/plan_build.rs +++ b/crates/spark-connect/src/plan_build.rs @@ -49,10 +49,13 @@ mod tests { use super::*; use crate::column::Column; - use crate::expression::Expression; + use crate::expression::{Expression, ExpressionNode, MAX_EXPRESSION_DEPTH}; use crate::functions::{col, row_number}; use crate::window::Window; + /// Taken by the tests that set the listener of the process, which run one at a time. + static LISTENER_TESTS: Mutex<()> = Mutex::new(()); + /// A listener that records what the plan builds of one thread tell it, as other tests build /// plans at the same time, and that fails a window of no partitions once `fail_windows` is /// set. @@ -107,6 +110,7 @@ mod tests { #[test] fn a_plan_build_tells_the_listener_as_pysparks_client_builds_the_plan() { + let _serial = LISTENER_TESTS.lock().unwrap_or_else(|e| e.into_inner()); let unordered = || row_number().over(Window::order_by(["id"])); let conjunction = Column::new(Expression::with_plan_build_origin( col("a").and(col("b")).expression().clone(), @@ -169,4 +173,111 @@ mod tests { assert_eq!(listener.take(), ["window"]); set_listener(None); } + + /// A Python UDF of `arguments`, whose pickler records its pickling in the calls of + /// `listener`, and fails with the error of `tag` when it `fails`. + fn recording_udf( + listener: &Arc, + tag: &'static str, + fails: bool, + arguments: Vec, + ) -> Column { + let pickler = crate::udf::CommandPickler::new(Arc::clone(listener), move |listener| { + listener.calls.lock().unwrap().push(format!("pickle {tag}")); + if fails { + return Err(SparkError::value_msg(tag)); + } + Ok(tag.as_bytes().to_vec()) + }); + let payload = crate::udf::PythonUDFPayload::pickled_at_build( + crate::types::DataType::Long, + crate::udf::eval_type::SQL_BATCHED_UDF, + pickler, + "3.11".to_string(), + ); + Column::new(Expression::new( + ExpressionNode::CommonInlineUserDefinedFunction(Box::new( + crate::udf::CommonInlineUserDefinedFunctionExpression::new( + "f".to_string(), + true, + arguments.iter().map(|c| c.expression().clone()).collect(), + payload, + ), + )), + )) + } + + #[test] + fn a_window_after_a_udf_is_told_in_its_turn_among_the_picklings_as_pysparks_build() { + let _serial = LISTENER_TESTS.lock().unwrap_or_else(|e| e.into_inner()); + let listener = Arc::new(Recording { + thread: std::thread::current().id(), + calls: Mutex::default(), + fail_windows: Default::default(), + }); + set_listener(Some(listener.clone())); + let unordered = || row_number().over(Window::order_by(["id"])); + let udf = |tag, fails, arguments| recording_udf(&listener, tag, fails, arguments); + let build = + |columns: Vec| crate::functions::r#struct(columns).try_to_proto().map(drop); + + // As PySpark's build of a plan, which pickles each UDF and warns of each window of no + // partitions as it meets them, once, though this one builds a plan of UDFs twice. + build(vec![ + udf("a", false, vec![]), + unordered(), + udf("b", false, vec![]), + unordered(), + ]) + .unwrap(); + assert_eq!( + listener.take(), + ["pickle a", "window", "pickle b", "window"] + ); + // The first pickling that fails ends the build: a window after it is not told. + let error = build(vec![ + udf("a", false, vec![]), + unordered(), + udf("bad", true, vec![]), + unordered(), + ]) + .unwrap_err(); + assert_eq!(error.message(), "bad"); + assert_eq!(listener.take(), ["pickle a", "window", "pickle bad"]); + build(vec![udf("bad", true, vec![]), unordered()]).unwrap_err(); + assert_eq!(listener.take(), ["pickle bad"]); + // The arguments of a UDF are built before its function is pickled. + build(vec![udf("bad", true, vec![unordered()])]).unwrap_err(); + assert_eq!(listener.take(), ["window", "pickle bad"]); + + // An error of a window after a UDF fails the build before the UDFs after it are pickled. + listener + .fail_windows + .store(true, std::sync::atomic::Ordering::SeqCst); + let error = build(vec![ + udf("a", false, vec![]), + unordered(), + udf("b", false, vec![]), + ]) + .unwrap_err(); + assert_eq!(error.message(), "no partitions"); + assert_eq!(listener.take(), ["pickle a", "window"]); + listener + .fail_windows + .store(false, std::sync::atomic::Ordering::SeqCst); + + // An expression too deep to be sent is the `RecursionError` of the request, and no + // function is pickled; its windows are told, as in a build of no UDFs. + let mut deep = col("x"); + for _ in 0..MAX_EXPRESSION_DEPTH { + deep = deep + col("y"); + } + let error = build(vec![udf("a", false, vec![]), unordered(), deep]).unwrap_err(); + assert_eq!( + error.kind, + spark_connect_core::error::SparkErrorKind::RecursionError + ); + assert_eq!(listener.take(), ["window"]); + set_listener(None); + } } diff --git a/dev/parity/known-differences.md b/dev/parity/known-differences.md index 576bdf65..4c333ee4 100644 --- a/dev/parity/known-differences.md +++ b/dev/parity/known-differences.md @@ -337,14 +337,17 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan `docs/compatibility.md`). These cases still differ, all of copies and pickles: - **A pickled Column or DataFrame** (intentional). It carries the commands that its functions - pickled to when it was pickled, in the protobuf of its expression or relation, which the - server's Python worker unpickles without the native client. PySpark's carries the functions, - which its requests pickle again: a value that a function reads, changed after the pickle, is - seen by PySpark's unpickled Column of a function that `pickle` pickles by reference, such as a - module-level function of `__main__`, but not by this client's. So `pickle.dumps` of a Column of - a UDF of a lambda pickles, where PySpark's raises the `PicklingError` of the lambda, and that of - a function that does not pickle by value raises its `PicklingError`, where PySpark's pickles a - module-level function by reference. + pickled to when it was pickled, in the protobuf of its expression or relation. PySpark's carries + the functions, which its requests pickle again: a value that a function reads, changed after the + pickle, is seen by PySpark's unpickled Column of a function that `pickle` pickles by reference, + such as a module-level function of `__main__`, but not by this client's. So `pickle.dumps` of a + Column of a UDF of a lambda pickles, where PySpark's raises the `PicklingError` of the lambda, + and that of a function that does not pickle by value raises its `PicklingError`, where + PySpark's pickles a module-level function by reference. Nothing requires the commands: the + server's Python worker could rebuild PySpark's UDFs of the functions, as it rebuilds PySpark's + Column of the expression. Carrying the functions, which would close these cases, is the + trade-off: a plain pickle of a Column of a UDF of a lambda, which pickles here, would then fail + as PySpark's does, and the pickles of every holder of a plan would change. - **A deep copy of a DataFrame into another session** (tracked). A deep copy whose session copies to another one, of a `memo` that maps it or of another active session of its ID, is a DataFrame of the relation of the copied one, so that its Observations are copies, as PySpark's, and so it @@ -354,9 +357,6 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan PySpark's do: `copy.copy` shares them, and `copy.deepcopy` deep-copies each with the `memo` of the call, which gives a function itself, and a copy of a callable object, a bound method or a `functools.partial`. -- **The UDFs of a `mergeInto` clause** (tracked). Its condition and assignments are pickled when - the clause is added, as PySpark's `_build_merge_action` builds them, and again when `merge` runs, - whose command carries the second pickle; PySpark's carries the first. - **Garbage collection** (tracked). A DataFrame or a Column reports the functions of its UDFs that it alone holds to Python's garbage collector, and so do grouped data and their cogroups, a table argument, a window specification, the writers, `na`, `stat`, a merge builder and its clauses, the @@ -366,11 +366,14 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan they hold, as PySpark's `__dict__` holds them. Those that hold their DataFrame report it, which reports the functions of the plan that they share (`crate::gc::Holders`). A function that a Column and a DataFrame of it share, or two DataFrames, or a DataFrame and a table argument of it, which - holds the plan and not the DataFrame, is reported by none, so that no reference is counted twice: - a reference cycle through it, such as through a bound method of an object that keeps both the - Column and the DataFrame, is not collected, where PySpark collects it. A stream writer holds its - DataFrame, where PySpark's holds the plan, so a DataFrame that keeps its stream writer is freed by - the collector rather than at its last reference. + holds the plan and not the DataFrame, is reported by none: the holders share one reference to it, + which the Rust value of the plan or of the expression holds, and which the collector would count + as that of the holder that reports it, so it could clear the function with that holder while + another holder, still alive, reaches it. So a reference cycle through it, such as through a bound + method of an object that keeps both the Column and the DataFrame, is not collected, where PySpark + collects it, as each of its holders, a Python object, holds a reference of its own. A stream + writer holds its DataFrame, where PySpark's holds the plan, so a DataFrame that keeps its stream + writer is freed by the collector rather than at its last reference. ## Session and client (Connect) diff --git a/docs/compatibility.md b/docs/compatibility.md index 14c1bbe5..b16a75ed 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -372,6 +372,15 @@ Code that constructs the variant directly sets it to `None`; `profile_id` is sen of the schema of the Python client's `DataFrame.to`, which a request calls when it builds the plan, as PySpark converts the schema object then. Code that constructs the variant directly sets it to `None`; `DataFrame::to` sends its schema as before. +`SentExpression`, the expression of a pickled Column (`ExpressionNode::Sent`), adds a `subqueries` +field: the plans that its subqueries refer to, of `Expression::sent_with_subqueries`, which the +relation that holds the expression sends in its `WithRelations`, as it sends the plan of a +`SubqueryExpression`. Code that constructs the struct directly sets it to +`SentSubqueries::default()`, of no plans, as `Expression::sent` gives it. +`PreparedExecution::collect_record_batches_checked`, which is not a stable API, is +`collect_record_batches_and_info`: it gives the `ExecutionInfo` of the execution too, which the +DataFrame keeps once `DataFrame::keep_execution_info` is given it, as PySpark's `toPandas` sets +the information of the execution once it has converted the result. `CoGroupedData::input_columns` is removed. It gave the columns of both sides for the arguments of the function of a cogroup, but that function binds no columns, as PySpark's `CoGroupMap` builds it: @@ -2646,11 +2655,12 @@ function, a lambda or a closure is the function itself, and a callable object, a `copy.deepcopy([c1, c2])` do. `copy.copy` shares the output type of a UDF and the return type of a UDTF too, and `copy.deepcopy` copies them, as PySpark's copy of the plan does: a type changed in place after a deep copy is not that of the copy, nor one changed after a pickle that of the -unpickled DataFrame. A request warns once of a window of no partitions, as PySpark's, -though the drop-in builds the plan of a request twice when it has such functions, as it pickles -them between the two builds: so where a function that does not pickle comes before such a window -in the build, the drop-in warns of the window before it raises the error of the pickling, where -PySpark raises it without the warning. +unpickled DataFrame. A request warns once of a window of no partitions, as PySpark's, in the order +of the build, though the drop-in builds the plan of a request twice when it has such functions, as +it pickles them between the two builds: a window that comes after a function in the build warns in +its turn among the picklings, so a function before it that does not pickle raises the error of its +pickling without the warning, and the error of the warning, of an `"error"` filter of the +warnings, is raised before the functions after the window are pickled, as PySpark's. A pickled Column or DataFrame, and a deep copy of a DataFrame into another session, whose session copies to another one (of a `memo` that maps it, or another active session of its ID), carry the @@ -2661,8 +2671,21 @@ copy, and by its unpickled Column of a function that `pickle` pickles by referen function of `__main__`, but not by the drop-in's. `pickle.dumps` of a Column of a UDF of a function that does not pickle by value raises the `PicklingError` of its command, where PySpark's pickles a module-level function by reference, and `pickle.dumps` of the Column of a UDF of a lambda pickles, -where PySpark's raises the `PicklingError` of the lambda. `CloudPickleSerializer.dumps` of such a -Column raises the same error in both. +where PySpark's raises the `PicklingError` of the lambda, as a pickle of the functions would here +too (see "UDFs" in `dev/parity/known-differences.md`). `CloudPickleSerializer.dumps` of such a +Column raises the same error in both. An output type of a UDF or a return type of a UDTF that the +build of the plan does not convert, such as `AtomicType()` of a UDF without Arrow, or a +`StructType` changed to hold it after the call, does not fail the pickle: as PySpark's, which keeps +the type object, the requests of the unpickled Column and table argument raise its +`UNSUPPORTED_OPERATION`, and so does the unpickle of a DataFrame, which builds its plan. + +A pickled Column of subqueries, of `scalar()`, `exists()`, `isin` of a DataFrame or a table +argument, carries the relations of the plans that they refer to, of their plan IDs, as PySpark's +carries the plan objects of its subqueries, which keep their `_plan_id`: a plan of the unpickled +Column sends them in its `WithRelations`, as a plan of the Column does, also on the server's Python +worker, where the Column is PySpark Connect's. `Column._expr` pickles as the expression of the +unpickled Column, and `TableArg._subquery_expr` as that of a Column of its subquery, as PySpark's +expressions pickle their fields. A DataFrame or a Column reports the functions of its UDFs to Python's garbage collector, as PySpark's plan of Python objects is seen by it, so a reference cycle through a function and the @@ -2680,21 +2703,24 @@ flags and the metadata dict that they hold, and the lists of a struct, as PySpar them, so a cycle through a type that one holds, such as a `DecimalType` with an attribute that keeps the struct, is collected. A function that a Column and a DataFrame of it share, or two DataFrames, or a DataFrame and a table argument of it, which holds its plan and not the DataFrame, as PySpark's -does, is reported by neither, so that no reference is counted twice: a cycle through it is freed -only when it is broken, where PySpark collects it. A stream writer holds its DataFrame, where -PySpark's holds the plan, so a DataFrame that keeps its stream writer is freed by the collector, -where PySpark frees it with its last reference. The functions of `foreachBatch` and `foreach` are -kept as their pickles, as PySpark's stream writer keeps them, so a function that keeps its writer -is freed with its last reference, and a copy or a pickle of the writer carries the pickle of the -call. The registration of `dataSource` reports its session, and the builder of +does, is reported by neither: they share one reference to it, which the collector would count as +that of the one that reports it, so it could free the function with that one while the other lives +and reaches it. A cycle through such a function is freed only when it is broken, where PySpark +collects it. A stream writer holds its DataFrame, where PySpark's holds the plan, so a DataFrame +that keeps its stream writer is freed by the collector, where PySpark frees it with its last +reference. The functions of `foreachBatch` and `foreach` are kept as their pickles, as PySpark's +stream writer keeps them, so a function that keeps its writer is freed with its last reference, and +a copy or a pickle of the writer carries the pickle of the call. The registration of `dataSource` reports its session, and the builder of `pyspark.sql.connect.session.SparkSession` its channel builder and the list of its hook factories, which its shallow copies share, as PySpark's plain objects hold them, so a session that keeps its `dataSource`, or a channel builder or a hook factory that keeps its builder or a copy of it, is collected. -The condition and the assignments of a clause of `mergeInto` are pickled when the clause is added, -which raises there, as PySpark's, and again when the merge runs, whose request carries the later -pickle, where PySpark's carries the one of the clause. +The condition and the assignments of a clause of `mergeInto` are built, and the functions of their +UDFs pickled, when the clause is added, which raises there, and the merge sends them as they were +built then, as PySpark's sends the protos that `_build_merge_action` built for the clause: so a +value that a function reads, changed after the clause, is not that of the merge, also of a copy of +the builder, and a window of no partitions of a clause warns once, at the clause. The drop-in has one class for each UDF and UDTF, the Connect class of `pyspark.sql.connect.udf` and `pyspark.sql.connect.udtf`, which the classic names `pyspark.sql.udf.UserDefinedFunction` and diff --git a/python/pyspark/sql/connect/_column_reduce.py b/python/pyspark/sql/connect/_column_reduce.py index 8430e6ad..2631e59c 100644 --- a/python/pyspark/sql/connect/_column_reduce.py +++ b/python/pyspark/sql/connect/_column_reduce.py @@ -148,6 +148,7 @@ def column_of( levels: int = 1, build_error: Optional[tuple[str, str, Any]] = None, repr_error: Optional[tuple[str, str, Any]] = None, + subqueries: Optional[list[tuple[bytes, int]]] = None, ) -> Any: """The Column of ``expression``, the bytes of a ``spark.connect.Expression`` of ``levels`` levels, and of its rendering ``rendered``, which sends the expression as it is: of the Rust @@ -156,7 +157,10 @@ def column_of( is of PySpark. For a CASE WHEN, ``branches`` are the (condition, value) Columns of its branches and ``else_value`` the Column of its else value, so that it is a CASE WHEN again. For a sort order, ``sort`` is its child Column and the name of its sort method, such as - ``desc_nulls_last``, so that it is a sort order again.""" + ``desc_nulls_last``, so that it is a sort order again. ``subqueries`` are the bytes and the + levels of the relation of each plan that the subqueries of the expression refer to, of its + plan ID, which a plan of the Column sends in its ``WithRelations``, as PySpark's plan sends + the plans of the subqueries of its expressions.""" if sort is not None: # The child is a Column of the client, or PySpark Connect's on a worker, whose sort # method gives back the sort order, as PySpark's SortOrder of its child. @@ -178,15 +182,28 @@ def column_of( ) ) + from pyspark.sql.connect.expressions import SubqueryExpression + # Without annotations: the class body looks its names up by name, which cloudpickle # does not pickle with the function. class Sent(Expression): - """An expression as it was sent, of its rendering.""" + """An expression as it was sent, of its rendering, whose children are a subquery of + the plan of each relation that its subqueries refer to, which a plan of it collects + as its references (``LogicalPlan._collect_references``), and so sends.""" - def __init__(self, expression, rendered): # type: ignore[no-untyped-def] + def __init__(self, expression, rendered, subqueries): # type: ignore[no-untyped-def] super().__init__() self._expression = expression self._rendered = rendered + # The type of such a subquery is not sent: the expression is sent as it is. + self._subqueries = [ + SubqueryExpression(sent_plan(parse_deep(proto.Relation, relation)), "scalar") + for relation, _ in subqueries or [] + ] + + @property + def children(self): # type: ignore[no-untyped-def] + return self._subqueries def to_plan(self, session): # type: ignore[no-untyped-def] if build_error is not None: @@ -200,10 +217,10 @@ def __repr__(self): # type: ignore[no-untyped-def] raise_error(repr_error) return self._rendered - return ConnectColumn(Sent(parse_deep(proto.Expression, expression), rendered)) + return ConnectColumn(Sent(parse_deep(proto.Expression, expression), rendered, subqueries)) return _pyspark.Column._of_expression( - expression, rendered, branches, else_value, levels, build_error, repr_error + expression, rendered, branches, else_value, levels, build_error, repr_error, subqueries ) @@ -236,29 +253,9 @@ def table_arg_of( except ImportError: from pyspark.sql.connect import proto from pyspark.sql.connect.expressions import SubqueryExpression - from pyspark.sql.connect.plan import LogicalPlan from pyspark.sql.connect.table_arg import TableArg - # Without annotations: the class body looks its names up by name, which cloudpickle - # does not pickle with the function. - class Relation(LogicalPlan): - """The plan of a relation as it was sent, of its plan ID, which the subquery of the - table argument refers to.""" - - def __init__(self, relation): # type: ignore[no-untyped-def] - super().__init__(None) - self._relation = relation - if relation.common.HasField("plan_id"): - self._root_plan_id = relation.common.plan_id - - def plan(self, session): # type: ignore[no-untyped-def] - if build_error is not None: - raise_error(build_error) - sent = proto.Relation() - sent.CopyFrom(self._relation) - return sent - - plan = Relation(parse_deep(proto.Relation, relation)) + plan = sent_plan(parse_deep(proto.Relation, relation), build_error) table = TableArg(SubqueryExpression(plan, subquery_type="table_arg")) else: from pyspark.sql.connect.session import SparkSession @@ -274,6 +271,35 @@ def plan(self, session): # type: ignore[no-untyped-def] return table +def sent_plan(relation: Any, build_error: Optional[tuple[str, str, Any]] = None) -> Any: + """PySpark Connect's plan of ``relation``, a ``spark.connect.Relation`` as it was sent, of its + plan ID, which a subquery refers to, on a Python worker of the server: its ``plan`` sends the + relation as it is, or raises the error of ``build_error``, the description of the error of a + plan that did not build.""" + from pyspark.sql.connect import proto + from pyspark.sql.connect.plan import LogicalPlan + + # Without annotations: the class body looks its names up by name, which cloudpickle does not + # pickle with the function. + class Relation(LogicalPlan): + """The plan of a relation as it was sent, of its plan ID.""" + + def __init__(self, relation): # type: ignore[no-untyped-def] + super().__init__(None) + self._relation = relation + if relation.common.HasField("plan_id"): + self._root_plan_id = relation.common.plan_id + + def plan(self, session): # type: ignore[no-untyped-def] + if build_error is not None: + raise_error(build_error) + sent = proto.Relation() + sent.CopyFrom(self._relation) + return sent + + return Relation(relation) + + def raise_error(error: tuple[str, str, Any]) -> None: """Raise the error of ``error``, the ``(kind, text, payload)`` of a pickled Column or DataFrame of an expression or a plan that a request did not build, or of an expression whose diff --git a/python/pyspark/sql/connect/column.py b/python/pyspark/sql/connect/column.py index 6e1103c9..8a1266b7 100644 --- a/python/pyspark/sql/connect/column.py +++ b/python/pyspark/sql/connect/column.py @@ -35,17 +35,21 @@ def _reduce_column( levels: int = 1, build_error: Optional[tuple[str, str, Any]] = None, repr_error: Optional[tuple[str, str, Any]] = None, + subqueries: Optional[list[tuple[bytes, int]]] = None, ) -> tuple[Callable[..., Any], tuple[Any, ...]]: """The ``__reduce__`` of a Column of the bytes of its ``spark.connect.Expression`` and of its rendering, as PySpark Connect's Column pickles its expression: the function of ``pyspark.sql.connect._column_reduce``, which gives back the Column, which sends the expression as it is. A CASE WHEN gives its branches and else value too, as Columns, so the Column gives back a CASE WHEN. A sort order gives its child Column and its sort method, so - the Column gives back a sort order. cloudpickle pickles that function by value, also to a - Python worker of the server, where it gives back PySpark Connect's Column, and ``pickle`` by - reference.""" + the Column gives back a sort order. A Column of subqueries gives the bytes and the levels of + the relation of each plan that they refer to, which the Column sends with the expression. + cloudpickle pickles that function by value, also to a Python worker of the server, where it + gives back PySpark Connect's Column, and ``pickle`` by reference.""" args: tuple[Any, ...] = (expression, rendered, branches, else_value, sort, levels) - if build_error is not None or repr_error is not None: + if build_error is not None or repr_error is not None or subqueries: # The errors of an expression that a request does not build, and of its ``__repr__``. args += (build_error, repr_error) + if subqueries: + args += (subqueries,) return _column_reduce.column_of, args diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py index 3661a671..c98f150a 100644 --- a/python/tests/test_dropin_e2e.py +++ b/python/tests/test_dropin_e2e.py @@ -2953,6 +2953,40 @@ def eval(self, row): session.range(1).asTable().orderBy("id") +def test_pickled_subquery_columns_return_the_rows_of_pyspark(session, tmp_path): + # As PySpark's pickle of a Column of a subquery, which keeps the plans of its subqueries: the + # unpickled scalar, EXISTS and IN subqueries return the rows of the Columns built without the + # pickling, and a foreachBatch function that captures a scalar subquery selects it on the + # Python worker of the server, with PySpark's Column, which sends the plan of the subquery. + import pickle + + df = session.range(3) + largest = session.range(5).select(F.max("id")).scalar() + for column in (largest, (F.col("id") + largest).alias("plus"), + F.col("id").isin(session.range(2))): + copied = pickle.loads(pickle.dumps(column)) + assert df.select(copied).collect() == df.select(column).collect() + missing = session.range(1).where(F.col("id") > 10).exists() + assert df.where(~pickle.loads(pickle.dumps(missing))).count() == 3 + + (tmp_path / "input.txt").write_text("a\nbb\n") + captured = session.range(5).select(F.max("id")).scalar() + table = "foreach_batch_captured_subquery" + + def func(batch, batch_id): + batch.select((F.length("value") + captured).alias("n")).write.mode("append").saveAsTable( + table) + + query = session.readStream.format("text").load(str(tmp_path)).writeStream.foreachBatch(func) + query = query.start() + try: + query.processAllAvailable() + assert sorted(row.n for row in session.table(table).collect()) == [5, 6] + finally: + query.stop() + session.sql(f"DROP TABLE IF EXISTS {table}").collect() + + def test_columns_of_another_session_do_not_resolve_in_this_one(session): # As PySpark's, plan IDs come from one counter of the process, so a column of a DataFrame of # another session names no node of this one: CANNOT_RESOLVE_DATAFRAME_COLUMN, not the diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index b12e990a..9f5b2afb 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -4914,6 +4914,57 @@ def unsupported(request, operation): assert (len(service.analyze_requests), len(service.execute_requests)) == sent +def test_pickles_of_types_that_do_not_convert_fail_where_the_plan_is_built_as_pyspark( + mock_connect): + # As PySpark's pickles of a Column, a DataFrame and a table argument, which keep the output + # type of a UDF and the return type of a UDTF as they are, and convert them when a plan of the + # unpickled object is built: a type that pyspark_types_to_proto_types does not convert, such + # as AtomicType() of a UDF without Arrow or a StructType changed to hold it after the call, + # pickles, and raises its UNSUPPORTED_OPERATION at the requests of the unpickled Column and + # table argument, and at the unpickle of a DataFrame, which builds its plan. A function that + # does not pickle still raises at the pickle (see "UDFs" in known-differences.md). + import threading + + import cloudpickle + from pyspark.errors import PySparkValueError + from pyspark.sql import functions as F + from pyspark.sql import types as T + + spark, service = mock_connect + lock = threading.Lock() + + class Handler: + def eval(self, x): + yield (x,) + + def unsupported(call, operation): + with pytest.raises(PySparkValueError) as error: + call() + assert error.value.getCondition() == "UNSUPPORTED_OPERATION" + assert error.value.getMessageParameters() == {"operation": operation} + + df = spark.range(2) + mutated = T.StructType([T.StructField("a", T.LongType())]) + columns = [F.udf(lambda x: x, T.AtomicType(), useArrow=False)("id"), + F.udf(lambda x: x, mutated)("id")] + mutated.fields[0].dataType = T.AtomicType() + sent = (len(service.analyze_requests), len(service.execute_requests)) + for column in columns: + for dumps, loads in ((pickle.dumps, pickle.loads), + (cloudpickle.dumps, cloudpickle.loads)): + restored = loads(dumps(column)) + assert repr(restored) == repr(column) + unsupported(lambda: df.select(restored).schema, "data type AtomicType()") + frame = F.udtf(Handler, returnType=5)(F.lit(1)) + data = cloudpickle.dumps(frame) + unsupported(lambda: cloudpickle.loads(data), "data type 5") + table = pickle.loads(pickle.dumps(frame.asTable())) + unsupported(lambda: F.udtf(Handler, returnType="x int")(table).schema, "data type 5") + assert (len(service.analyze_requests), len(service.execute_requests)) == sent + with pytest.raises(pickle.PicklingError): + pickle.dumps(F.udf(lambda x: (lock, x)[1], "long")("id")) + + def test_udtfs_are_created_as_pyspark(mock_connect): # As PySpark's _create_py_udtf and _create_pyarrow_udtf: a UDTF is not deterministic # unless asDeterministic() marks it so, in place; arrow_udtf() makes a PyArrow-native @@ -8719,7 +8770,9 @@ def test_decimal_type_methods(): def test_columns_pickle_as_their_expressions_as_pyspark(): # As PySpark Connect's Columns, which pickle their expressions: pickle, cloudpickle and # deepcopy give back a Column of the expression as it was sent, of its repr, and the - # expressions over it are those over the original. + # expressions over it are those over the original. The expression of a Column, `_expr`, + # pickles too, as PySpark's expression, a Python object, pickles its fields: as the + # expression of the unpickled Column. import copy from pyspark.serializers import CloudPickleSerializer @@ -8744,6 +8797,90 @@ def test_columns_pickle_as_their_expressions_as_pyspark(): over, over_original = F.coalesce(restored, F.lit(0)), F.coalesce(column, F.lit(0)) assert repr(over) == repr(over_original) assert over.to_plan(None) == over_original.to_plan(None) + expression = column._expr + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + restored = pickle.loads(pickle.dumps(expression, protocol)) + assert type(restored) is type(expression) + assert repr(restored) == repr(expression) + assert restored.to_plan(None) == expression.to_plan(None) + restored = serializer.loads(serializer.dumps(expression)) + assert restored.to_plan(None) == expression.to_plan(None) + + +def test_sent_plans_of_pickled_relations_send_them_as_they_were_sent(): + # The plan that a pickled subquery Column or table argument makes of a relation that it sent, + # on a Python worker of the server, whose PySpark Connect builds the request of its plan: the + # plan sends a copy of the relation as it was sent, of its plan ID, which the subquery refers + # to, or raises the error of a plan that did not build, as the client raised it. + from pyspark.errors import PySparkTypeError + from pyspark.sql.connect import proto + from pyspark.sql.connect._column_reduce import sent_plan + + relation = proto.Relation() + relation.common.plan_id = 7 + relation.range.end = 3 + plan = sent_plan(relation) + assert plan._root_plan_id == 7 + sent = plan.plan(None) + assert sent == relation and sent is not relation + unnumbered = proto.Relation() + unnumbered.range.end = 3 + assert sent_plan(unnumbered).plan(None) == unnumbered + with pytest.raises(ValueError, match="^bad count$"): + sent_plan(relation, ("ValueError", "bad count", None)).plan(None) + with pytest.raises(PySparkTypeError) as error: + sent_plan(relation, ("PySparkTypeError", "NOT_ITERABLE", {"objectName": "Column"})).plan( + None + ) + assert error.value.getCondition() == "NOT_ITERABLE" + + +def test_pickled_columns_of_subqueries_send_the_plans_of_their_subqueries_as_pyspark( + mock_connect): + # As PySpark's pickle of a Column of a subquery, of `scalar()`, `exists()`, `isin` of a + # DataFrame or of a table argument, which keeps the plan objects of its subqueries, of their + # plan IDs: the plan of the unpickled Column sends their relations in its WithRelations, as + # the plan of the Column does, also of an expression over it and of its pickle in turn; and + # the unpickled `_expr` of the Column and `_subquery_expr` of a table argument build the + # expressions of the originals. + from pyspark.serializers import CloudPickleSerializer + from pyspark.sql import functions as F + from pyspark.sql.connect.proto import base_pb2 + + spark, service = mock_connect + serializer = CloudPickleSerializer() + df = spark.range(3) + scalar = spark.range(5).select(F.max("id")).scalar() + + def sent(column): + # The relations that the schema request of a select of `column` sends with it. + df.select(column).schema + request = base_pb2.AnalyzePlanRequest.FromString(service.analyze_requests[-1]) + return list(request.schema.plan.root.with_relations.references) + + for column in [ + scalar, + spark.range(1).where(F.col("id") > 10).exists(), + F.col("id").isin(spark.range(2)), + (F.col("id") + scalar).alias("plus"), + F.col("id") + spark.range(5).select(F.max("id") + scalar).scalar(), + ]: + relations = sent(column) + assert relations + unpickled = pickle.loads(pickle.dumps(column)) + for restored in ( + unpickled, + serializer.loads(serializer.dumps(column)), + pickle.loads(pickle.dumps(unpickled)), + ): + assert repr(restored) == repr(column) + assert sent(restored) == relations + assert sent(F.coalesce(restored, F.lit(0))) == sent(F.coalesce(column, F.lit(0))) + expression = pickle.loads(pickle.dumps(column._expr)) + assert expression.to_plan(None) == column._expr.to_plan(None) + table = spark.range(4).asTable() + expression = pickle.loads(pickle.dumps(table._subquery_expr)) + assert expression.to_plan(None) == table._subquery_expr.to_plan(None) def test_column_repr_and_operations(): @@ -15573,6 +15710,43 @@ def test_execution_info_is_one_object_until_another_execution_as_pyspark(mock_co assert frame() is None +def test_a_to_pandas_that_fails_to_convert_leaves_the_execution_info_as_pyspark( + mock_connect, monkeypatch): + # As PySpark's `_to_pandas`, which sets `_execution_info` once `SparkConnectClient.to_pandas` + # has converted the result: a toPandas that fails after its execution, in its conversion, + # leaves `executionInfo` as it was, None or the object of the last execution, and one that + # converts sets that of its execution. `collect` sets it before it converts its rows, as + # PySpark's `_to_table` does. + import pyspark.sql.connect.client as client + + spark, service = mock_connect + # The settings that toPandas reads. + service.confs = {"spark.sql.session.timeZone": "UTC", + "spark.sql.execution.pandas.structHandlingMode": "legacy", + "spark.sql.execution.arrow.pyspark.selfDestruct.enabled": "false"} + _two_row_range_results(service) + convert = client._to_pandas + + def fails(*args, **kwargs): + raise ValueError("conversion") + + monkeypatch.setattr(client, "_to_pandas", fails) + df = spark.range(2) + with pytest.raises(ValueError, match="conversion"): + df.toPandas() + assert df.executionInfo is None + df.collect() + info = df.executionInfo + assert info.operation_id == _request_operation_id(service.execute_requests[-1]) + with pytest.raises(ValueError, match="conversion"): + df.toPandas() + assert df.executionInfo is info + monkeypatch.setattr(client, "_to_pandas", convert) + assert len(df.toPandas()) == 2 + assert df.executionInfo is not info + assert df.executionInfo.operation_id == _request_operation_id(service.execute_requests[-1]) + + def test_copies_and_unpickled_dataframes_keep_support_repr_html_as_pyspark(mock_connect): # As PySpark's DataFrame.__reduce__, whose state carries _support_repr_html to a copy, a deep # copy and an unpickled DataFrame: once _repr_html_ was called, their repr is the string of the @@ -26933,6 +27107,79 @@ def test_windows_of_no_partitions_warn_once_a_request_of_udfs_as_pyspark(mock_co assert sum("No Partition Defined" in str(warning.message) for warning in record) == 2 +def test_windows_of_no_partitions_warn_in_turn_with_the_picklings_of_udfs_as_pyspark(mock_connect): + # As PySpark's client, which builds the plan of a request once, in its order, the child of a + # plan before its own expressions and the arguments of a UDF before its function, and pickles + # the function of each UDF and warns of each window of no partitions as it builds them: a + # pickling that fails ends the build, so a window after it does not warn, and the "error" + # filter raises the warning of a window before the functions after it are pickled. The + # drop-in builds a plan of UDFs twice, and warns of a window after a UDF in its turn among the + # picklings (pyspark-client 4.3.0 gives these sequences). + import pickle + import warnings + + from pyspark.sql import Window + from pyspark.sql import functions as F + + spark, service = mock_connect + log = [] + + class Traced: + def __init__(self, tag, fails=False): + self.tag, self.fails = tag, fails + + def __call__(self, x): + return x + + def __reduce__(self): + log.append("P:" + self.tag) + if self.fails: + raise TypeError("cannot pickle " + self.tag) + return (Traced, (self.tag,)) + + def udf(tag, fails=False): + return F.udf(Traced(tag, fails), "long") + + def window(name): + return F.row_number().over(Window.orderBy("id")).alias(name) + + def show(message, category, filename, lineno, file=None, line=None): + log.append("W") + + df = spark.range(2) + cases = [ + (lambda: df.select(udf("a")("id"), window("w"), udf("bad", True)("id")), + (pickle.PicklingError, "P:a W P:bad"), (UserWarning, "P:a")), + (lambda: df.select(udf("bad", True)("id"), window("w")), + (pickle.PicklingError, "P:bad"), (pickle.PicklingError, "P:bad")), + (lambda: df.select(window("w1"), udf("a")("id"), window("w2"), udf("b")("id"), window("w3")), + (None, "W P:a W P:b W"), (UserWarning, "")), + (lambda: df.select( + window("w1"), udf("a")("id"), window("w2"), udf("bad", True)("id"), window("w3")), + (pickle.PicklingError, "W P:a W P:bad"), (UserWarning, "")), + (lambda: df.select(udf("bad", True)(F.row_number().over(Window.orderBy("id")))), + (pickle.PicklingError, "W P:bad"), (UserWarning, "")), + (lambda: df.select(udf("a")("id").alias("g"), window("r")) + .select("g", udf("bad", True)("g"), window("w")), + (pickle.PicklingError, "P:a W P:bad"), (UserWarning, "P:a")), + ] + for make, always, error in cases: + for action, (raised, sequence) in (("always", always), ("error", error)): + # Made first: `udf` warns of pandas 3, as PySpark's does. + frame = make() + del log[:] + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + warnings.filterwarnings(action, message="WARN WindowExpression") + warnings.showwarning = show + if raised is None: + frame.schema + else: + with pytest.raises(raised): + frame.schema + assert " ".join(log) == sequence, (action, sequence) + + def test_chained_partition_by_and_order_by_replace_the_spec_as_pyspark(): # As PySpark's WindowSpec.partitionBy and orderBy (connect/window.py), which make a spec of the # new partitioning or ordering in place of the old one and keep the other parts, as Spark's @@ -31774,6 +32021,55 @@ def keys(action): assert len(service.execute_requests) == requests +def test_merges_send_the_clauses_as_their_actions_built_them_as_pyspark(mock_connect): + # As PySpark's MergeIntoWriter, whose clause actions keep the `MergeAction` protos that + # `_build_merge_action` builds, which the merge sends: the functions of the UDFs of a clause + # are pickled once, when its action adds it, so a value that a function reads, changed after + # that, is not that of the merge, of a copy of the builder either; and a window of no + # partitions of a clause warns once, at its action. The condition of mergeInto is built by + # the merge. A clause of a function that does not pickle raises at its action. + import copy + import threading + import warnings + + from pyspark.sql import Window + from pyspark.sql import functions as F + from pyspark.sql.connect.proto import base_pb2 + + spark, service = mock_connect + state = [""] + df = spark.range(3) + merge = df.mergeInto("t", F.udf(lambda x: state[0] == "", "boolean")("id")) + merge.whenMatched(F.udf(lambda x: state[0] == "", "boolean")("id")).update( + {"v": F.udf(lambda x: state[0], "string")("id")}) + state[0] = "" + for writer in (merge, copy.copy(merge)): + writer.merge() + command = base_pb2.ExecutePlanRequest.FromString( + service.execute_requests[-1]).plan.command.merge_into_table_command + action = command.match_actions[0].merge_action + for sent in (action.condition, action.assignments[0].value): + udf = sent.common_inline_user_defined_function.python_udf.command + assert b"" in udf and b"" not in udf + condition = command.merge_condition.common_inline_user_defined_function.python_udf.command + assert b"" in condition + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + clause = df.mergeInto("t", F.lit(True)).whenMatched( + F.row_number().over(Window.orderBy("id")) > 0) + assert record == [] + writer = clause.delete() + assert len(record) == 1 + writer.merge() + assert len(record) == 1 + lock = threading.Lock() + requests = len(service.execute_requests) + with pytest.raises(pickle.PicklingError): + df.mergeInto("t", F.lit(True)).whenMatched( + F.udf(lambda x: bool(lock), "boolean")("id")).delete() + assert len(service.execute_requests) == requests + + # --------------------------------------------------------------------------- artifact uploads that stream their chunks, and class files def test_add_artifacts_refuses_the_paths_pyspark_refuses(tmp_path):