From 252280410a1e31081039f5bc3870ad3d16f39c4a Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Thu, 19 Nov 2026 02:29:45 +0000 Subject: [PATCH 1/2] [SPARK-62072][CONNECT][PYTHON] Keep the _name, _result and _error of an Observation in its __dict__ and set its metrics at the action, and let the plan compression lookup catch a signal handler's Exception, as PySpark PySpark Connect's Observation (connect/observation.py:34-86) is a plain class: `_name`, `_result` and `_error` are the attributes of its __dict__, which `get` and `_on` read, which a program sets and deletes, and which its client sets at each action (connect/client/core.py:1766-1799): it reads the `_result` of each Observation, asserts that it is not None, converts the metrics and updates it in place, and gives the error of collecting them to `_set_error`. The drop-in's Observation had them as read-only getters of its native state: vars(o) was empty, they could not be set, each read of `_error` made a new exception, a reference cycle through the result dict, which the native state held, was never freed, and the metrics were converted when `get` was read, so a metric that does not convert, such as a timestamp of the year 1, failed `get` where PySpark fails the action and leaves `get` empty. PySpark's lookup of the plan compression settings of the first plan (`_get_plan_compression_threshold_and_algorithm`, core.py:2714-2740) catches `Exception`, also the exception that a signal handler raises while it waits, and its execution interrupts the operation of a KeyboardInterrupt only (core.py:1943-1950). So a SIGINT handler that raises an Exception subclass while the exit hook's clean_cache waits does not end the clean_cache when it arrives during the lookup, and the clean_cache waits again; the drop-in ended it at the first signal, and it interrupted the operation of any handler's exception, so the next signal only ended the wait of the interrupt. - The Observation keeps `_name`, `_result` and `_error` in its __dict__: `__init__` sets them, `get` raises `_error` or gives `_result` as PySpark's, `_on` checks `_result`, names an unnamed one by a random UUID and sets `_result` to a new dict, `_set_error` sets `_error`, and the copies, deep copies and pickles are of the __dict__. Each `_on` observes by a new state of the core, whose mirror, which holds the Observation weakly, is asked whether it is used, of its `_result`, converts the metrics and updates its `_result` in place, and gives the error of the server to its `_set_error`: their errors, such as PySpark's AssertionError of a `_result` that is None or the ValueError of a metric that does not convert, are those of the action. An Observation that the client is given for an execution is set by its attributes too, and DataFrame.observe gives what the `_on` of an Observation gives, of a subclass too. - The core's MetricsMirror says whether the observation is used, and its update and its error are fallible, which the routing of the observed metrics of an execution returns; a mirror replaces the one before. - The core runtime has `catching`, a part of a wait whose caller catches the exception of a signal handler that is an Exception, and `catch_signal`, which ends it. The lookup of the plan compression settings, which then disables the compression for the client, and the read of the stack-trace settings of an error run in one. The binding's watch of the signals catches the exception of a handler that is an Exception there, and ends the wait at once, without an interrupt, for the exception of a handler that is not a KeyboardInterrupt. - docs/compatibility.md describes the attributes of an Observation and the signal handling of the waits; known-differences.md notes that an Observation renamed after `observe` keeps its first name in the plan. Co-authored-by: Isaac --- crates/pyspark-rs/src/client.rs | 2 +- crates/pyspark-rs/src/dataframe.rs | 37 +- crates/pyspark-rs/src/interrupt.rs | 50 +- crates/pyspark-rs/src/observation.rs | 481 +++++++----------- crates/spark-connect-core/src/client.rs | 48 +- crates/spark-connect-core/src/runtime.rs | 89 ++++ .../tests/server_session.rs | 65 +++ crates/spark-connect/src/dataframe.rs | 13 +- crates/spark-connect/src/observation.rs | 257 +++++++--- dev/parity/known-differences.md | 10 +- docs/compatibility.md | 33 +- python/tests/test_dropin_offline.py | 238 ++++++++- python/tests/test_exit_cleanup_offline.py | 53 +- 13 files changed, 945 insertions(+), 431 deletions(-) diff --git a/crates/pyspark-rs/src/client.rs b/crates/pyspark-rs/src/client.rs index eb917418..173d35e0 100644 --- a/crates/pyspark-rs/src/client.rs +++ b/crates/pyspark-rs/src/client.rs @@ -84,7 +84,7 @@ pub(crate) fn execute_plan( .into_iter() .map(|(name, observation)| { let observation = observation.cast::()?; - Ok((name, observation.borrow().inner.clone())) + Ok((name, crate::observation::state_of_execution(observation)?)) }) .collect::>>()?; let execution = interruptible(py, session, || { diff --git a/crates/pyspark-rs/src/dataframe.rs b/crates/pyspark-rs/src/dataframe.rs index 1d4500c3..21e82299 100644 --- a/crates/pyspark-rs/src/dataframe.rs +++ b/crates/pyspark-rs/src/dataframe.rs @@ -324,7 +324,7 @@ impl PyDataFrame { /// this one keeps apart from the core, as PySpark's `alias`, `filter`, `limit`, `sort`, the /// repartitions and the other methods that copy the `_cached_schema` of their input copy it /// whatever it is. The core keeps a cached struct itself. - fn of_same_schema(&self, mut result: PyDataFrame) -> PyDataFrame { + pub(crate) fn of_same_schema(&self, mut result: PyDataFrame) -> PyDataFrame { if let Some(schema) = &self.schema_object { result.schema_object = Some(Python::attach(|py| schema.clone_ref(py))); } @@ -2503,14 +2503,15 @@ impl PyDataFrame { } /// Observe metrics, as PySpark's `DataFrame.observe(observation, *exprs)`: the Columns - /// `exprs`, which an Observation receives, or which are observed by a name. + /// `exprs`, which an Observation receives by its `_on`, of a subclass too, or which are + /// observed by a name. #[pyo3(signature = (observation, *exprs))] - fn observe( - &self, - py: Python<'_>, - observation: &Bound<'_, PyAny>, - exprs: Vec>, - ) -> PyResult { + fn observe<'py>( + slf: &Bound<'py, Self>, + py: Python<'py>, + observation: &Bound<'py, PyAny>, + exprs: Vec>, + ) -> PyResult> { if exprs.is_empty() { return Err(crate::column::raise_pyspark( py, @@ -2537,18 +2538,26 @@ impl PyDataFrame { } } } - if let Ok(observation) = observation.cast::() { - let observed = crate::observation::observe(py, self, observation, columns)?; - return Ok(self.of_same_schema(observed)); + if observation.is_instance_of::() { + // As PySpark's, which gives the DataFrame of the `_on` of the Observation, of a + // subclass too. + let mut arguments = vec![slf.clone().into_any()]; + arguments.extend(exprs); + return observation.call_method1( + pyo3::intern!(py, "_on"), + pyo3::types::PyTuple::new(py, arguments)?, + ); } + let this = slf.borrow(); if observation.is_instance_of::() { // Its `CollectMetrics` plan sets the name when it builds the plan, before the metrics. let mut args = crate::plan_args::PlanArgs::new(); let name = args.str(observation)?; let args = args.before(&mut columns); - return Ok(self.of_same_schema(PyDataFrame::new( - args.plan(self.dataframe.observe(&name, columns)), - ))); + let observed = this.of_same_schema(PyDataFrame::new( + args.plan(this.dataframe.observe(&name, columns)), + )); + return Ok(Bound::new(py, observed)?.into_any()); } let type_name = observation.get_type().name()?.to_string(); Err(crate::column::raise_pyspark( diff --git a/crates/pyspark-rs/src/interrupt.rs b/crates/pyspark-rs/src/interrupt.rs index 3a8d8ca5..b65db77e 100644 --- a/crates/pyspark-rs/src/interrupt.rs +++ b/crates/pyspark-rs/src/interrupt.rs @@ -3,9 +3,13 @@ //! //! The wait runs on the calling thread, as PySpark's does, so the progress handlers and the //! other callbacks of the execution run there too. The wait polls the signals of the process, -//! which only the main thread of Python runs the handlers of. A handler that raises interrupts the -//! execution in flight of the calling thread on the server; the wait then ends with the server's -//! cancellation, and the exception of the handler is raised. +//! which only the main thread of Python runs the handlers of. A handler that raises a +//! KeyboardInterrupt, as the default handler of SIGINT does, interrupts the execution in flight of +//! the calling thread on the server; the wait then ends with the server's cancellation, and the +//! exception of the handler is raised. The exception of another handler ends the wait at once, as +//! PySpark's execution interrupts the operation of a KeyboardInterrupt only, unless the part of +//! the call that waits catches it, as PySpark's lookup of the plan compression settings catches an +//! `Exception` (see `spark_connect_core::runtime::catching`). use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; @@ -122,10 +126,35 @@ impl SignalWatch { { let mut error = self.error.lock().unwrap_or_else(|e| e.into_inner()); if error.is_none() { - // Takes the GIL for the check only, as the wait runs without it. - match Python::attach(|py| py.check_signals()) { - Ok(()) => return Watch::Continue, - Err(signal) => *error = Some(signal), + // Takes the GIL for the check only, as the wait runs without it. The exception of a + // handler that is an `Exception` ends the part of the call that PySpark's `except + // Exception` covers, if one runs, such as the lookup of the plan compression + // settings, and the call goes on, as PySpark's does (see `runtime::catching`). + let raised = Python::attach(|py| match py.check_signals() { + Ok(()) => None, + Err(signal) + if signal.is_instance_of::(py) + && spark_connect_core::runtime::catch_signal() => + { + None + } + Err(signal) => { + let keyboard_interrupt = + signal.is_instance_of::(py); + Some((signal, keyboard_interrupt)) + } + }); + match raised { + None => return Watch::Continue, + Some((signal, keyboard_interrupt)) => { + *error = Some(signal); + // As PySpark's execution, whose `except KeyboardInterrupt` interrupts the + // operation on the server: the exception of another handler ends the + // wait at once, as it goes out of the execution as it is. + if !keyboard_interrupt { + return Watch::Abort; + } + } } } else if self.interrupted.load(Ordering::SeqCst) { // A failed interrupt ends the wait at once, as PySpark raises the error of the @@ -159,9 +188,10 @@ impl SignalWatch { } /// Run `f` without the GIL, on the calling thread, as the signals of the process are polled while -/// it waits for the server. When a handler raises, the execution that the calling thread waits for -/// is interrupted, and the exception of the handler is raised once `f` has ended. Otherwise the -/// result of `f` is returned. +/// it waits for the server. When a handler raises a KeyboardInterrupt, the execution that the +/// calling thread waits for is interrupted, and the exception of the handler is raised once `f` +/// has ended; the exception of another handler ends the waits of `f` at once, and is raised. +/// Otherwise the result of `f` is returned. /// /// The execution is interrupted only when a wait of the calling thread is for one. The waits /// that a signal ends at once, as PySpark's do (a retry sleep, or a call that the server does not diff --git a/crates/pyspark-rs/src/observation.rs b/crates/pyspark-rs/src/observation.rs index e1b63e38..af05d8d9 100644 --- a/crates/pyspark-rs/src/observation.rs +++ b/crates/pyspark-rs/src/observation.rs @@ -2,8 +2,9 @@ use std::sync::Arc; +use pyo3::intern; use pyo3::prelude::*; -use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::types::{PyDict, PyString, PyTuple, PyWeakrefReference}; use spark_connect::expression::Expression; use spark_connect::observation::{MetricsMirror, Observation}; use spark_connect::SparkSession; @@ -11,11 +12,13 @@ use spark_connect_core::error::{SparkError, SparkErrorKind}; use crate::column::{raise_pyspark, PyColumn}; use crate::dataframe::PyDataFrame; -use crate::errors::ResultExt; /// Python wrapper for an Observation of metrics, as PySpark Connect's `Observation`, a plain -/// class, which is subclassed: an Observation of a subclass is constructed of the arguments of -/// its `__init__`, and its copies and pickles are of its class. +/// class, which is subclassed. Its `_name`, `_result` and `_error` are attributes of its +/// `__dict__`, as PySpark's, which a program reads, sets and deletes, and which `get`, `_on` and +/// the actions read and set as PySpark's do (see [`PyObservationMirror`]). An Observation of a +/// subclass is constructed of the arguments of its `__init__`, and its copies and pickles are of +/// its class. #[pyclass( name = "Observation", module = "pyspark.sql.connect.observation", @@ -24,50 +27,10 @@ use crate::errors::ResultExt; dict )] pub struct PyObservation { + /// The state of the observation in the core, of its last `_on`, which the plans of the + /// DataFrames that it observed hold, and whose metrics and errors the actions set in the + /// attributes of this object. pub(crate) inner: Observation, - /// The `_result` and `_error` of an Observation unpickled from the state of PySpark's (see - /// `__setstate__`), which no action sets, and which `get` gives back as PySpark's does. - restored: Restored, - /// A name that UTF-8 cannot encode, as it was given, which `_name` gives back: PySpark keeps - /// it, and its `CollectMetrics` plan sets it when it builds the plan, which fails the requests - /// of each DataFrame that it observes (see [`observe`]). `inner` is of its rendering. - unencodable_name: Option>, -} - -/// The state of an Observation of the name `name`, None or a str, and the name itself if UTF-8 -/// cannot encode it (see `PyObservation::unencodable_name`). -fn named(name: Option<&Bound<'_, PyAny>>) -> PyResult<(Observation, Option>)> { - let Some(name) = name.filter(|name| !name.is_none()) else { - return Ok((Observation::unnamed(), None)); - }; - let name = name.cast::()?; - Ok(match name.to_cow() { - Ok(text) => (Observation::new(&text), None), - Err(_) => ( - Observation::new(&crate::plan_args::rendered(name.as_any())?), - Some(name.clone().unbind()), - ), - }) -} - -/// The `_result` and `_error` of the state of a pickled Observation, which its Rust state does -/// not hold: none for an Observation that was not unpickled. -#[derive(Default)] -struct Restored { - /// The metrics of an Observation unpickled from a used one, a dict as PySpark's. - result: Option>, - /// The exception that `get` raises. - error: Option>, -} - -impl Restored { - /// The same objects, as a shallow copy of PySpark's Observation keeps them. - fn clone_ref(&self, py: Python<'_>) -> Restored { - Restored { - result: self.result.as_ref().map(|result| result.clone_ref(py)), - error: self.error.as_ref().map(|error| error.clone_ref(py)), - } - } } #[pymethods] @@ -84,14 +47,12 @@ impl PyObservation { fn new(_args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>) -> Self { PyObservation { inner: Observation::unnamed(), - restored: Restored::default(), - unencodable_name: None, } } /// `Observation(name=None)`, as PySpark's `__init__`: the name is a non-empty string, and an - /// observation without one is of a random UUID once it is used. As PySpark's, it makes the - /// observation anew, unused. + /// observation without one is of a random UUID once it is used. As PySpark's, it sets the + /// attributes `_name`, `_result` and `_error` of the observation, which is unused. #[pyo3(signature = (name=None))] fn __init__(slf: &Bound<'_, Self>, name: Option<&Bound<'_, PyAny>>) -> PyResult<()> { let py = slf.py(); @@ -118,73 +79,40 @@ impl PyObservation { )); } } - let (inner, unencodable_name) = named(name)?; - let mut this = slf.borrow_mut(); - this.inner = inner; - this.restored = Restored::default(); - this.unencodable_name = unencodable_name; + slf.setattr(intern!(py, "_name"), name)?; + slf.setattr(intern!(py, "_result"), py.None())?; + slf.setattr(intern!(py, "_error"), py.None())?; + slf.borrow_mut().inner = Observation::unnamed(); Ok(()) } - /// The name of the observation, as PySpark's `_name`: None for an unnamed observation - /// before it is used, and a name that UTF-8 cannot encode as it was given. - #[getter(_name)] - fn name(&self, py: Python<'_>) -> Option> { - if let Some(name) = &self.unencodable_name { - return Some(name.clone_ref(py).into_any()); - } - self.inner - .name() - .map(|name| PyString::new(py, &name).into_any().unbind()) - } - - /// The observed metrics, as PySpark's `Observation.get`: its `_result`, the dict of the metrics - /// that the actions run so far observed, by their names, and empty before an action runs, - /// which is one dict, that the actions update in place, as PySpark's client updates it. The - /// metrics of an execution that PySpark's client does not convert (see - /// [`check_observed_metrics`]) are never set, so these are those of the executions before it, - /// as PySpark's. An unpickled Observation gives back its `_error` and its `_result`, as - /// PySpark's. + /// The observed metrics, as PySpark's `Observation.get`: its `_error`, raised if it is not + /// None, else its `_result`, the dict of the metrics that the actions run so far observed, by + /// their names, which the actions update in place, and `NO_OBSERVE_BEFORE_GET` while it is + /// None, before the observation is used. #[getter] - fn get<'py>(&self, py: Python<'py>) -> PyResult> { - if let Some(error) = &self.restored.error { - return Err(PyErr::from_value(error.bind(py).clone())); - } - if let Some(result) = &self.restored.result { - return Ok(result.bind(py).clone()); - } - // The error of collecting the metrics, or NO_OBSERVE_BEFORE_GET of an unused observation. - self.inner.get().to_pyerr()?; - match result_dict(py, &self.inner)? { - Some(result) => Ok(result.into_any()), - None => Ok(PyDict::new(py).into_any()), + fn get<'py>(slf: &Bound<'py, Self>) -> PyResult> { + let py = slf.py(); + let error = slf.getattr(intern!(py, "_error"))?; + if !error.is_none() { + return Err(PyErr::from_value(error)); } - } - - /// The result of the observation, as PySpark's `_result`: None before it is used, and then - /// the dict that `get` gives, which the actions that set its metrics update in place. - #[getter(_result)] - fn result<'py>(&self, py: Python<'py>) -> PyResult>> { - if let Some(result) = &self.restored.result { - return Ok(Some(result.bind(py).clone())); + let result = slf.getattr(intern!(py, "_result"))?; + if result.is_none() { + return Err(raise_pyspark( + py, + "PySparkAssertionError", + "NO_OBSERVE_BEFORE_GET", + &[], + )); } - Ok(result_dict(py, &self.inner)?.map(Bound::into_any)) + Ok(result) } - /// The error of collecting the metrics, which the server reported, as PySpark's `_error`, or - /// None: the exception that `get` raises. - #[getter(_error)] - fn error<'py>(&self, py: Python<'py>) -> Option> { - if let Some(error) = &self.restored.error { - return Some(error.bind(py).clone()); - } - let (_, error) = self.inner.state(); - error.map(|error| { - crate::errors::spark_error_to_py_exception(error) - .value(py) - .clone() - .into_any() - }) + /// Set the error of collecting the metrics, as PySpark's `_set_error`, which its client calls + /// when the server reports one: the `_error` that `get` raises. + fn _set_error(slf: &Bound<'_, Self>, exc: &Bound<'_, PyAny>) -> PyResult<()> { + slf.setattr(intern!(slf.py(), "_error"), exc) } /// Observe the metrics of the Columns `exprs` on `df` with this observation, as PySpark's @@ -199,85 +127,43 @@ impl PyObservation { .iter() .map(|c| c.column.expression().clone()) .collect(); - observe(slf.py(), &df, slf, exprs) + let observed = observe(slf.py(), &df, slf, exprs)?; + // The cached schema of the DataFrame, which observing does not change. + Ok(df.of_same_schema(observed)) } - /// `copy.copy`, as of PySpark's Observation, a plain object: an Observation of the class, the - /// name, the state and the attributes of this one, which shares the metrics of a used one, as - /// the copy of PySpark's shares its result dict (see `Observation::shallow_copy`). + /// `copy.copy`, as of PySpark's Observation, a plain object: an Observation of the class and + /// of the attributes of this one, the same objects, so the copy of a used one shares its + /// result dict, which the actions of this one update. fn __copy__<'py>(slf: &Bound<'py, Self>) -> PyResult> { - let py = slf.py(); - let this = slf.borrow(); - let copy = PyObservation { - inner: this.inner.shallow_copy(), - restored: this.restored.clone_ref(py), - unencodable_name: this - .unencodable_name - .as_ref() - .map(|name| name.clone_ref(py)), - }; - let copy = crate::reduce::new_of_class(&slf.get_type(), copy)?; + let copy = crate::reduce::new_of_class( + &slf.get_type(), + PyObservation { + inner: Observation::unnamed(), + }, + )?; copy_attributes(slf, ©, None)?; Ok(copy) } - /// `copy.deepcopy`, as of PySpark's Observation: an Observation of the class, the name and - /// the state of this one as they are now, which the actions of this one do not set, with deep - /// copies of its attributes. + /// `copy.deepcopy`, as of PySpark's Observation: an Observation of the class and of deep + /// copies of the attributes of this one, which the actions of this one do not set. The copy + /// is the Observation of the state that the deep copy of a DataFrame of the call holds, which + /// the actions of that copy set. fn __deepcopy__<'py>( slf: &Bound<'py, Self>, memo: &Bound<'py, PyAny>, ) -> PyResult> { - let py = slf.py(); let inner = slf.borrow().inner.detached(); - let unencodable_name = slf - .borrow() - .unencodable_name - .as_ref() - .map(|name| name.clone_ref(py)); - let copy = crate::reduce::new_of_class( - &slf.get_type(), - PyObservation { - inner, - restored: Restored::default(), - unencodable_name, - }, - )?; + let copy = crate::reduce::new_of_class(&slf.get_type(), PyObservation { inner })?; // The copy is the Observation of the state that the deep copy of a DataFrame of the call // holds, which a deep copy of that DataFrame copies with the copy. keep_handle(©)?; // As copy.deepcopy of a plain object, which records the copy in `memo` before it copies // the attributes, so that an attribute that refers to this observation refers to the copy. memo.set_item(slf.as_ptr() as usize, ©)?; - let deepcopy = py.import("copy")?.getattr("deepcopy")?; - let copied = |object: &Option>| -> PyResult>> { - object - .as_ref() - .map(|object| Ok(deepcopy.call1((object, memo))?.unbind())) - .transpose() - }; - let restored = { - let this = slf.borrow(); - Restored { - result: copied(&this.restored.result)?, - error: copied(&this.restored.error)?, - } - }; - copy.borrow_mut().restored = restored; - // As the deep copy of PySpark's `_result`, of the memo: a copy of the dict that `get` gave, - // with what was set on it, which the actions of this observation do not update. One that - // `get` did not give yet is made of the copied metrics when the copy's is read. - let result = slf.borrow().inner.metrics_mirror(); - if let Some(result) = result.as_deref().and_then(mirrored_dict) { - let copied = deepcopy.call1((result.bind(py), memo))?; - if let Ok(copied) = copied.cast_into::() { - let mirror: Arc = Arc::new(PyMetricsMirror { - dict: copied.unbind(), - }); - let _ = copy.borrow().inner.set_metrics_mirror(mirror); - } - } copy_attributes(slf, ©, Some(memo))?; + mirror_into(©)?; Ok(copy) } @@ -295,96 +181,93 @@ impl PyObservation { } /// The state that a pickled Observation keeps, that of PySpark's, its `__dict__`: `_name`, - /// `_result`, a dict of the metrics or None before it is used, `_error`, the exception that - /// `get` raises or None, and the attributes of this one. So on a Python worker of the server, - /// whose PySpark has no Rust client, it unpickles as PySpark's Observation, and here as this - /// one (see `__setstate__`). + /// `_result`, `_error` and the other attributes. So on a Python worker of the server, whose + /// PySpark has no Rust client, it unpickles as PySpark's Observation, and here as this one. fn __getstate__<'py>(slf: &Bound<'py, Self>) -> PyResult> { - let py = slf.py(); - let this = slf.borrow(); - let (_, error) = this.inner.state(); - let result = match &this.restored.result { - Some(result) => Some(result.bind(py).clone()), - None => result_dict(py, &this.inner)?.map(Bound::into_any), - }; - let error = match &this.restored.error { - Some(error) => Some(error.bind(py).clone()), - None => error.map(|error| { - crate::errors::spark_error_to_py_exception(error) - .value(py) - .clone() - .into_any() - }), - }; - let state = PyDict::new(py); - state.set_item("_name", this.name(py))?; - state.set_item("_result", result)?; - state.set_item("_error", error)?; - state.call_method1("update", (slf.getattr(pyo3::intern!(py, "__dict__"))?,))?; - Ok(state) + let attributes = slf.getattr(intern!(slf.py(), "__dict__"))?; + attributes.cast_into::()?.copy() } /// The Observation of the state of `__getstate__`, as pickle sets the `__dict__` of - /// PySpark's: of the name `_name`, whose `get` gives back `_error` and `_result`, which no - /// action sets, and so used if `_result` is not None, and of the other attributes. + /// PySpark's: of its attributes, so used if `_result` is not None, which no action sets. fn __setstate__<'py>(slf: &Bound<'py, Self>, state: &Bound<'py, PyDict>) -> PyResult<()> { let py = slf.py(); - let state = state.copy()?; - let take = |key: &str| -> PyResult>> { - let value = state.get_item(key)?; - if value.is_some() { - state.del_item(key)?; - } - Ok(value.filter(|value| !value.is_none())) - }; - let (inner, unencodable_name) = named(take("_name")?.as_ref())?; - let result = take("_result")?.map(Bound::unbind); - let error = take("_error")?.map(Bound::unbind); - { - let mut this = slf.borrow_mut(); - this.inner = inner; - this.unencodable_name = unencodable_name; - this.restored = Restored { result, error }; - } - slf.getattr(pyo3::intern!(py, "__dict__"))? - .call_method1(pyo3::intern!(py, "update"), (state,))?; + slf.borrow_mut().inner = Observation::unnamed(); + slf.getattr(intern!(py, "__dict__"))? + .call_method1(intern!(py, "update"), (state,))?; Ok(()) } - /// The `_result` and `_error` of an unpickled Observation, Python objects, which Python's - /// garbage collector sees, as those of PySpark's: so a reference cycle through them is freed. - /// Only `Option>` fields are read, which are `None` while the object is created, so no - /// [`crate::gc::GcReady`] is needed. - fn __traverse__(&self, visit: pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> { - if let Some(result) = &self.restored.result { - visit.call(result)?; - } - if let Some(error) = &self.restored.error { - visit.call(error)?; - } + /// The attributes of an Observation are those of its `__dict__`, which Python's garbage + /// collector sees, as those of PySpark's: so a reference cycle through them, as through its + /// result dict, is freed. The state of the core holds no Python object of it. + fn __traverse__(&self, _visit: pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> { Ok(()) } +} - /// Break a reference cycle that the garbage collector found unreachable. - fn __clear__(&mut self) { - self.restored = Restored::default(); - } +/// The mirror of the state of an Observation of the core of a used Python Observation, which the +/// actions set as PySpark's client sets the attributes of the Observation of their plans: they +/// read its `_result`, which must not be None, update it in place with the metrics, and give the +/// error of collecting them to its `_set_error`. Their errors are those of the actions. The mirror +/// holds the Python Observation weakly, as the plans of its DataFrames hold it. +struct PyObservationMirror { + observation: Py, } -/// The result dict of a used Observation, as PySpark's `_result`: the dict that `get` gives, -/// which the actions that set the metrics of the Observation update in place, as PySpark's client -/// does, the shallow copies of the Observation sharing it. -struct PyMetricsMirror { - dict: Py, +impl PyObservationMirror { + /// The Python Observation of this mirror, while it is alive. + fn observation<'py>(&self, py: Python<'py>) -> Option> { + self.observation.bind(py).upgrade() + } } -impl MetricsMirror for PyMetricsMirror { - fn update(&self, keys: &[String], values: &[spark_connect_proto::expression::Literal]) { +impl MetricsMirror for PyObservationMirror { + fn is_used(&self) -> Result { Python::attach(|py| { - // Of metrics that the check of the session found PySpark converts, before they were - // set (see `check_observed_metrics`). - let _ = set_metrics(py, self.dict.bind(py), keys.iter().zip(values)); - }); + let Some(observation) = self.observation(py) else { + return Ok(true); + }; + match observation.getattr(intern!(py, "_result")) { + Ok(result) => Ok(!result.is_none()), + Err(error) => Err(crate::errors::callback_error(error)), + } + }) + } + + fn update( + &self, + keys: &[String], + values: &[spark_connect_proto::expression::Literal], + ) -> Result<(), SparkError> { + Python::attach(|py| { + let Some(observation) = self.observation(py) else { + return Ok(()); + }; + let updated = || -> PyResult<()> { + let result = observation.getattr(intern!(py, "_result"))?; + // Of metrics that the check of the session found PySpark converts, before they + // were set (see `check_observed_metrics`). + let metrics = PyDict::new(py); + set_metrics(py, &metrics, keys.iter().zip(values))?; + result.call_method1(intern!(py, "update"), (metrics,))?; + Ok(()) + }; + updated().map_err(crate::errors::callback_error) + }) + } + + fn error(&self, error: &SparkError) -> Result<(), SparkError> { + Python::attach(|py| { + let Some(observation) = self.observation(py) else { + return Ok(()); + }; + let exception = crate::errors::spark_error_to_py_exception(error.clone()); + observation + .call_method1(intern!(py, "_set_error"), (exception.value(py),)) + .map(drop) + .map_err(crate::errors::callback_error) + }) } fn as_any(&self) -> &dyn std::any::Any { @@ -392,12 +275,31 @@ impl MetricsMirror for PyMetricsMirror { } } -/// The dict of `mirror`, of an Observation of this binding. -fn mirrored_dict(mirror: &dyn MetricsMirror) -> Option<&Py> { - mirror - .as_any() - .downcast_ref::() - .map(|mirror| &mirror.dict) +/// The state of the core of `observation`, an Observation that `SparkConnectClient` was given for +/// an execution, as PySpark's client sets the Observations of its `observations` by their +/// attributes: that of its last `_on`, else a used one, whose mirror reads its `_result`, so that +/// the execution updates the `_result` of an Observation that no `_on` used, such as one that a +/// pickle restored, and raises PySpark's `AssertionError` of one that is None. +pub(crate) fn state_of_execution(observation: &Bound<'_, PyObservation>) -> PyResult { + let inner = observation.borrow().inner.clone(); + if inner.metrics_mirror().is_some() { + return Ok(inner); + } + let used = Observation::used(&inner.name().unwrap_or_default()); + let weak = PyWeakrefReference::new(observation.as_any())?.unbind(); + let mirror: Arc = Arc::new(PyObservationMirror { observation: weak }); + let _ = used.set_metrics_mirror(mirror); + Ok(used) +} + +/// Make the attributes of `observation` those that the actions set of the state of its core, +/// if that state is used: the metrics that the actions set before the mirror, which a deep copy +/// keeps in its copied `_result`, are not set again. +fn mirror_into(observation: &Bound<'_, PyObservation>) -> PyResult<()> { + let weak = PyWeakrefReference::new(observation.as_any())?.unbind(); + let mirror: Arc = Arc::new(PyObservationMirror { observation: weak }); + let _ = observation.borrow().inner.set_metrics_mirror(mirror); + Ok(()) } /// Set the metrics `metrics` in `dict` by their names, as PySpark's client updates the result of @@ -413,33 +315,6 @@ fn set_metrics<'a>( Ok(()) } -/// The result dict of `observation`, PySpark's `_result`, or None before it is used: the one that -/// was made of it, else a new one of its metrics so far, which the actions update from now on. -fn result_dict<'py>( - py: Python<'py>, - observation: &Observation, -) -> PyResult>> { - if let Some(mirror) = observation.metrics_mirror() { - if let Some(dict) = mirrored_dict(mirror.as_ref()) { - return Ok(Some(dict.bind(py).clone())); - } - } - let dict = PyDict::new(py); - let made = Arc::new(PyMetricsMirror { - dict: dict.clone().unbind(), - }); - let Some((mirror, metrics)) = observation.set_metrics_mirror(made) else { - return Ok(None); - }; - if let Some(metrics) = metrics { - set_metrics(py, &dict, metrics.iter().map(|(name, value)| (name, value)))?; - } - Ok(Some(match mirrored_dict(mirror.as_ref()) { - Some(kept) => kept.bind(py).clone(), - None => dict, - })) -} - /// Keep a weak reference to `observation` on its state, by which the deep copy of a DataFrame /// whose plan holds the state finds it (see [`deep_copy_in`]). fn keep_handle(observation: &Bound<'_, PyObservation>) -> PyResult<()> { @@ -489,8 +364,9 @@ fn held_by_the_plan(observation: &Bound<'_, PyObservation>) -> Observation { /// The Python Observation of `observation`, a state that the plan of a DataFrame holds, as the /// Observation that PySpark's plan holds itself: the one that the plan keeps alive, or that of -/// [`keep_handle`] while it is alive, else a new Observation of the state, whose `get` the actions -/// of the plan set as they set the state, and which later reads give back while it is alive. +/// [`keep_handle`] while it is alive, else a new Observation of the state, of its name, of a +/// result of its metrics once it is used and of its error, which the actions of the plan set from +/// now on, and which later reads give back while it is alive. pub(crate) fn python_observation_of<'py>( py: Python<'py>, observation: &Observation, @@ -505,10 +381,29 @@ pub(crate) fn python_observation_of<'py>( py, PyObservation { inner: observation.clone(), - restored: Restored::default(), - unencodable_name: None, }, )?; + let (_, error) = observation.state(); + python.setattr(intern!(py, "_name"), observation.name())?; + python.setattr(intern!(py, "_result"), py.None())?; + let error = error.map(|error| { + crate::errors::spark_error_to_py_exception(error) + .value(py) + .clone() + }); + python.setattr(intern!(py, "_error"), error)?; + // The metrics so far, and the mirror of those that the actions set from now on. + let weak = PyWeakrefReference::new(python.as_any())?.unbind(); + let mirror: Arc = Arc::new(PyObservationMirror { observation: weak }); + if let Some(metrics) = observation.set_metrics_mirror(mirror) { + let result = PyDict::new(py); + set_metrics( + py, + &result, + metrics.iter().map(|(name, value)| (name, value)), + )?; + python.setattr(intern!(py, "_result"), result)?; + } keep_handle(&python)?; Ok(python) } @@ -557,8 +452,10 @@ fn copy_attributes( } /// `df` observing the metrics `exprs` with `observation`, as PySpark's `Observation._on`, which -/// `DataFrame.observe` calls. The actions of the DataFrames of it set the metrics, as those of -/// an Observation of their plans. +/// `DataFrame.observe` calls: `REUSE_OBSERVATION` of an observation whose `_result` is not None, +/// a random UUID as the `_name` of an unnamed one, the check that `df` is not streaming, and then +/// an empty `_result`, of the observation of a new state of the core. The actions of the +/// DataFrames of it set the metrics, as those of an Observation of their plans. pub(crate) fn observe( py: Python<'_>, frame: &PyDataFrame, @@ -566,8 +463,7 @@ pub(crate) fn observe( mut exprs: Vec, ) -> PyResult { let df = &frame.dataframe; - // An Observation unpickled from a used one is used, as PySpark's of a `_result`. - if observation.borrow().restored.result.is_some() { + if !observation.getattr(intern!(py, "_result"))?.is_none() { return Err(raise_pyspark( py, "PySparkAssertionError", @@ -575,18 +471,25 @@ pub(crate) fn observe( &[], )); } - // A name that UTF-8 cannot encode fails the requests of the DataFrame, as PySpark's - // `CollectMetrics` sets it when it builds the plan, before the metrics. - let mut args = crate::plan_args::PlanArgs::new(); - if let Some(name) = &observation.borrow().unencodable_name { - args.str(name.bind(py).as_any())?; + let mut name = observation.getattr(intern!(py, "_name"))?; + if name.is_none() { + name = py.import("uuid")?.call_method0("uuid4")?.str()?.into_any(); + observation.setattr(intern!(py, "_name"), &name)?; } + // A name that protobuf does not take, as one that UTF-8 cannot encode, fails the requests of + // the DataFrame, as PySpark's `CollectMetrics` sets it when it builds the plan, before the + // metrics. + let mut args = crate::plan_args::PlanArgs::new(); + let name = args.str(&name)?; let args = args.before(&mut exprs); + observation.borrow_mut().inner = Observation::new(&name); let inner = held_by_the_plan(observation); // As PySpark's `Observation._on`, which reads the `isStreaming` of the DataFrame, asked once. let observed = crate::interrupt::interruptible(py, &df.spark_session(), || { df.observe_with_streaming(&inner, exprs, || frame.streaming_once()) })?; + observation.setattr(intern!(py, "_result"), PyDict::new(py))?; + mirror_into(observation)?; // The plan holds the Observation itself, as PySpark's does (see `held_by_the_plan`), which a // deep copy of the plan copies with the memo of the call. The clones of its state that do not // keep it alive find it by its weak reference. diff --git a/crates/spark-connect-core/src/client.rs b/crates/spark-connect-core/src/client.rs index 61f00aed..f16ae56f 100644 --- a/crates/spark-connect-core/src/client.rs +++ b/crates/spark-connect-core/src/client.rs @@ -1722,20 +1722,24 @@ impl SparkConnectClient { if known != PlanCompression::Unknown { return known; } - let looked_up = match self - .get_configs(&[ - PLAN_COMPRESSION_THRESHOLD_CONF, - PLAN_COMPRESSION_ALGORITHM_CONF, - ]) - .await + // As PySpark's `except Exception` of the lookup, which catches the exception that a signal + // handler raises while the lookup waits, unless it is a `BaseException` only, such as the + // KeyboardInterrupt of Ctrl-C: then the settings are those of a failed lookup, kept for the + // later plans, and the call goes on (see `runtime::catching`). + let looked_up = match crate::runtime::catching(self.get_configs(&[ + PLAN_COMPRESSION_THRESHOLD_CONF, + PLAN_COMPRESSION_ALGORITHM_CONF, + ])) + .await { - Ok(values) => match values.as_slice() { + None => PlanCompression::Disabled, + Some(Ok(values)) => match values.as_slice() { [threshold, algorithm] => { plan_compression_of(threshold.as_deref(), algorithm.as_deref()) } _ => PlanCompression::Disabled, }, - Err(error) => { + Some(Err(error)) => { let aborted = is_wait_aborted(&error); // As PySpark's `except Exception` of the lookup, which catches its failure, and the // exception of the warning of its retries once they are spent. @@ -2451,19 +2455,21 @@ impl SparkConnectClient { self.observe_on(&mut request.client_observed_server_side_session_id); let request = &request; let mut retries = Retrying::new(&self.retry_policy()).ending_with_abort(); - let attempts = self - .retried(&mut retries, || { - let deadline = self.rpc_deadlines.config; - let mut req = with_deadline(Request::new(request.clone()), deadline); - self._attach_metadata(&mut req); - let mut stub = self.stub.clone(); - async move { - within_deadline(deadline, stub.config(req)) - .await - .map(Response::into_inner) - } - }) - .await; + // As the `except Exception` of `_display_server_stack_trace`, which catches the exception + // that a signal handler raises while the config is read, unless it is a `BaseException` + // only: the config is then not read (see `runtime::catching`). + let attempts = crate::runtime::catching(self.retried(&mut retries, || { + let deadline = self.rpc_deadlines.config; + let mut req = with_deadline(Request::new(request.clone()), deadline); + self._attach_metadata(&mut req); + let mut stub = self.stub.clone(); + async move { + within_deadline(deadline, stub.config(req)) + .await + .map(Response::into_inner) + } + })) + .await?; let response = match attempts { Ok(response) => response, Err(RetryError::Exceeded(_)) => { diff --git a/crates/spark-connect-core/src/runtime.rs b/crates/spark-connect-core/src/runtime.rs index e6b9eb87..df66e741 100644 --- a/crates/spark-connect-core/src/runtime.rs +++ b/crates/spark-connect-core/src/runtime.rs @@ -138,6 +138,60 @@ pub async fn until_abort(level: u8, f: F) -> Option>> = + const { RefCell::new(Vec::new()) }; +} + +/// Pops the scope of [`catching`] that it was made for, when the scope ends or is dropped. +struct CatchingScope; + +impl Drop for CatchingScope { + fn drop(&mut self) { + CATCHING.with(|scopes| { + scopes.borrow_mut().pop(); + }); + } +} + +/// `f`, a part of a wait whose caller catches the exception of a signal handler that is not a +/// `BaseException` only, as the `except Exception` of PySpark's lookup of the plan compression +/// settings catches the exception that a handler raises while the lookup waits: `f` ends at +/// once, unfinished, with None, when the watch of the wait ends the scope with [`catch_signal`] +/// while `f` runs. The wait goes on after it, and the later signals are those of the wait. Outside +/// a wait that a watch polls, `f` is awaited as it is. Not a stable API. +#[doc(hidden)] +pub async fn catching(f: F) -> Option { + if WAIT_WATCH.with(|watch| watch.borrow().is_none()) { + return Some(f.await); + } + let (sender, mut receiver) = tokio::sync::watch::channel(false); + CATCHING.with(|scopes| scopes.borrow_mut().push(sender)); + let _scope = CatchingScope; + tokio::pin!(f); + tokio::select! { + biased; + _ = receiver.wait_for(|caught| *caught) => None, + output = &mut f => Some(output), + } +} + +/// End the innermost scope of [`catching`] that a wait of the calling thread runs, for the +/// exception of a signal handler that the caller of the scope catches: true then, and the watch +/// that calls it lets the wait go on; false when no such scope runs. Not a stable API. +#[doc(hidden)] +pub fn catch_signal() -> bool { + CATCHING.with(|scopes| match scopes.borrow().last() { + Some(scope) => { + scope.send_replace(true); + true + } + None => false, + }) +} + /// Run `f` on this thread, with `watch` called by the waits for the server that `f` makes on this /// thread, as [`block_on`] makes them: the waits poll `watch` every [`POLL_INTERVAL`] while the /// future of each waits. A wait on another thread (see [`block_on`]) does not poll it. The watch of @@ -787,6 +841,41 @@ mod tests { assert_eq!(block_on(until_abort(ABORT_ALL, async { 8 })), Some(8)); } + #[test] + fn test_a_catching_scope_ends_for_a_caught_signal_and_the_wait_goes_on() { + // A watch whose signal the scope of the wait catches ends that scope, unfinished, and the + // wait goes on to its end; a watch that catches nothing finds no scope outside one. + let caught = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let flag = Arc::clone(&caught); + let watch: Arc = Arc::new(move || { + if !flag.swap(true, std::sync::atomic::Ordering::SeqCst) { + assert!(catch_signal()); + } + Watch::Continue + }); + let (scoped, after) = with_wait_watch(watch, || { + block_on(async { + let scoped = catching(std::future::pending::()).await; + tokio::time::sleep(POLL_INTERVAL * 2).await; + (scoped, 7) + }) + }); + assert_eq!((scoped, after), (None, 7)); + assert!(caught.load(std::sync::atomic::Ordering::SeqCst)); + assert!(!catch_signal()); + // A scope that the watch does not end gives the output of its future, and outside a watch + // the future is awaited as it is. + let ended = with_wait_watch(Arc::new(|| Watch::Continue), || { + block_on(catching(async { + tokio::time::sleep(POLL_INTERVAL * 2).await; + 8 + })) + }); + assert_eq!(ended, Some(8)); + assert_eq!(block_on(catching(async { 9 })), Some(9)); + assert!(CATCHING.with(|scopes| scopes.borrow().is_empty())); + } + #[test] fn test_with_wait_watch_restores_the_enclosing_watch() { let outer: Arc = Arc::new(|| Watch::Continue); diff --git a/crates/spark-connect-core/tests/server_session.rs b/crates/spark-connect-core/tests/server_session.rs index 403b5015..056c6d00 100644 --- a/crates/spark-connect-core/tests/server_session.rs +++ b/crates/spark-connect-core/tests/server_session.rs @@ -53,6 +53,8 @@ struct State { failures: Mutex>, /// The method and the bytes of every request, in the order they arrived. requests: Mutex)>>, + /// Whether the Config calls are not answered, as of a server that hangs. + hang_config: Mutex, } impl State { @@ -101,6 +103,7 @@ impl Fixture { drop_next_execute: Mutex::new(false), failures: Mutex::new(VecDeque::new()), requests: Mutex::new(Vec::new()), + hang_config: Mutex::new(false), }); let server = service(state.clone()).serve().await; let address = server.address; @@ -218,6 +221,9 @@ impl UnaryService> for Unary { if let Some(status) = self.0.failure(&self.1) { return Box::pin(async { Err(status) }); } + if self.1 == "Config" && *self.0.hang_config.lock().unwrap() { + return Box::pin(std::future::pending()); + } let session_id = session_of(&self.1, &bytes); let server_side_session_id = self.0.server_session_id(); let body = match self.1.as_str() { @@ -794,6 +800,65 @@ async fn a_set_client_session_id_is_that_of_the_calls_and_not_of_the_artifacts_a ); } +#[test] +fn a_signal_that_the_plan_compression_lookup_catches_ends_it_and_the_execution_goes_on() { + // As the `except Exception` of PySpark's lookup of the plan compression settings, which + // catches the exception that a signal handler raises while the lookup waits: the lookup of a + // server that does not answer it ends, the compression is disabled for the client, which looks + // nothing up again, and the execution goes on, as the later executions do. + use spark_connect_core::runtime::{block_on, catch_signal, with_wait_watch, WaitWatch, Watch}; + use spark_connect_proto::{plan, Plan, Relation}; + use std::sync::atomic::{AtomicBool, Ordering}; + + let runtime = spark_connect_core::get_runtime(); + let fixture = runtime.block_on(Fixture::start()); + let client = runtime.block_on(fixture.client()); + *fixture.state.hang_config.lock().unwrap() = true; + let caught = Arc::new(AtomicBool::new(false)); + let flag = Arc::clone(&caught); + let state = Arc::clone(&fixture.state); + // A signal whose exception the caller of the lookup catches, once the lookup waits for the + // server. + let watch: Arc = Arc::new(move || { + let waits = state + .requests + .lock() + .unwrap() + .iter() + .any(|(m, _)| m == "Config"); + if waits && !flag.load(Ordering::SeqCst) && catch_signal() { + flag.store(true, Ordering::SeqCst); + } + Watch::Continue + }); + let execute = || { + let execution = ExecutePlanRequest { + plan: Some(Plan { + op_type: Some(plan::OpType::Root(Relation::default())), + }), + ..execute_request(&client) + }; + with_wait_watch(Arc::clone(&watch), || { + block_on(async { + let mut stream = client.execute_plan(execution).await.unwrap(); + while stream.message().await.unwrap().is_some() {} + }) + }) + }; + execute(); + assert!(caught.load(Ordering::SeqCst)); + execute(); + let methods: Vec = fixture + .state + .requests + .lock() + .unwrap() + .iter() + .map(|(method, _)| method.clone()) + .collect(); + assert_eq!(methods, ["Config", "ExecutePlan", "ExecutePlan"]); +} + #[tokio::test] async fn executions_ask_the_server_to_chunk_arrow_batches() { use spark_connect_proto::execute_plan_request::request_option::RequestOption; diff --git a/crates/spark-connect/src/dataframe.rs b/crates/spark-connect/src/dataframe.rs index f2ec88e5..07fd8a6e 100644 --- a/crates/spark-connect/src/dataframe.rs +++ b/crates/spark-connect/src/dataframe.rs @@ -3722,7 +3722,10 @@ fn capture_execution( /// set, as PySpark's client raises when it converts them. Before that, the metrics of an /// Observation that is not used, which `observations` may hold when the caller gives them, as /// the `observations` of `SparkConnectClient.to_table`, are the `AssertionError` of PySpark's -/// client, which asserts that the result of the Observation is not None. +/// client, which asserts that the result of the Observation is not None. The error of the mirror +/// of a binding of an Observation, which is told of its metrics and of its error (see +/// [`crate::observation::MetricsMirror`]), is returned, as PySpark's client raises the error of +/// the update of the result of an Observation, or of its `_set_error`. fn set_observations( session: &SparkSession, observations: &[(String, crate::observation::Observation)], @@ -3740,7 +3743,7 @@ fn set_observations( observation.set_error(SparkError::from_observation_errors( root_error_idx, observed.errors.clone(), - )); + ))?; } } None if observed.name == crate::profiler::PYTHON_ACCUMULATOR_METRICS => session @@ -3748,13 +3751,13 @@ fn set_observations( .accumulate(std::slice::from_ref(observed)), None => { if let Some(observation) = observation { - if !observation.is_used() { + if !observation.is_used()? { return Err(SparkError::builtin_assertion()); } if let Some(check) = &check { check(observed)?; } - observation.update(&observed.keys, &observed.values); + observation.update(&observed.keys, &observed.values)?; } } } @@ -7279,7 +7282,7 @@ mod plan_construction_tests { ); assert_eq!(err.operation_id.as_deref(), Some("op-1")); assert!(used.get().unwrap().is_empty()); - assert!(!unused.is_used()); + assert!(!unused.is_used().unwrap()); let mut failed = observed("u"); failed.root_error_idx = Some(0); diff --git a/crates/spark-connect/src/observation.rs b/crates/spark-connect/src/observation.rs index 6aeda80a..cf919e66 100644 --- a/crates/spark-connect/src/observation.rs +++ b/crates/spark-connect/src/observation.rs @@ -24,14 +24,32 @@ pub struct Observation { /// The metrics of an observation, by their names, in order. type Metrics = Vec<(String, proto::expression::Literal)>; -/// What a binding of another language keeps of the metrics of a used observation, such as the -/// result dict of the Python client, which the observation tells of each update of its metrics, -/// as PySpark's client updates the result dict of an Observation in place. The shallow copies of -/// the observation share it with its metrics. Not a stable API. +/// What a binding of another language keeps of the state of a used observation, such as the +/// Python Observation of the Python client, whose `_result` and `_error` the observation sets as +/// PySpark's client sets them: it is asked whether the observation is used, and told of each +/// update of the metrics and of the error that the server reports. The shallow copies of the +/// observation share it with its metrics. Not a stable API. #[doc(hidden)] pub trait MetricsMirror: Send + Sync { - /// The metrics of the names `keys` were set to `values` by an action, in their order. - fn update(&self, keys: &[String], values: &[proto::expression::Literal]); + /// Whether the observation of the binding is used, as PySpark's client asserts that the + /// `_result` of an Observation is not None before it sets its metrics, which a program may + /// have set to None: its error, such as the `AttributeError` of a `_result` that the program + /// deleted, is that of the action. Used, unless the binding says otherwise. + fn is_used(&self) -> Result { + Ok(true) + } + + /// The metrics of the names `keys` were set to `values` by an action, in their order: its + /// error is that of the action, as PySpark's client raises the error of the update of the + /// result of an Observation. + fn update(&self, keys: &[String], values: &[proto::expression::Literal]) -> Result<()>; + + /// The server reported `error` of collecting the metrics: its error is that of the action, as + /// PySpark's client raises the error of the `_set_error` of an Observation. + fn error(&self, error: &SparkError) -> Result<()> { + let _ = error; + Ok(()) + } /// This mirror, by which the binding finds its own type. fn as_any(&self) -> &dyn std::any::Any; @@ -42,7 +60,17 @@ pub trait MetricsMirror: Send + Sync { #[derive(Default)] struct UsedMetrics { metrics: Mutex, - mirror: std::sync::OnceLock>, + mirror: Mutex>>, +} + +impl UsedMetrics { + /// The mirror of a binding, if it set one. + fn mirror(&self) -> Option> { + self.mirror + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone() + } } #[derive(Default)] @@ -84,6 +112,16 @@ impl Observation { Self::default() } + /// A used observation of the name `name`, of no metrics until an action observes them, as an + /// observation once a DataFrame observes by it, for a binding whose own object of it says + /// whether it is used (see [`MetricsMirror::is_used`]). Not a stable API. + #[doc(hidden)] + pub fn used(name: &str) -> Self { + let observation = Self::new(name); + observation.start(); + observation + } + /// The name of the observation; None for an unnamed observation before it is used. pub fn name(&self) -> Option { self.lock().name.clone() @@ -126,47 +164,51 @@ impl Observation { (result, state.error.clone()) } - /// Whether the observation is used, as PySpark's Observation once its `_result` is not None. - pub(crate) fn is_used(&self) -> bool { - self.lock().result.is_some() + /// Whether the observation is used, as PySpark's Observation once its `_result` is not None: + /// the mirror of a binding, if it set one, says it (see [`MetricsMirror::is_used`]). + pub(crate) fn is_used(&self) -> Result { + let Some(result) = self.lock().result.clone() else { + return Ok(false); + }; + match result.mirror() { + Some(mirror) => mirror.is_used(), + None => Ok(true), + } } - /// The mirror of the metrics of this observation that a binding set (see + /// The mirror of the state of this observation that a binding set (see /// [`Observation::set_metrics_mirror`]), if any. Not a stable API. #[doc(hidden)] pub fn metrics_mirror(&self) -> Option> { - let result = self.lock().result.clone()?; - result.mirror.get().cloned() + self.lock().result.clone()?.mirror() } - /// Make `mirror` the mirror of the metrics of this used observation (see [`MetricsMirror`]), - /// unless a binding set one already: the mirror that it has then, and, if it is `mirror`, the - /// metrics set so far, which the caller fills it with, as each update after it is told to it. - /// None before the observation is used. No lock of the observation is held while the caller - /// fills the mirror, or while a mirror is told of an update, so either may take a lock of its - /// own. Not a stable API. + /// Make `mirror` the mirror of the state of this used observation (see [`MetricsMirror`]), + /// in the place of the one that a binding set before, if any: the metrics set so far, which + /// the caller fills it with, as each update after it is told to it. None before the + /// observation is used. No lock of the observation is held while the caller fills the + /// mirror, or while a mirror is told of an update, so either may take a lock of its own. Not + /// a stable API. #[doc(hidden)] - #[allow(clippy::type_complexity)] pub fn set_metrics_mirror( &self, mirror: Arc, - ) -> Option<( - Arc, - Option>, - )> { + ) -> Option> { let result = self.lock().result.clone()?; // Set under the lock of the metrics, so that an update is either in the metrics given to // the caller or told to the mirror. - let (kept, metrics, rejected) = { + let (metrics, replaced) = { let metrics = lock_metrics(&result.metrics); - match result.mirror.set(mirror) { - Ok(()) => (result.mirror.get().cloned(), Some(metrics.clone()), None), - Err(rejected) => (result.mirror.get().cloned(), None, Some(rejected)), - } + let replaced = result + .mirror + .lock() + .unwrap_or_else(|e| e.into_inner()) + .replace(mirror); + (metrics.clone(), replaced) }; // Dropped once the lock is released, as the drop of a mirror may run code of the binding. - drop(rejected); - kept.map(|kept| (kept, metrics)) + drop(replaced); + Some(metrics) } /// A clone of this observation that keeps `owner` alive while it is, as the plan of an @@ -273,10 +315,15 @@ impl Observation { } /// Set the metrics that an action observed, as PySpark's client updates the result of - /// the observation with them: a metric replaces the value of a metric of its name. - pub(crate) fn update(&self, keys: &[String], values: &[proto::expression::Literal]) { + /// the observation with them: a metric replaces the value of a metric of its name. The error + /// of the mirror of a binding, which is told of them, is returned. + pub(crate) fn update( + &self, + keys: &[String], + values: &[proto::expression::Literal], + ) -> Result<()> { let Some(result) = self.lock().result.clone() else { - return; + return Ok(()); }; { let mut metrics = lock_metrics(&result.metrics); @@ -289,14 +336,24 @@ impl Observation { } // Told once the locks are released, as a mirror may take a lock of its own, such as the // GIL of Python, under which a thread may want the locks of the observation. - if let Some(mirror) = result.mirror.get() { - mirror.update(keys, values); + match result.mirror() { + Some(mirror) => mirror.update(keys, values), + None => Ok(()), } } - /// Set the error of collecting the metrics, which `get` returns. - pub(crate) fn set_error(&self, error: SparkError) { - self.lock().error = Some(error); + /// Set the error of collecting the metrics, which `get` returns. The error of the mirror of a + /// binding, which is told of it, is returned. + pub(crate) fn set_error(&self, error: SparkError) -> Result<()> { + let mirror = { + let mut state = self.lock(); + state.error = Some(error.clone()); + state.result.as_ref().and_then(|result| result.mirror()) + }; + match mirror { + Some(mirror) => mirror.error(&error), + None => Ok(()), + } } fn lock(&self) -> MutexGuard<'_, ObservationState> { @@ -350,8 +407,11 @@ mod tests { let clone = obs.clone(); assert!(clone.same(&obs) && !Observation::unnamed().same(&obs)); - clone.update(&["a".to_string(), "b".to_string()], &[long(1), long(2)]); - obs.update(&["b".to_string(), "c".to_string()], &[long(3), long(4)]); + clone + .update(&["a".to_string(), "b".to_string()], &[long(1), long(2)]) + .unwrap(); + obs.update(&["b".to_string(), "c".to_string()], &[long(3), long(4)]) + .unwrap(); let names: Vec<(String, Option)> = obs .get() .unwrap() @@ -367,7 +427,7 @@ mod tests { ] ); - obs.set_error(SparkError::connect_msg("failed")); + obs.set_error(SparkError::connect_msg("failed")).unwrap(); assert!(obs.get().unwrap_err().message().contains("failed")); assert!(format!("{obs:?}").contains(&name)); } @@ -385,10 +445,11 @@ mod tests { assert!(obs.get().is_err()); obs.start(); - obs.update(&["a".to_string()], &[long(1)]); + obs.update(&["a".to_string()], &[long(1)]).unwrap(); let copy = obs.detached(); - copy.update(&["a".to_string(), "b".to_string()], &[long(2), long(3)]); - obs.update(&["c".to_string()], &[long(4)]); + copy.update(&["a".to_string(), "b".to_string()], &[long(2), long(3)]) + .unwrap(); + obs.update(&["c".to_string()], &[long(4)]).unwrap(); let values = |o: &Observation| -> Vec<(String, Option)> { o.get() .unwrap() @@ -414,7 +475,7 @@ mod tests { // A failed observation keeps its error in the copy. let failed = Observation::new("f"); failed.start(); - failed.set_error(SparkError::connect_msg("failed")); + failed.set_error(SparkError::connect_msg("failed")).unwrap(); assert!(failed .detached() .get() @@ -443,31 +504,52 @@ mod tests { // PySpark's copy of an Observation shares its result dict, and is used already. let obs = Observation::new("s"); obs.start(); - obs.update(&["a".to_string()], &[long(1)]); + obs.update(&["a".to_string()], &[long(1)]).unwrap(); let copy = obs.shallow_copy(); assert!(!copy.same(&obs)); assert_eq!(copy.name().as_deref(), Some("s")); - obs.update(&["b".to_string()], &[long(2)]); + obs.update(&["b".to_string()], &[long(2)]).unwrap(); assert_eq!(names(©), vec!["a", "b"]); let err = copy.name_to_observe().unwrap_err(); assert_eq!(err.get_condition().as_deref(), Some("REUSE_OBSERVATION")); // An error that an action reports later is set on the observation only, and a copy of a // failed observation keeps its error. - obs.set_error(SparkError::connect_msg("failed")); + obs.set_error(SparkError::connect_msg("failed")).unwrap(); assert!(obs.get().is_err()); assert_eq!(names(©), vec!["a", "b"]); let failed = obs.shallow_copy().get().unwrap_err(); assert!(failed.message().contains("failed")); } - /// A mirror that records what it is told, as the result dict of a binding. + /// A mirror that records what it is told, as the Python Observation of a binding, and fails + /// as it is told to. #[derive(Default)] - struct Recorded(Mutex>); + struct Recorded { + told: Mutex>, + unused: std::sync::atomic::AtomicBool, + failing: std::sync::atomic::AtomicBool, + } impl MetricsMirror for Recorded { - fn update(&self, keys: &[String], _values: &[proto::expression::Literal]) { - self.0.lock().unwrap().extend(keys.iter().cloned()); + fn is_used(&self) -> Result { + Ok(!self.unused.load(std::sync::atomic::Ordering::SeqCst)) + } + + fn update(&self, keys: &[String], _values: &[proto::expression::Literal]) -> Result<()> { + if self.failing.load(std::sync::atomic::Ordering::SeqCst) { + return Err(SparkError::connect_msg("the update failed")); + } + self.told.lock().unwrap().extend(keys.iter().cloned()); + Ok(()) + } + + fn error(&self, error: &SparkError) -> Result<()> { + self.told + .lock() + .unwrap() + .push(format!("error {}", error.message())); + Ok(()) } fn as_any(&self) -> &dyn std::any::Any { @@ -475,48 +557,67 @@ mod tests { } } - fn recorded(mirror: &Arc) -> Vec { - let recorded = mirror.as_any().downcast_ref::().unwrap(); - recorded.0.lock().unwrap().clone() + fn recorded(mirror: &Arc) -> &Recorded { + mirror.as_any().downcast_ref::().unwrap() + } + + fn told(mirror: &Arc) -> Vec { + recorded(mirror).told.lock().unwrap().clone() } #[test] - fn a_mirror_is_set_once_of_the_metrics_and_told_of_each_update() { - // An unused observation has no mirror, as PySpark's has no result dict before `_on`. + fn a_mirror_is_told_of_each_update_and_error_and_says_whether_it_is_used() { + // An unused observation has no mirror, as PySpark's has no result before `_on`. let obs = Observation::new("m"); - assert!(!obs.is_used()); + assert!(!obs.is_used().unwrap()); let first: Arc = Arc::new(Recorded::default()); assert!(obs.set_metrics_mirror(Arc::clone(&first)).is_none()); obs.start(); - assert!(obs.is_used()); - obs.update(&["a".to_string()], &[long(1)]); + assert!(obs.is_used().unwrap()); + obs.update(&["a".to_string()], &[long(1)]).unwrap(); assert!(obs.metrics_mirror().is_none()); - // It is set once, with the metrics so far, which the caller fills it with: a second one is - // not set, and the first is given back, without metrics. - let (mirror, metrics) = obs.set_metrics_mirror(Arc::clone(&first)).unwrap(); - assert!(Arc::ptr_eq(&mirror, &first)); - let names: Vec = metrics.unwrap().into_iter().map(|(name, _)| name).collect(); + // It is set with the metrics so far, which the caller fills it with. + let metrics = obs.set_metrics_mirror(Arc::clone(&first)).unwrap(); + let names: Vec = metrics.into_iter().map(|(name, _)| name).collect(); assert_eq!(names, vec!["a"]); - let second: Arc = Arc::new(Recorded::default()); - let (again, none) = obs.set_metrics_mirror(Arc::clone(&second)).unwrap(); - assert!(Arc::ptr_eq(&again, &first) && none.is_none()); - assert_eq!(Arc::strong_count(&second), 1); assert!(Arc::ptr_eq(&obs.metrics_mirror().unwrap(), &first)); // Each update is told to it, after the metrics are set, also through a clone, and a - // shallow copy shares it, as PySpark's copy shares its result dict; a detached copy does - // not. - obs.update(&["b".to_string(), "c".to_string()], &[long(2), long(3)]); - obs.clone().update(&["a".to_string()], &[long(4)]); - assert_eq!(recorded(&mirror), vec!["b", "c", "a"]); + // shallow copy shares it; a detached copy does not. + obs.update(&["b".to_string(), "c".to_string()], &[long(2), long(3)]) + .unwrap(); + obs.clone().update(&["a".to_string()], &[long(4)]).unwrap(); + assert_eq!(told(&first), vec!["b", "c", "a"]); let shallow = obs.shallow_copy(); - assert!(Arc::ptr_eq(&mirror, &shallow.metrics_mirror().unwrap())); + assert!(Arc::ptr_eq(&first, &shallow.metrics_mirror().unwrap())); let detached = obs.detached(); assert!(detached.metrics_mirror().is_none()); - detached.update(&["d".to_string()], &[long(5)]); - assert_eq!(recorded(&mirror), vec!["b", "c", "a"]); + detached.update(&["d".to_string()], &[long(5)]).unwrap(); + assert_eq!(told(&first), vec!["b", "c", "a"]); assert_eq!(detached.get().unwrap().len(), 4); + + // It says whether the observation is used, and its errors are those of the update. + recorded(&first) + .unused + .store(true, std::sync::atomic::Ordering::SeqCst); + assert!(!obs.is_used().unwrap()); + recorded(&first) + .failing + .store(true, std::sync::atomic::Ordering::SeqCst); + let failed = obs.update(&["e".to_string()], &[long(6)]).unwrap_err(); + assert!(failed.message().contains("the update failed")); + + // Another mirror replaces it, of the metrics so far, and is told of the error. + let second: Arc = Arc::new(Recorded::default()); + assert_eq!( + obs.set_metrics_mirror(Arc::clone(&second)).unwrap().len(), + 4 + ); + assert!(Arc::ptr_eq(&obs.metrics_mirror().unwrap(), &second)); + obs.set_error(SparkError::connect_msg("failed")).unwrap(); + assert_eq!(told(&second), vec!["error failed"]); + assert!(obs.get().unwrap_err().message().contains("failed")); } #[test] diff --git a/dev/parity/known-differences.md b/dev/parity/known-differences.md index bfbbcba8..ca63b45c 100644 --- a/dev/parity/known-differences.md +++ b/dev/parity/known-differences.md @@ -563,7 +563,15 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan readers, writers, builders and the other objects of a session or a DataFrame hold their state in the Rust core: their `__dict__` holds the attributes that a user sets on them, which their copies and pickles keep as PySpark's do, but not their state, which PySpark's - holds there (`vars(spark.streams)` is empty, where PySpark's has `_session` and `_sqlb`). + holds there (`vars(spark.streams)` is empty, where PySpark's has `_session` and `_sqlb`). An + `Observation` is not one of them: its `_name`, `_result` and `_error` are of its `__dict__`, as + PySpark's. +- **The name of an Observation renamed after `observe`** (intentional; the plan is native). The + `CollectMetrics` of a DataFrame that an Observation observes is of the name that the + Observation had then, which the Rust core keeps in the plan, where PySpark's plan reads the + Observation's `_name` each time it is built: after `o._name = "other"`, the plan that the + server explains names the first name, where PySpark's names `other`. The metrics that its + actions observe set that Observation all the same, as in PySpark. - **Subclasses** (intentional; the state is native). PySpark's classes are plain classes that a user subclasses. The drop-in's DataFrame, `Observation`, `SparkSession`, `Catalog`, `DataSourceRegistration`, `DataFrameReader`, `DataStreamReader`, `SparkSession.Hook`, diff --git a/docs/compatibility.md b/docs/compatibility.md index caf6041b..29634e75 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -1929,12 +1929,19 @@ them, as PySpark's DataFrame gives them to its client. An Observation that is no the name of metrics that the plan observes (`client.to_table(plan, {"x": Observation("x")})`), fails the execution with PySpark's `AssertionError` when those metrics arrive. -As PySpark's, the `get` of an Observation is its `_result`, one dict, which the executions that set -its metrics update in place: a dict read before an action has the metrics once the action ran, what -is set on it stays, and a `copy.copy` of the Observation shares it. `_result` is None before the -Observation is used, and `_error` is the error that `get` raises, a new exception of it at each read -where PySpark's is the one that its client set. The three are attributes of the class, not of the -instance, so `vars()` of an Observation does not show them, and they cannot be set. +As PySpark's, `_name`, `_result` and `_error` are the attributes of an Observation, which `vars()` +shows and a program sets and deletes, and `DataFrame.observe` of an Observation gives what its `_on` +gives, of a subclass too. The `get` of an Observation raises its `_error` if it is not +None, and is its `_result` otherwise: one dict, which the executions that set its metrics update in +place, as it is then, so a dict read before an action has the metrics once the action ran, what is +set on it stays, and a `copy.copy` of the Observation shares it. `_result` is None before the +Observation is used, which `observe` checks, so an Observation whose `_result` a program sets to +None is used again, and an execution of its metrics then raises PySpark's `AssertionError`. The +executions convert the metrics when they set them, so a metric that does not convert, such as a +timestamp of the year 1 that `datetime` cannot make local, raises the error of its conversion from +the action, and none is set. The error of collecting them that the server reports is given to the +`_set_error` of the Observation, which sets `_error`, the one exception that `get` raises. A +reference cycle through the attributes, as through the result dict, is freed. The errors of the RPCs that PySpark's client leaves to grpc, those of `clone` (and `SparkSession.cloneSession`) and of `copy_from_local_to_fs` (and `SparkSession.copyFromLocalToFs`), @@ -2435,13 +2442,25 @@ implementation that is the server's order too, and with upb, protobuf's default, hash table, which changes from one process to the next. For its own errors, PySpark keeps the order of each call site. +A signal ends the wait of a call for the server as it ends PySpark's. The `KeyboardInterrupt` of a +handler, as of Ctrl-C, interrupts the execution in flight on the server, as the `except +KeyboardInterrupt` of PySpark's execution does, and a second signal ends the wait without waiting +for the interrupt; the exception of another handler ends the wait at once and is raised, without +an interrupt. The lookup of the plan compression settings of the first plan of a session, and the +read of the stack-trace settings of an error, catch the exception of a handler that is an +`Exception`, as the `except Exception` of PySpark's do: the lookup is then of no settings, as a +failed one, and the call goes on and waits again. + At interpreter exit, a session that is not stopped has its ML cache cleaned by one `clean_cache` request, as PySpark's client does. Its errors are ignored, but when the server is gone the exit waits on that request's retries, as PySpark's does, so the exit can take as long as those retries. A signal ends that wait, as it ends PySpark's, whose cleanup catches `Exception` only: the `KeyboardInterrupt` of Ctrl-C is raised out of the exit hook, which `atexit` reports before it runs the next one, and the session is then neither released nor -closed. A stopped session sends nothing at exit. PySpark 4.3.0 does not exit at all while a +closed. The exception of a handler that is an `Exception` ends the request that waits, which the +exit hook goes on from, unless it arrives while the session looks the plan compression settings +up for the `clean_cache`, as for its first plan: the lookup catches it, as PySpark's, and the +`clean_cache` is sent, which waits again. A stopped session sends nothing at exit. PySpark 4.3.0 does not exit at all while a session other than its default one is left open to the end, such as one of `newSession()` or of `SparkSession(connection=...)`: its `SparkSession.__del__` closes the client in the teardown of the interpreter, whose grpcio `Channel.close()` waits forever. The drop-in exits. diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 4d5cf660..a414d85a 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -15521,6 +15521,148 @@ def observed(name, literal, key): return batch, observed +def test_the_attributes_of_an_observation_are_those_of_its_dict_as_pyspark(mock_connect): + # As PySpark Connect's Observation, a plain class: its _name, _result and _error are the + # entries of vars(o), which observe and the actions set, and which a program sets and deletes, + # of their effects on get, on a new observe and on the metrics of the actions, which update the + # _result of the observation as it is then. Its _error is the one object that each read and + # get give, of the error that the server reports, which the action gives to its _set_error. + from pyspark.errors import PySparkAssertionError + from pyspark.sql import functions as F + from pyspark.sql.connect.observation import Observation + + spark, service = mock_connect + observation = Observation("o") + assert vars(observation) == {"_name": "o", "_result": None, "_error": None} + assert list(vars(observation)) == ["_name", "_result", "_error"] + df = spark.range(3).observe(observation, F.count(F.lit(1)).alias("c")) + assert vars(observation) == {"_name": "o", "_result": {}, "_error": None} + batch, metrics = _count_batch_and_observed_metrics(service, df) + service.execute_payload = batch + metrics("o", varint_field(7, 3), "c") + df.collect() + assert vars(observation) == {"_name": "o", "_result": {"c": 3}, "_error": None} + + # A result that is set is the one that get gives and that the actions update. + replaced = {"kept": True} + observation._result = replaced + df.collect() + assert observation.get is replaced and replaced == {"kept": True, "c": 3} + # Of None, get raises, an action raises PySpark's AssertionError, and observe uses it again. + observation._result = None + with pytest.raises(PySparkAssertionError) as error: + observation.get + assert error.value.getCondition() == "NO_OBSERVE_BEFORE_GET" + with pytest.raises(AssertionError): + df.collect() + observation._name = "again" + again = spark.range(3).observe(observation, F.count(F.lit(1)).alias("c")) + assert vars(observation) == {"_name": "again", "_result": {}, "_error": None} + batch, metrics = _count_batch_and_observed_metrics(service, again) + service.execute_payload = batch + metrics("again", varint_field(7, 5), "c") + again.collect() + assert observation.get == {"c": 5} + + # An error that is set is raised by get, the same object each time. + raised = ValueError("set") + observation._error = raised + with pytest.raises(ValueError) as error: + observation.get + assert error.value is raised and observation._error is raised + del observation._error + with pytest.raises(AttributeError): + observation.get + observation._error = None + observation.other = 1 + assert list(vars(observation).items()) == [ + ("_name", "again"), ("_result", {"c": 5}), ("_error", None), ("other", 1)] + + # The error that the server reports is given to _set_error by the action, and is the one + # object of each read. ExecutePlanResponse.observed_metrics=6: ObservedMetrics.name=1, + # root_error_idx=5, errors=6; Error.error_type_hierarchy=1, message=2. + class Recording(Observation): + def _set_error(self, exc): + self.recorded = exc + super()._set_error(exc) + + failed = Recording("f") + failing = spark.range(3).observe(failed, F.count(F.lit(1)).alias("c")) + error = (message_field(1, "org.apache.spark.SparkArithmeticException") + + message_field(2, "[DIVIDE_BY_ZERO] Division by zero. SQLSTATE: 22012")) + service.execute_payload = batch + message_field( + 6, message_field(1, "f") + varint_field(5, 0) + message_field(6, error)) + failing.count() + assert failed._error is failed._error is failed.recorded + with pytest.raises(Exception, match="DIVIDE_BY_ZERO") as raised: + failed.get + assert raised.value is failed._error + assert list(vars(failed)) == ["_name", "_result", "_error", "recorded"] + + +def test_observe_gives_the_dataframe_of_the_on_of_an_observation_as_pyspark(mock_connect): + # As PySpark's DataFrame.observe, which checks its Columns and then gives what the _on of the + # Observation gives, of a subclass too. + from pyspark.errors import PySparkValueError + from pyspark.sql import functions as F + from pyspark.sql.connect.observation import Observation + + spark, _ = mock_connect + calls = [] + + class Mine(Observation): + def _on(self, df, *exprs): + calls.append((df, exprs)) + return "of _on" + + observation = Mine("m") + df = spark.range(3) + column = F.count(F.lit(1)).alias("c") + assert df.observe(observation, column) == "of _on" + ((seen, exprs),) = calls + assert seen is df and len(exprs) == 1 and exprs[0] is column + with pytest.raises(PySparkValueError) as error: + df.observe(observation) + assert error.value.getCondition() == "CANNOT_BE_EMPTY" and len(calls) == 1 + assert vars(observation) == {"_name": "m", "_result": None, "_error": None} + + +def test_an_observation_in_a_cycle_through_its_result_is_collected_as_pyspark(mock_connect): + # As PySpark's Observation, whose result dict is an attribute of its __dict__, which Python's + # garbage collector sees: an observation that its result refers to is freed with it. + import gc + import weakref + + from pyspark.sql import functions as F + from pyspark.sql.connect.observation import Observation + + spark, _ = mock_connect + observation = Observation("cycle") + df = spark.range(3).observe(observation, F.count(F.lit(1)).alias("c")) + observation.get["self"] = observation + gone = weakref.ref(observation) + del observation, df + gc.collect() + assert gone() is None + + +def test_an_action_converts_the_metrics_of_its_observations_as_pyspark(mock_connect): + # As PySpark's client, which converts the metrics of an Observation when the action sets them + # (LiteralExpression._to_value): a timestamp of the year 1, whose conversion to a local + # datetime fails, raises the ValueError of the conversion from the action, which sets none of + # its metrics. + from pyspark.sql import functions as F + from pyspark.sql.connect.observation import Observation + + spark, service = mock_connect + observation = Observation("t") + df = spark.range(3).observe(observation, F.min(F.current_timestamp()).alias("m")) + batch, metrics = _count_batch_and_observed_metrics(service, df) + # Expression.Literal.timestamp=17, the microseconds of 0001-01-01 00:00:00 UTC. + service.execute_payload = batch + metrics("t", varint_field(17, -62135596800000000), "m") + with pytest.raises(ValueError, match="year 0 is out of range"): + df.collect() + assert observation.get == {} + + def test_only_the_observations_of_the_plan_of_an_action_receive_its_metrics_as_pyspark( mock_connect): # As PySpark's client, which converts and sets the metrics of the Observations of the plan of @@ -30467,13 +30609,15 @@ def test_sigint_during_a_native_kmeans_fit_raises_keyboard_interrupt(mock_connec assert len(service.interrupt_requests) == 1 -def test_exception_of_a_signal_handler_is_raised_by_collect_as_pyspark(mock_connect): +@pytest.mark.parametrize("raised", [Exception, KeyboardInterrupt]) +def test_exception_of_a_signal_handler_is_raised_by_collect_as_pyspark(mock_connect, raised): # As PySpark's: an exception that the handler of the signal raises is the one that the - # waiting call raises, and the operation is interrupted. + # waiting call raises, at once, and the operation is interrupted for a KeyboardInterrupt only, + # as the `except KeyboardInterrupt` of PySpark's execution interrupts it. import signal import threading - class Stop(Exception): + class Stop(raised): pass def stop(signum, frame): @@ -30483,6 +30627,7 @@ def stop(signum, frame): service.block_execute = True previous = signal.signal(signal.SIGINT, stop) timer = threading.Timer(0.3, os.kill, (os.getpid(), signal.SIGINT)) + started = time.monotonic() timer.start() try: with pytest.raises(Stop): @@ -30490,7 +30635,9 @@ def stop(signum, frame): finally: timer.cancel() signal.signal(signal.SIGINT, previous) - assert len(service.interrupt_requests) == 1 + service.release_response() + assert time.monotonic() - started < 3 + assert len(service.interrupt_requests) == (1 if raised is KeyboardInterrupt else 0) def _two_row_range_results(service): @@ -30838,6 +30985,89 @@ def test_sigint_during_the_plan_compression_lookup_raises_keyboard_interrupt_at_ assert service.interrupt_requests == [] +@pytest.mark.parametrize("action", [ + pytest.param(lambda df: df.collect(), id="collect"), + pytest.param(lambda df: list(df.toLocalIterator()), id="toLocalIterator"), +]) +def test_the_plan_compression_lookup_catches_the_exception_of_a_signal_handler_as_pyspark( + mock_connect, action): + # As PySpark's `_get_plan_compression_threshold_and_algorithm`, whose `except Exception` + # catches the exception that a signal handler raises while the lookup of the first plan waits, + # unless it is a KeyboardInterrupt (see above): the lookup ends, the compression is disabled + # for the session, which looks nothing up again, and the action goes on. + import signal + import threading + + class HandlerError(Exception): + pass + + calls = [] + + def handler(signum, frame): + calls.append(signum) + raise HandlerError("of the handler") + + spark, service = mock_connect + service.plan_compression_lookup_delay = 5.0 + previous = signal.signal(signal.SIGINT, handler) + timer = threading.Timer(0.3, os.kill, (os.getpid(), signal.SIGINT)) + started = time.monotonic() + timer.start() + try: + action(spark.range(3)) + action(spark.range(3)) + finally: + timer.cancel() + signal.signal(signal.SIGINT, previous) + assert calls == [signal.SIGINT] + assert time.monotonic() - started < 4 + assert service.plan_compression_lookups == 1 + assert len(service.execute_requests) == 2 + + +def test_the_stack_trace_settings_of_an_error_catch_the_exception_of_a_handler_as_pyspark( + mock_connect): + # As PySpark's `_display_server_stack_trace`, whose `except Exception` catches the exception + # that a signal handler raises while it reads the stack-trace settings of an error: the error + # of the execution is raised, with its stack trace, and not the exception of the handler. + import signal + import threading + + from pyspark.errors import AnalysisException + + class HandlerError(Exception): + pass + + calls = [] + + def handler(signum, frame): + calls.append(signum) + raise HandlerError("of the handler") + + spark, service = mock_connect + spark.range(1).collect() + classes = ["org.apache.spark.sql.AnalysisException", "java.lang.Exception"] + service.execute_error = ("[TABLE_OR_VIEW_NOT_FOUND] t", classes[0], + _error_info(classes, errorClass="TABLE_OR_VIEW_NOT_FOUND")) + service.block_config = True + previous = signal.signal(signal.SIGINT, handler) + timer = threading.Timer(0.3, os.kill, (os.getpid(), signal.SIGINT)) + started = time.monotonic() + timer.start() + try: + with pytest.raises(AnalysisException) as error: + spark.range(1).collect() + finally: + timer.cancel() + signal.signal(signal.SIGINT, previous) + service.block_config = False + service.release_response() + service.execute_error = None + assert calls == [signal.SIGINT] + assert error.value.getCondition() == "TABLE_OR_VIEW_NOT_FOUND" + assert time.monotonic() - started < 4 + + def test_sigint_ends_the_backoff_before_the_reattach_of_a_dropped_stream(mock_connect): # As PySpark's: a stream that fails after a response is reattached after the backoff of the # retry policy, here the server's RetryInfo delay of 5 s, as an attempt of its `Retrying`, diff --git a/python/tests/test_exit_cleanup_offline.py b/python/tests/test_exit_cleanup_offline.py index 3e1260ef..7bad088b 100644 --- a/python/tests/test_exit_cleanup_offline.py +++ b/python/tests/test_exit_cleanup_offline.py @@ -80,9 +80,20 @@ spark.stop() print("ready", flush=True) sys.stdin.readline() - if mode == "live_wait": + if mode in ("live_wait", "live_handler_wait"): import atexit + if mode == "live_handler_wait": + import signal + + class HandlerError(Exception): + pass + + def handler(signum, frame): + print("signal", flush=True) + raise HandlerError("of the handler") + + signal.signal(signal.SIGINT, handler) print("ready", flush=True) sys.stdin.readline() # Registered after the session, so it runs before the session's exit hook. @@ -374,3 +385,43 @@ def test_f_a_signal_ends_the_wait_of_the_exit_hook_when_the_server_is_gone(varia assert elapsed < 30, elapsed assert "Exception ignored in atexit callback" in stderr, stderr assert "KeyboardInterrupt" in stderr, stderr + + +def test_f_the_plan_compression_lookup_of_the_exit_hook_catches_the_exception_of_a_handler( + exit_service): + # (f) The clean_cache of the exit hook of a session that ran no plan looks the plan compression + # up first, as PySpark's `_set_command_in_plan` does, whose `except Exception` catches the + # exception that a signal handler raises while the lookup waits: the lookup ends, and the exit + # hook goes on to send its clean_cache, as PySpark's keeps waiting, and the child exits with + # status 0, of no exception reported by atexit. + exit_service.plan_compression_lookup_delay = 10.0 + child = subprocess.Popen( + [sys.executable, "-c", _CHILD, exit_service.remote, "live_handler_wait"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=_child_env(None), + ) + try: + (session_id,) = child.stdout.readline().split() + assert child.stdout.readline().strip() == "ready" + child.stdin.write("\n") + child.stdin.close() + assert child.stdout.readline().strip() == "exiting" + # The exit hook of the session waits on its lookup by now. + time.sleep(1) + child.send_signal(signal.SIGINT) + started = time.monotonic() + status = child.wait(timeout=60) + elapsed = time.monotonic() - started + stdout, stderr = child.stdout.read(), child.stderr.read() + finally: + if child.poll() is None: + child.kill() + child.wait() + assert status == 0, stderr + assert stdout.split() == ["signal"] + assert elapsed < 8, elapsed + assert "Exception ignored" not in stderr, stderr + assert _exit_requests(exit_service, session_id) == ["clean_cache"] From 345e77deee466db4bb2ba660c829b2b7e106dd8c Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Thu, 19 Nov 2026 04:12:55 +0000 Subject: [PATCH 2/2] [SPARK-62072][CONNECT][PYTHON] Build the CollectMetrics of an Observation of the str() of its _name each time, as PySpark PySpark's CollectMetrics (connect/plan.py:1817-1837) sets `str(self._observation._name)` in the name of its message each time it builds the plan, after its input, and its `observations` name the Observation by that str, by which the client sets the metrics. Observation._on does not read the name. So an Observation renamed after `observe` (`o._name = "other"`) is sent, explained and set by its new name, a name of another object is sent as its str (5 as "5", None as "None", b"n" as "b'n'"), and a name whose `__str__` raises, or a str that UTF-8 cannot encode, fails each request with that error, the exception of the `__str__` or protobuf's UnicodeEncodeError, until it is renamed again. The drop-in built the plan of the name that the Observation had at `observe`, which the Rust core kept in the plan: the server saw the first name, and a name that was not a str was taken as protobuf takes a string field, bytes as their text and another object as protobuf's TypeError of each request, where PySpark sends its str(). The first commit of this PR noted the renamed Observation as a known difference. - The core's MetricsMirror can give the name of the observation of its binding as it is now (`name`, None by default), and `Observation::name_of_plan` asks the mirror of a used observation for it. The build of the CollectMetrics of an `Observe` node, after its input, is of that name, else of the name of the node, and its error is the error of the request (`record_unbuildable`), as of the other values that a plan does not take, so the request is not sent. `LogicalPlan::observations` names the Observations by that name too, and so does the rebuild of a deep copy, which finds the copy of an Observation by it. - The binding's mirror reads `_name` from the Python Observation, which it holds weakly, takes its `str()`, and gives the error of a str that UTF-8 cannot encode as protobuf's, and an error of the read or of `__str__` as it is. `observe` reads no str of the name; the state of the core is of the text of a str name. - known-differences.md no longer lists the renamed Observation; docs/compatibility.md describes the name of each build. Tests: the offline test_the_plan_of_an_observation_is_built_of_its_name_as_it_is_then_as_pyspark sends the renamed Observation's new name and sets its metrics by it, sends 5, None and bytes as their str, fails the requests of a lone surrogate and of a raising `__str__` without sending them, observes by an Observation of a raising name, and deep-copies the DataFrame of the renamed Observation; the core test a_plan_of_an_observation_is_built_of_the_name_that_its_mirror_gives_then builds an Observe node of a mirror's name and error. Probes on both clients (rename, bad names, each request kind of a bad name with the number of `__str__` calls, deep copy) are identical. Co-authored-by: Isaac --- crates/pyspark-rs/src/observation.rs | 37 ++++++++--- crates/spark-connect/src/observation.rs | 85 +++++++++++++++++++++++++ crates/spark-connect/src/plan.rs | 38 ++++++++++- crates/spark-connect/src/udf.rs | 7 ++ dev/parity/known-differences.md | 6 -- docs/compatibility.md | 7 +- python/tests/test_dropin_offline.py | 62 ++++++++++++++++++ 7 files changed, 223 insertions(+), 19 deletions(-) diff --git a/crates/pyspark-rs/src/observation.rs b/crates/pyspark-rs/src/observation.rs index af05d8d9..24a0181d 100644 --- a/crates/pyspark-rs/src/observation.rs +++ b/crates/pyspark-rs/src/observation.rs @@ -270,6 +270,26 @@ impl MetricsMirror for PyObservationMirror { }) } + fn name(&self) -> Option> { + Python::attach(|py| { + let observation = self.observation(py)?; + // As PySpark's `CollectMetrics`, which sets `str(_name)` in the string field of its + // name each time it builds the plan: a str that UTF-8 cannot encode fails the request + // with protobuf's error, and an error of reading the name, or of its `__str__`, as it + // is. + let name = || -> PyResult> { + let name = observation.getattr(intern!(py, "_name"))?.str()?; + Ok( + match crate::dataframe::unbuildable_str_error(name.as_any())? { + Some(error) => Err(error), + None => Ok(name.extract()?), + }, + ) + }; + Some(name().unwrap_or_else(|error| Err(crate::errors::callback_error(error)))) + }) + } + fn as_any(&self) -> &dyn std::any::Any { self } @@ -460,7 +480,7 @@ pub(crate) fn observe( py: Python<'_>, frame: &PyDataFrame, observation: &Bound<'_, PyObservation>, - mut exprs: Vec, + exprs: Vec, ) -> PyResult { let df = &frame.dataframe; if !observation.getattr(intern!(py, "_result"))?.is_none() { @@ -476,13 +496,12 @@ pub(crate) fn observe( name = py.import("uuid")?.call_method0("uuid4")?.str()?.into_any(); observation.setattr(intern!(py, "_name"), &name)?; } - // A name that protobuf does not take, as one that UTF-8 cannot encode, fails the requests of - // the DataFrame, as PySpark's `CollectMetrics` sets it when it builds the plan, before the - // metrics. - let mut args = crate::plan_args::PlanArgs::new(); - let name = args.str(&name)?; - let args = args.before(&mut exprs); - observation.borrow_mut().inner = Observation::new(&name); + // The plans of the DataFrame read the `_name` of the observation each time they are built (see + // `PyObservationMirror::name`), as PySpark's `CollectMetrics` sets `str(_name)` when it builds + // the plan, after its input, and `_on` does not read it: a name whose `__str__` raises, or a + // str that UTF-8 cannot encode, fails each request, not `observe`. The state of the core is of + // the text of a str name, in the place of the name of a plan without the mirror. + observation.borrow_mut().inner = Observation::new(&crate::plan_args::rendered(&name)?); let inner = held_by_the_plan(observation); // As PySpark's `Observation._on`, which reads the `isStreaming` of the DataFrame, asked once. let observed = crate::interrupt::interruptible(py, &df.spark_session(), || { @@ -494,7 +513,7 @@ pub(crate) fn observe( // deep copy of the plan copies with the memo of the call. The clones of its state that do not // keep it alive find it by its weak reference. keep_handle(observation)?; - Ok(PyDataFrame::new(args.plan(observed))) + Ok(PyDataFrame::new(observed)) } /// Have the executions of `session` convert the metrics of the Observations of their plans as diff --git a/crates/spark-connect/src/observation.rs b/crates/spark-connect/src/observation.rs index cf919e66..7815f378 100644 --- a/crates/spark-connect/src/observation.rs +++ b/crates/spark-connect/src/observation.rs @@ -51,6 +51,15 @@ pub trait MetricsMirror: Send + Sync { Ok(()) } + /// The name of the observation of the binding as it is now, which a plan of it is built of, + /// as PySpark's `CollectMetrics` sets the `str` of the `_name` of its Observation each time it + /// builds the plan, and by which the metrics of the plan are set: its error, as that of a name + /// that the plan does not take, is that of the request. None, unless the binding gives it, for + /// the name of the observation. + fn name(&self) -> Option> { + None + } + /// This mirror, by which the binding finds its own type. fn as_any(&self) -> &dyn std::any::Any; } @@ -176,6 +185,14 @@ impl Observation { } } + /// The name that a plan of this observation is built of now: the name that the mirror of a + /// binding gives, if it gives one (see [`MetricsMirror::name`]), of its error, else none, for + /// the name of the observation. Not a stable API. + #[doc(hidden)] + pub fn name_of_plan(&self) -> Option> { + self.metrics_mirror()?.name() + } + /// The mirror of the state of this observation that a binding set (see /// [`Observation::set_metrics_mirror`]), if any. Not a stable API. #[doc(hidden)] @@ -565,6 +582,74 @@ mod tests { recorded(mirror).told.lock().unwrap().clone() } + /// A mirror that gives the name of the observation of its binding as it is now. + struct Named(Mutex>); + + impl MetricsMirror for Named { + fn is_used(&self) -> Result { + Ok(true) + } + + fn update(&self, _keys: &[String], _values: &[proto::expression::Literal]) -> Result<()> { + Ok(()) + } + + fn name(&self) -> Option> { + Some(self.0.lock().unwrap().clone()) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } + } + + #[test] + fn a_plan_of_an_observation_is_built_of_the_name_that_its_mirror_gives_then() { + let built_name = |plan: &crate::plan::LogicalPlan| match plan.try_to_proto() { + Ok(proto::Relation { + rel_type: Some(proto::relation::RelType::CollectMetrics(metrics)), + .. + }) => Ok(metrics.name), + Ok(other) => panic!("a CollectMetrics, not {other:?}"), + Err(error) => Err(error), + }; + let names = |plan: &crate::plan::LogicalPlan| -> Vec { + plan.observations() + .into_iter() + .map(|(name, _)| name) + .collect() + }; + let obs = Observation::used("first"); + let plan = crate::plan::LogicalPlan::Observe { + input: Arc::new(crate::plan::range(0, 3, 1)), + name: "first".to_string(), + exprs: vec![crate::expression::Expression::column("id")], + observation: Some(obs.clone()), + }; + // Without a mirror of a name, the plan is of the name of the observation. + assert!(obs.name_of_plan().is_none()); + assert_eq!(built_name(&plan).unwrap(), "first"); + assert_eq!(names(&plan), vec!["first"]); + + // With one, of the name that it gives at each build, by which the plan names it. + let mirror = Arc::new(Named(Mutex::new(Ok("now".to_string())))); + assert!(obs + .set_metrics_mirror(Arc::clone(&mirror) as Arc) + .is_some()); + assert_eq!(built_name(&plan).unwrap(), "now"); + assert_eq!(names(&plan), vec!["now"]); + + // Its error is that of each build, until it gives a name again. + *mirror.0.lock().unwrap() = Err(SparkError::connect_msg("no name")); + for _ in 0..2 { + let error = built_name(&plan).unwrap_err(); + assert!(error.message().contains("no name"), "{error}"); + } + *mirror.0.lock().unwrap() = Ok("again".to_string()); + assert_eq!(built_name(&plan).unwrap(), "again"); + assert_eq!(names(&plan), vec!["again"]); + } + #[test] fn a_mirror_is_told_of_each_update_and_error_and_says_whether_it_is_used() { // An unused observation has no mirror, as PySpark's has no result before `_on`. diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs index e148e7ac..adf263f8 100644 --- a/crates/spark-connect/src/plan.rs +++ b/crates/spark-connect/src/plan.rs @@ -1777,6 +1777,20 @@ pub(crate) fn too_deep_a_plan(depth: usize) -> SparkError { )) } +/// The name that the `CollectMetrics` of an `Observe` of `name` and `observation` is built of, as +/// PySpark's sets the `str` of the `_name` of its Observation each time it builds the plan: that +/// which the binding of the Observation gives now (see +/// [`crate::observation::MetricsMirror::name`]), or its error, else `name`. +fn observed_name( + name: &str, + observation: Option<&crate::observation::Observation>, +) -> Result { + match observation.and_then(crate::observation::Observation::name_of_plan) { + Some(current) => current, + None => Ok(name.to_string()), + } +} + /// Serialize a shared plan node for a request: the node and every node below it carry /// their stable plan IDs. A plan deeper than [`MAX_PLAN_DEPTH`], with the plans of its /// subqueries, or with an expression deeper than [`crate::expression::MAX_EXPRESSION_DEPTH`], is @@ -2558,7 +2572,13 @@ impl LogicalPlan { name, observation: Some(observation), .. - } => keep(name, observation), + } => { + // By the name of the plan as it is built now, as PySpark's `observations` + // names the Observation by the `str` of its `_name`. + let name = + observed_name(name, Some(observation)).unwrap_or_else(|_| name.clone()); + keep(&name, observation) + } LogicalPlan::Raw { observations: held, .. } => { @@ -3517,11 +3537,23 @@ impl LogicalPlan { } LogicalPlan::Observe { - input, name, exprs, .. + input, + name, + exprs, + observation, } => { let mut collect_metrics = proto::CollectMetrics::default(); collect_metrics.input = Some(Box::new(relation_of(input, plan_ids))); - collect_metrics.name = name.clone(); + // As PySpark's `CollectMetrics`, which sets the `str` of the `_name` of its + // Observation when it builds the plan, after its input: its error fails the + // request. + collect_metrics.name = match observed_name(name, observation.as_ref()) { + Ok(name) => name, + Err(error) => { + crate::expression::record_unbuildable(&error); + name.clone() + } + }; for expr in exprs { collect_metrics.metrics.push(expr.to_proto()); } diff --git a/crates/spark-connect/src/udf.rs b/crates/spark-connect/src/udf.rs index 05baf4e2..00bdc366 100644 --- a/crates/spark-connect/src/udf.rs +++ b/crates/spark-connect/src/udf.rs @@ -509,6 +509,13 @@ impl<'c> PicklerRebuild<'c> { name: &str, observation: &mut Option, ) -> bool { + // By the name of the plan as it is built now, by which the copies are named (see + // `LogicalPlan::observations`). + let current = observation + .as_ref() + .and_then(crate::observation::Observation::name_of_plan) + .and_then(Result::ok); + let name = current.as_deref().unwrap_or(name); match (observation.is_some(), self.copy_of_observation(name)) { (true, Some(copy)) => { *observation = Some(copy); diff --git a/dev/parity/known-differences.md b/dev/parity/known-differences.md index ca63b45c..a7ea07cd 100644 --- a/dev/parity/known-differences.md +++ b/dev/parity/known-differences.md @@ -566,12 +566,6 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan holds there (`vars(spark.streams)` is empty, where PySpark's has `_session` and `_sqlb`). An `Observation` is not one of them: its `_name`, `_result` and `_error` are of its `__dict__`, as PySpark's. -- **The name of an Observation renamed after `observe`** (intentional; the plan is native). The - `CollectMetrics` of a DataFrame that an Observation observes is of the name that the - Observation had then, which the Rust core keeps in the plan, where PySpark's plan reads the - Observation's `_name` each time it is built: after `o._name = "other"`, the plan that the - server explains names the first name, where PySpark's names `other`. The metrics that its - actions observe set that Observation all the same, as in PySpark. - **Subclasses** (intentional; the state is native). PySpark's classes are plain classes that a user subclasses. The drop-in's DataFrame, `Observation`, `SparkSession`, `Catalog`, `DataSourceRegistration`, `DataFrameReader`, `DataStreamReader`, `SparkSession.Hook`, diff --git a/docs/compatibility.md b/docs/compatibility.md index 29634e75..3c502796 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -1931,7 +1931,12 @@ fails the execution with PySpark's `AssertionError` when those metrics arrive. As PySpark's, `_name`, `_result` and `_error` are the attributes of an Observation, which `vars()` shows and a program sets and deletes, and `DataFrame.observe` of an Observation gives what its `_on` -gives, of a subclass too. The `get` of an Observation raises its `_error` if it is not +gives, of a subclass too. Each build of a plan of the Observation, at each request of its +DataFrames, reads its `_name`, as PySpark's `CollectMetrics` does: the plan is of `str(_name)`, by +which the metrics are set, so an Observation renamed after `observe` is observed by its new name, +and a name whose `__str__` raises, or a str that UTF-8 cannot encode, fails each request with that +error until it is renamed again; `observe` does not read it. A deep copy of a DataFrame holds a copy +of the Observation, of its name then. The `get` of an Observation raises its `_error` if it is not None, and is its `_result` otherwise: one dict, which the executions that set its metrics update in place, as it is then, so a dict read before an action has the metrics once the action ran, what is set on it stays, and a `copy.copy` of the Observation shares it. `_result` is None before the diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index a414d85a..006a86fe 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -15625,6 +15625,68 @@ def _on(self, df, *exprs): assert vars(observation) == {"_name": "m", "_result": None, "_error": None} +def test_the_plan_of_an_observation_is_built_of_its_name_as_it_is_then_as_pyspark(mock_connect): + # As PySpark's CollectMetrics, which sets the str() of the _name of its Observation each time + # it builds the plan, after its input, and whose observations name it so: a request of an + # Observation renamed after observe carries its new name, of any object, by which its metrics + # are set, and a name whose __str__ raises, or a str that UTF-8 cannot encode, fails each + # request with that error, until it is renamed again. Observation._on does not read the name. + # A deep copy of the DataFrame holds a copy of the Observation, of its name then, whose metrics + # its actions set, as PySpark's deep copy of the plan. + import copy + + import pyspark.sql.connect.proto as pb2 + from pyspark.sql import functions as F + from pyspark.sql.connect.observation import Observation + + class Unnamed: + def __str__(self): + raise ValueError("no name") + + def sent_name(): + request = pb2.ExecutePlanRequest.FromString(service.execute_requests[-1]) + return request.plan.root.collect_metrics.name + + spark, service = mock_connect + observation = Observation("before") + df = spark.range(3).observe(observation, F.count(F.lit(1)).alias("c")) + observation._name = "after" + batch, metrics = _count_batch_and_observed_metrics(service, df) + assert sent_name() == "after" + service.execute_payload = batch + metrics("after", varint_field(7, 3), "c") + df.collect() + assert observation.get == {"c": 3} + for name, text in ((5, "5"), (None, "None"), (b"n", "b'n'")): + observation._name = name + df.collect() + assert sent_name() == text + sent = len(service.execute_requests) + for name, error in (("lone \ud800", UnicodeEncodeError), (Unnamed(), ValueError)): + observation._name = name + with pytest.raises(error): + df.collect() + assert len(service.execute_requests) == sent + observation._name = "again" + batch, metrics = _count_batch_and_observed_metrics(service, df) + assert sent_name() == "again" + service.execute_payload = batch + metrics("again", varint_field(7, 4), "c") + df.collect() + assert observation.get == {"c": 4} + copied = copy.deepcopy(df) + (held,) = copied._plan.observations.values() + assert held is not observation and list(copied._plan.observations) == ["again"] + service.execute_payload = batch + metrics("again", varint_field(7, 5), "c") + copied.collect() + assert held.get == {"c": 5} and observation.get == {"c": 4} + unnamed = Observation("unnamed") + unnamed._name = Unnamed() + observed = spark.range(3).observe(unnamed, F.count(F.lit(1)).alias("c")) + sent = len(service.execute_requests) + with pytest.raises(ValueError, match="no name"): + observed.collect() + assert len(service.execute_requests) == sent + + def test_an_observation_in_a_cycle_through_its_result_is_collected_as_pyspark(mock_connect): # As PySpark's Observation, whose result dict is an attribute of its __dict__, which Python's # garbage collector sees: an observation that its result refers to is freed with it.