Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/pyspark-rs/src/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ pub(crate) fn execute_plan(
.into_iter()
.map(|(name, observation)| {
let observation = observation.cast::<crate::observation::PyObservation>()?;
Ok((name, observation.borrow().inner.clone()))
Ok((name, crate::observation::state_of_execution(observation)?))
})
.collect::<PyResult<Vec<_>>>()?;
let execution = interruptible(py, session, || {
Expand Down
37 changes: 23 additions & 14 deletions crates/pyspark-rs/src/dataframe.rs
Original file line number Diff line number Diff line change
Expand Up @@ -324,7 +324,7 @@ impl PyDataFrame {
/// this one keeps apart from the core, as PySpark's `alias`, `filter`, `limit`, `sort`, the
/// repartitions and the other methods that copy the `_cached_schema` of their input copy it
/// whatever it is. The core keeps a cached struct itself.
fn of_same_schema(&self, mut result: PyDataFrame) -> PyDataFrame {
pub(crate) fn of_same_schema(&self, mut result: PyDataFrame) -> PyDataFrame {
if let Some(schema) = &self.schema_object {
result.schema_object = Some(Python::attach(|py| schema.clone_ref(py)));
}
Expand Down Expand Up @@ -2503,14 +2503,15 @@ impl PyDataFrame {
}

/// Observe metrics, as PySpark's `DataFrame.observe(observation, *exprs)`: the Columns
/// `exprs`, which an Observation receives, or which are observed by a name.
/// `exprs`, which an Observation receives by its `_on`, of a subclass too, or which are
/// observed by a name.
#[pyo3(signature = (observation, *exprs))]
fn observe(
&self,
py: Python<'_>,
observation: &Bound<'_, PyAny>,
exprs: Vec<Bound<'_, PyAny>>,
) -> PyResult<PyDataFrame> {
fn observe<'py>(
slf: &Bound<'py, Self>,
py: Python<'py>,
observation: &Bound<'py, PyAny>,
exprs: Vec<Bound<'py, PyAny>>,
) -> PyResult<Bound<'py, PyAny>> {
if exprs.is_empty() {
return Err(crate::column::raise_pyspark(
py,
Expand All @@ -2537,18 +2538,26 @@ impl PyDataFrame {
}
}
}
if let Ok(observation) = observation.cast::<crate::observation::PyObservation>() {
let observed = crate::observation::observe(py, self, observation, columns)?;
return Ok(self.of_same_schema(observed));
if observation.is_instance_of::<crate::observation::PyObservation>() {
// As PySpark's, which gives the DataFrame of the `_on` of the Observation, of a
// subclass too.
let mut arguments = vec![slf.clone().into_any()];
arguments.extend(exprs);
return observation.call_method1(
pyo3::intern!(py, "_on"),
pyo3::types::PyTuple::new(py, arguments)?,
);
}
let this = slf.borrow();
if observation.is_instance_of::<pyo3::types::PyString>() {
// Its `CollectMetrics` plan sets the name when it builds the plan, before the metrics.
let mut args = crate::plan_args::PlanArgs::new();
let name = args.str(observation)?;
let args = args.before(&mut columns);
return Ok(self.of_same_schema(PyDataFrame::new(
args.plan(self.dataframe.observe(&name, columns)),
)));
let observed = this.of_same_schema(PyDataFrame::new(
args.plan(this.dataframe.observe(&name, columns)),
));
return Ok(Bound::new(py, observed)?.into_any());
}
let type_name = observation.get_type().name()?.to_string();
Err(crate::column::raise_pyspark(
Expand Down
50 changes: 40 additions & 10 deletions crates/pyspark-rs/src/interrupt.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,13 @@
//!
//! The wait runs on the calling thread, as PySpark's does, so the progress handlers and the
//! other callbacks of the execution run there too. The wait polls the signals of the process,
//! which only the main thread of Python runs the handlers of. A handler that raises interrupts the
//! execution in flight of the calling thread on the server; the wait then ends with the server's
//! cancellation, and the exception of the handler is raised.
//! which only the main thread of Python runs the handlers of. A handler that raises a
//! KeyboardInterrupt, as the default handler of SIGINT does, interrupts the execution in flight of
//! the calling thread on the server; the wait then ends with the server's cancellation, and the
//! exception of the handler is raised. The exception of another handler ends the wait at once, as
//! PySpark's execution interrupts the operation of a KeyboardInterrupt only, unless the part of
//! the call that waits catches it, as PySpark's lookup of the plan compression settings catches an
//! `Exception` (see `spark_connect_core::runtime::catching`).

use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
Expand Down Expand Up @@ -122,10 +126,35 @@ impl SignalWatch {
{
let mut error = self.error.lock().unwrap_or_else(|e| e.into_inner());
if error.is_none() {
// Takes the GIL for the check only, as the wait runs without it.
match Python::attach(|py| py.check_signals()) {
Ok(()) => return Watch::Continue,
Err(signal) => *error = Some(signal),
// Takes the GIL for the check only, as the wait runs without it. The exception of a
// handler that is an `Exception` ends the part of the call that PySpark's `except
// Exception` covers, if one runs, such as the lookup of the plan compression
// settings, and the call goes on, as PySpark's does (see `runtime::catching`).
let raised = Python::attach(|py| match py.check_signals() {
Ok(()) => None,
Err(signal)
if signal.is_instance_of::<pyo3::exceptions::PyException>(py)
&& spark_connect_core::runtime::catch_signal() =>
{
None
}
Err(signal) => {
let keyboard_interrupt =
signal.is_instance_of::<pyo3::exceptions::PyKeyboardInterrupt>(py);
Some((signal, keyboard_interrupt))
}
});
match raised {
None => return Watch::Continue,
Some((signal, keyboard_interrupt)) => {
*error = Some(signal);
// As PySpark's execution, whose `except KeyboardInterrupt` interrupts the
// operation on the server: the exception of another handler ends the
// wait at once, as it goes out of the execution as it is.
if !keyboard_interrupt {
return Watch::Abort;
}
}
}
} else if self.interrupted.load(Ordering::SeqCst) {
// A failed interrupt ends the wait at once, as PySpark raises the error of the
Expand Down Expand Up @@ -159,9 +188,10 @@ impl SignalWatch {
}

/// Run `f` without the GIL, on the calling thread, as the signals of the process are polled while
/// it waits for the server. When a handler raises, the execution that the calling thread waits for
/// is interrupted, and the exception of the handler is raised once `f` has ended. Otherwise the
/// result of `f` is returned.
/// it waits for the server. When a handler raises a KeyboardInterrupt, the execution that the
/// calling thread waits for is interrupted, and the exception of the handler is raised once `f`
/// has ended; the exception of another handler ends the waits of `f` at once, and is raised.
/// Otherwise the result of `f` is returned.
///
/// The execution is interrupted only when a wait of the calling thread is for one. The waits
/// that a signal ends at once, as PySpark's do (a retry sleep, or a call that the server does not
Expand Down
Loading
Loading