Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 73 additions & 7 deletions crates/pyspark-rs/src/column.rs
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,27 @@ impl PyColumnExpression {
self.column = Column::new(spark_connect::expression::Expression::column(""));
}
}

/// An expression pickles, as PySpark's expression, a Python object, pickles its fields: as the
/// `_expr` of its Column, `getattr(column, "_expr")`, of the pickle of the Column (see
/// `PyColumn::__reduce__`), so that the unpickled expression is that of the unpickled Column,
/// also on a Python worker of the server, where it is the expression of PySpark Connect's
/// Column.
fn __reduce__<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyTuple>> {
let column = Py::new(py, PyColumn::new(self.column.clone()))?;
let args = PyTuple::new(
py,
[
column.into_any().into_bound(py),
pyo3::types::PyString::new(py, "_expr").into_any(),
],
)?;
PyTuple::new(
py,
[py.import("builtins")?.getattr("getattr")?, args.into_any()],
)
}

/// As PySpark's `__repr__` of the expression, which raises its error of an expression that it
/// does not render (a `ColumnAlias` of names that are not all `str`).
fn __repr__(&self) -> PyResult<String> {
Expand Down Expand Up @@ -930,7 +951,10 @@ impl PyColumn {
/// that `pickle` keeps it, as it does of PySpark's Column. The function of a UDF of the
/// Column is pickled now, into the command of the expression, as PySpark's pickle of the
/// Column pickles the function with it; one that does not pickle raises the error of its
/// pickling.
/// pickling. A Column of subqueries pickles the relations of the plans that they refer to
/// too, of their plan IDs, after the expression, as PySpark's pickle keeps the plan objects of
/// its subqueries: the unpickled Column sends them in the `WithRelations` of the relation that
/// holds it, as PySpark's plan sends the plans of its subqueries.
fn __reduce__<'py>(slf: &Bound<'py, Self>, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
let this = slf.borrow();
// Of an expression that a request fails to build, as PySpark's Column of an alias of a
Expand All @@ -948,10 +972,34 @@ impl PyColumn {
{
return Err(exception);
}
let build_error = unbuildable
let mut build_error = unbuildable
.as_ref()
.map(|error| error_description(py, error))
.transpose()?;
// A plan of a subquery that does not build pickles of its error, as a DataFrame does,
// which the requests of the unpickled Column raise; one that does not pickle, or that is
// too deep, raises its error. A CASE WHEN and a sort order are given back of their parts,
// which pickle their subqueries.
let mut subqueries = Vec::new();
let of_parts = matches!(
this.column.expression().node(),
ExpressionNode::CaseWhen(_) | ExpressionNode::SortOrder(_)
);
if build_error.is_none() && !of_parts {
match crate::origin::pickling(|| this.column.expression().subquery_relations()) {
Ok(relations) => {
for (relation, levels) in relations {
let relation =
pyo3::types::PyBytes::new(py, &crate::encode_on_grown_stack(relation));
subqueries.push((relation, levels));
}
}
Err(error) => match described_error(py, &error)? {
Some(description) => build_error = Some(description),
None => return Err(crate::errors::spark_error_to_py_exception(error)),
},
}
}
let repr_error = this
.column
.expression()
Expand Down Expand Up @@ -999,14 +1047,17 @@ impl PyColumn {
levels,
build_error,
repr_error,
subqueries,
))?;
crate::group::reduce_with_state(&reduce, slf.as_any())
}

/// The Column of `expression`, the bytes of a `spark.connect.Expression`, and of its
/// `rendered` rendering, of its `levels`, as a pickled Column unpickles: it sends the
/// expression as it is. The CASE WHEN of `branches` and `else_value`, the Columns of its
/// parts, is built as it was.
/// expression as it is, and the relations of `subqueries`, the bytes of the relation of each
/// plan that its subqueries refer to and its levels, in the `WithRelations` of the relation
/// that holds it. The CASE WHEN of `branches` and `else_value`, the Columns of its parts, is
/// built as it was.
#[staticmethod]
#[pyo3(signature = (
expression,
Expand All @@ -1015,7 +1066,8 @@ impl PyColumn {
else_value=None,
levels=1,
build_error=None,
repr_error=None
repr_error=None,
subqueries=None
))]
#[allow(clippy::too_many_arguments)]
fn _of_expression(
Expand All @@ -1026,6 +1078,7 @@ impl PyColumn {
levels: usize,
build_error: Option<&Bound<'_, PyAny>>,
repr_error: Option<&Bound<'_, PyAny>>,
subqueries: Option<Vec<(Bound<'_, pyo3::types::PyBytes>, usize)>>,
) -> PyResult<PyColumn> {
if let Some(branches) = branches {
let branches = branches
Expand All @@ -1048,12 +1101,25 @@ impl PyColumn {
)));
}
use prost::Message;
let subqueries: Vec<(&[u8], usize)> = subqueries
.iter()
.flatten()
.map(|(relation, levels)| (relation.as_bytes(), *levels))
.collect();
// Decoded, and then built or dropped, on a stack that holds its levels.
// The errors become Python exceptions on the caller's stack, which runs no Python code
// on the grown one.
let column = crate::on_grown_stack(|| {
spark_connect_proto::Expression::decode(expression)
.map(|expression| Column::sent(expression, rendered, levels))
let relations = subqueries
.iter()
.map(|(relation, levels)| {
spark_connect_proto::Relation::decode(*relation)
.map(|relation| (relation, *levels))
})
.collect::<Result<Vec<_>, _>>()?;
spark_connect_proto::Expression::decode(expression).map(|expression| {
Column::sent_with_subqueries(expression, rendered, levels, relations)
})
})
.map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?
.to_pyerr()?;
Expand Down
42 changes: 24 additions & 18 deletions crates/pyspark-rs/src/dataframe.rs
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,10 @@ impl PyDataFrame {
/// first plan of the session reads the plan compression settings then; then
/// `SparkConnectClient.to_pandas` reads the settings of the conversion, in one request,
/// and sends the plan; the Arrow result with the schema that the server sends is converted
/// by `pyspark.sql.connect.client._to_pandas`.
/// by `pyspark.sql.connect.client._to_pandas`. The DataFrame keeps the information of the
/// execution once the result is converted, as `_to_pandas` sets `_execution_info` after
/// `SparkConnectClient.to_pandas` returns: a conversion that fails leaves `executionInfo` as
/// it was.
pub(crate) fn to_pandas(
&self,
py: Python<'_>,
Expand All @@ -172,9 +175,9 @@ impl PyDataFrame {
"spark.sql.execution.arrow.pyspark.selfDestruct.enabled",
])
})?;
let (batches, schema) = interruptible(py, &session, || {
let (batches, schema, info) = interruptible(py, &session, || {
execution
.collect_record_batches_checked(check_schema_at_receipt)
.collect_record_batches_and_info(check_schema_at_receipt)
.map_err(Box::new)
})?;
check_schema_conversion(py, schema.as_ref())?;
Expand All @@ -191,13 +194,7 @@ impl PyDataFrame {
Some(schema) => crate::types::schema_to_py(py, schema)?,
None => py.None(),
};
let (metrics, observed) = match self.dataframe.execution_info() {
Ok(info) => execution_metrics(py, &info)?,
Err(_) => (
pyo3::types::PyList::empty(py),
pyo3::types::PyList::empty(py),
),
};
let (metrics, observed) = execution_metrics(py, &info)?;
let kwargs = pyo3::types::PyDict::new(py);
kwargs.set_item("timezone", &settings[0])?;
match struct_handling_mode {
Expand All @@ -210,6 +207,7 @@ impl PyDataFrame {
(table, schema, metrics, observed),
Some(&kwargs),
)?;
self.dataframe.keep_execution_info(info);
Ok(pdf.unbind())
}

Expand Down Expand Up @@ -4638,8 +4636,11 @@ type MergeClause = (
/// which asserts a str and sets it in a string field, and then its value, whose `to_plan` an
/// object that is not a Column lacks; and then the condition. A Column that does not build raises
/// its error there, such as a str that UTF-8 cannot encode, or the `RecursionError` of PySpark's
/// serializer of one too deep to be sent, and the clause is not added. The merge builds them again
/// when it runs. The assignments keep their order, in which the merge sends them.
/// serializer of one too deep to be sent, and the clause is not added. The clause keeps them as
/// they were built then ([`built_column`]), which the merge sends, as PySpark's merge sends the
/// protos that its clause built: so the functions of their UDFs are pickled once, when the clause
/// is added, and the warning of a window of no partitions is given then. The assignments keep
/// their order, in which the merge sends them.
fn merge_clause(
py: Python<'_>,
condition: &ClauseCondition,
Expand All @@ -4660,25 +4661,30 @@ fn merge_clause(
return Err(not_column(&value, "value"));
}
};
value.try_to_proto().to_pyerr()?;
built.push((key, value));
built.push((key, built_column(&value)?));
}
}
let condition = match condition {
ClauseCondition::None => None,
ClauseCondition::Column(column) => Some(column.clone()),
ClauseCondition::Column(column) => Some(built_column(column)?),
ClauseCondition::Object(object) => {
let object = object.bind(py);
object.getattr("to_plan")?;
return Err(not_column(object, "condition"));
}
};
if let Some(condition) = &condition {
condition.try_to_proto().to_pyerr()?;
}
Ok((condition, built))
}

/// The Column of the `spark.connect.Expression` of `column` as a request builds it now, of its
/// rendering and its levels, which sends that expression as it is (`Column::sent`), as the
/// `MergeAction` that PySpark's `_build_merge_action` builds holds the protos of its Columns; or
/// the error of the build.
fn built_column(column: &spark_connect::column::Column) -> PyResult<spark_connect::column::Column> {
let (expression, levels) = column.try_to_proto_with_levels().to_pyerr()?;
spark_connect::column::Column::sent(expression, column.expression().render(), levels).to_pyerr()
}

#[pymethods]
impl PyMergeIntoWriter {
/// The shallow copy of a merge builder, as PySpark's `copy.copy`: the copy shares the state
Expand Down
8 changes: 7 additions & 1 deletion crates/pyspark-rs/src/origin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ impl spark_connect::plan_build::PlanBuildListener for PlanBuild {
}

fn window_without_partitions(&self) -> spark_connect::Result<()> {
if PICKLING.with(std::cell::Cell::get) {
if is_pickling() {
return Ok(());
}
Python::attach(|py| {
Expand Down Expand Up @@ -95,6 +95,12 @@ thread_local! {
static PICKLING: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
}

/// Whether this thread builds the plan of a DataFrame, a Column or a table argument to pickle it
/// ([`pickling`]).
pub(crate) fn is_pickling() -> bool {
PICKLING.with(std::cell::Cell::get)
}

/// Run `pickle`, which builds the plan of a DataFrame or a Column to pickle it, without the
/// warning of a window of no partitions: PySpark's pickle keeps its plan objects, and builds no
/// plan.
Expand Down
69 changes: 45 additions & 24 deletions crates/pyspark-rs/src/udf_command.rs
Original file line number Diff line number Diff line change
Expand Up @@ -71,34 +71,55 @@ impl PythonCommand {
/// Of a UDTF, the `PythonUdtf` message that the function returns; of a UDF, the `PythonUdf`
/// message of its output type, converted first, and of the pickle of the object, as PySpark's
/// `PythonUDF.to_plan` converts `pyspark_types_to_proto_types(output_type)` before it pickles
/// `(func, output_type)`, of the type as it is then; of a data type, its message.
/// `(func, output_type)`, of the type as it is then; of a data type, its message. Its error is
/// that of [`pickle_error`].
fn pickle(&self) -> spark_connect_core::error::Result<Vec<u8>> {
Python::attach(|py| {
let object = self.object.bind(py);
match self.kind {
CommandKind::Udtf => command_of(&object.call0()?),
CommandKind::Udf => {
use prost::Message;
let output_type =
crate::types::py_to_proto_data_type(&object.get_item(1)?)?.to_proto();
let command = crate::dataframe::py_cloudpickle(py, object)?;
Ok(spark_connect_proto::PythonUdf {
output_type: Some(output_type),
command: command.into(),
..Default::default()
}
.encode_to_vec())
}
CommandKind::DataType => {
use prost::Message;
Ok(crate::types::py_to_proto_data_type(object)?
.to_proto()
.encode_to_vec())
Python::attach(|py| self.pickled(py).map_err(|error| pickle_error(py, error)))
}

fn pickled(&self, py: Python<'_>) -> PyResult<Vec<u8>> {
let object = self.object.bind(py);
match self.kind {
CommandKind::Udtf => command_of(&object.call0()?),
CommandKind::Udf => {
use prost::Message;
let output_type =
crate::types::py_to_proto_data_type(&object.get_item(1)?)?.to_proto();
let command = crate::dataframe::py_cloudpickle(py, object)?;
Ok(spark_connect_proto::PythonUdf {
output_type: Some(output_type),
command: command.into(),
..Default::default()
}
.encode_to_vec())
}
CommandKind::DataType => {
use prost::Message;
Ok(crate::types::py_to_proto_data_type(object)?
.to_proto()
.encode_to_vec())
}
})
.map_err(crate::errors::callback_error)
}
}
}

/// The error of a request of `exception`, which the pickling of a command raised: the exception
/// itself ([`crate::errors::callback_error`]). But while a DataFrame, a Column or a table argument
/// is pickled ([`crate::origin::pickling`]), the pickle keeps an error of one of PySpark's error
/// classes other than a `PySparkPicklingError`, which the conversion of a type raises before the
/// function is pickled, such as the `UNSUPPORTED_OPERATION` of a type that has no proto, as the
/// error of its plan (see `crate::column::described_error`): PySpark's pickle keeps the type
/// object as it is, and the conversion fails at the requests of the unpickled object, or at the
/// unpickle of a DataFrame, which builds its plan. The pickle raises the error of a pickling.
fn pickle_error(py: Python<'_>, exception: PyErr) -> spark_connect_core::error::SparkError {
if crate::origin::is_pickling() {
if let Ok(Some(error)) = crate::errors::pyspark_class_error(py, &exception) {
if error.kind != spark_connect_core::error::SparkErrorKind::PicklingError {
return error;
}
}
}
crate::errors::callback_error(exception)
}

/// The pickler of `udtf`, a Python function of no arguments that returns the `PythonUDTF`
Expand Down
16 changes: 16 additions & 0 deletions crates/spark-connect/src/column.rs
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,22 @@ impl Column {
Ok(Column::new(Expression::sent(expression, rendered, levels)?))
}

/// As [`Column::sent`], of an expression whose subqueries refer to the plans of the relations
/// of `subqueries`, each of its levels, which a pickled Column keeps
/// ([`Expression::subquery_relations`]): the relation that holds the Column sends them (see
/// [`Expression::sent_with_subqueries`]). Not a stable API.
#[doc(hidden)]
pub fn sent_with_subqueries(
expression: proto::Expression,
rendered: String,
levels: usize,
subqueries: Vec<(proto::Relation, usize)>,
) -> spark_connect_core::error::Result<Column> {
Ok(Column::new(Expression::sent_with_subqueries(
expression, rendered, levels, subqueries,
)?))
}

/// Mirrors `pyspark.sql.column.Column.over` - window function application.
pub fn over(self, window_spec: WindowSpec) -> Column {
// Convert window frame spec if present
Expand Down
Loading
Loading