Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 37 additions & 5 deletions crates/pyspark-rs/src/interrupt.rs
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ pub(crate) fn interruptible_outcome<T: Send, E: Send + Into<Box<SparkError>>>(
session: &SparkSession,
f: impl FnOnce() -> Result<T, E> + Send,
) -> PyResult<Result<T, Box<SparkError>>> {
run_interruptible(py, session, f, false)
run_interruptible(py, session, f, false, true)
}

/// As [`interruptible_outcome`], for a call that PySpark makes in a `try` whose `except Exception`
Expand All @@ -235,15 +235,43 @@ pub(crate) fn interruptible_caught<T: Send, E: Send + Into<Box<SparkError>>>(
session: &SparkSession,
f: impl FnOnce() -> Result<T, E> + Send,
) -> PyResult<Result<T, Box<SparkError>>> {
run_interruptible(py, session, f, true)
run_interruptible(py, session, f, true, true)
}

/// As [`interruptible`], for a call whose requests carry no user context extensions, as PySpark's
/// requests of the artifacts, `CloneSession`, `ReleaseSession` and `GetStatus` carry none: the
/// extensions are not serialized again before it (see [`refresh_user_context`]), so that one that
/// is not an `Any`, which fails each call whose requests carry them, does not fail it, as in
/// PySpark.
pub(crate) fn interruptible_without_user_context<T: Send, E: Send + Into<Box<SparkError>>>(
py: Python<'_>,
session: &SparkSession,
f: impl FnOnce() -> Result<T, E> + Send,
) -> PyResult<T> {
run_interruptible(py, session, f, false, false)?.to_pyerr()
}

/// As [`interruptible_outcome`], for a call whose requests carry no user context extensions (see
/// [`interruptible_without_user_context`]).
pub(crate) fn interruptible_outcome_without_user_context<
T: Send,
E: Send + Into<Box<SparkError>>,
>(
py: Python<'_>,
session: &SparkSession,
f: impl FnOnce() -> Result<T, E> + Send,
) -> PyResult<Result<T, Box<SparkError>>> {
run_interruptible(py, session, f, false, false)
}

/// Bring the user context extensions of `session` up to date before a call sends its requests,
/// on the calling thread, with the GIL: the function of `client` that the session keeps (see
/// `PySparkSession::_set_user_context_refresh`), which serializes the `google.protobuf.Any`
/// messages of its lists again, as PySpark serializes them when it builds each request, so that
/// a change of a message in place is sent. Its error is raised, as PySpark raises that of the
/// serialization from the call.
/// serialization from the call, such as upb's `TypeError` of an extension that is not an `Any`:
/// so a call whose requests carry no extensions does not run it (see
/// [`interruptible_without_user_context`]).
pub(crate) fn refresh_user_context(py: Python<'_>, session: &SparkSession) -> PyResult<()> {
if let Some(refresher) = session.user_context_refresher() {
if let Some(refresh) = refresher.downcast_ref::<Py<PyAny>>() {
Expand All @@ -254,14 +282,18 @@ pub(crate) fn refresh_user_context(py: Python<'_>, session: &SparkSession) -> Py
}

/// The wait of [`interruptible_outcome`], of whose results a failed interrupt is the inner one
/// when `catch_interrupt_failure`.
/// when `catch_interrupt_failure`, after the user context extensions are brought up to date when
/// `refresh_extensions`.
fn run_interruptible<T: Send, E: Send + Into<Box<SparkError>>>(
py: Python<'_>,
session: &SparkSession,
f: impl FnOnce() -> Result<T, E> + Send,
catch_interrupt_failure: bool,
refresh_extensions: bool,
) -> PyResult<Result<T, Box<SparkError>>> {
refresh_user_context(py, session)?;
if refresh_extensions {
refresh_user_context(py, session)?;
}
let watch = Arc::new(SignalWatch {
session: session.clone(),
interrupted: AtomicBool::new(false),
Expand Down
5 changes: 4 additions & 1 deletion crates/pyspark-rs/src/pbmsg/message.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1881,7 +1881,10 @@ impl PyMessage {

/// Pickle as protobuf's messages pickle: the class, and the state of the bytes of the
/// message, which `__setstate__` parses. A nested class pickles by its qualified name, which
/// the generated modules of protobuf have too, so either runtime loads the pickle of the other.
/// the generated modules of protobuf have too, so either runtime loads the pickle of the
/// other, but for the well-known types (`Any`, `Timestamp`): their classes are of the modules
/// of the runtime (`pyspark.sql.connect.proto._rust.any_pb2`), which `install` puts in
/// `sys.modules`, where upb's are of protobuf's (`google.protobuf.any_pb2`).
fn __reduce__<'py>(
slf: &Bound<'py, Self>,
) -> PyResult<(Bound<'py, PyType>, Bound<'py, PyTuple>, Bound<'py, PyDict>)> {
Expand Down
34 changes: 20 additions & 14 deletions crates/pyspark-rs/src/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1591,7 +1591,7 @@ impl PySparkSession {
return Err(unsupported_artifact_error(py, format));
}
check_local_artifact(py, &name, &local, pyfile)?;
interruptible(py, &self.session, || {
crate::interrupt::interruptible_without_user_context(py, &self.session, || {
self.session
.add_artifacts_of_flags(&[p.as_str()], pyfile, archive, file)
.map_err(Box::new)
Expand Down Expand Up @@ -3198,7 +3198,7 @@ impl PySparkSession {
local_path: String,
dest_path: String,
) -> PyResult<()> {
interruptible(py, &self.session, || {
crate::interrupt::interruptible_without_user_context(py, &self.session, || {
self.session
.copy_from_local_to_fs(&local_path, &dest_path)
.map_err(Box::new)
Expand Down Expand Up @@ -3304,9 +3304,11 @@ impl PySparkSession {
// session open, so nothing below runs. The core runs its stops of all sessions
// one at a time too, for the stops that do not hold `SparkSession._lock`; the
// resets below are idempotent, and run with the GIL, after the core's stop.
crate::interrupt::interruptible_outcome(py, &self.session, || {
self.session.stop().map_err(Box::new)
})?
crate::interrupt::interruptible_outcome_without_user_context(
py,
&self.session,
|| self.session.stop().map_err(Box::new),
)?
} else {
// As PySpark's stop, which closes the client under its lock all the same.
let _stopping = crate::interrupt::abortable_without_session(
Expand Down Expand Up @@ -3598,7 +3600,9 @@ impl PySparkSession {
/// Release the session on the server, without stopping it, for `client.release_session`.
fn _release_session(&self, py: Python<'_>) -> PyResult<()> {
let session = self.session.clone();
interruptible(py, &session, || session.release_session().map_err(Box::new))
crate::interrupt::interruptible_without_user_context(py, &session, || {
session.release_session().map_err(Box::new)
})
}

/// Close the session without releasing it on the server, for `client.close`.
Expand Down Expand Up @@ -3967,9 +3971,10 @@ impl PySparkSession {
use prost::Message;
let request = spark_connect_proto::GetStatusRequest::decode(request)
.map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?;
let response = interruptible(py, &self.session, || {
self.session.get_status_proto(request).map_err(Box::new)
})?;
let response =
crate::interrupt::interruptible_without_user_context(py, &self.session, || {
self.session.get_status_proto(request).map_err(Box::new)
})?;
Ok(pyo3::types::PyBytes::new(py, &response.encode_to_vec()))
}

Expand Down Expand Up @@ -4356,11 +4361,12 @@ impl PySparkSession {
py: Python<'_>,
new_session_id: Option<String>,
) -> PyResult<Py<PySparkSession>> {
let session = interruptible(py, &self.session, || {
self.session
.clone_session_with_id(new_session_id.as_deref())
.map_err(Box::new)
})?;
let session =
crate::interrupt::interruptible_without_user_context(py, &self.session, || {
self.session
.clone_session_with_id(new_session_id.as_deref())
.map_err(Box::new)
})?;
session_object(py, &session)
}
}
Expand Down
16 changes: 14 additions & 2 deletions dev/parity/known-differences.md
Original file line number Diff line number Diff line change
Expand Up @@ -456,8 +456,12 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan
transport's lists of them in step, where PySpark's is a `threading.local` of a plain set and
list. The global list of the extensions is a subclass of `list` too. A set of tags refuses an
object that is neither a str nor bytes when it is added, where PySpark's raises the `TypeError`
of the field of the tags when it builds a request. A list that the attribute of a `copy.copy` of
a client is set to is not one that the requests of the session carry.
of the field of the tags when it builds a request. An extension that is not an `Any` fails each
call whose requests carry the extensions, as PySpark's requests fail when they are built, of
upb's `TypeError`, which names the runtime's class of `Any`
(`pyspark.sql.connect.proto._rust.any_pb2.Any`) where PySpark's names
`google.protobuf.any_pb2.Any`. A list that the attribute of a `copy.copy` of a client is set to
is not one that the requests of the session carry.
- **Plans of a DataFrame's relation** (intentional; DESIGN). `df._plan.plan(client)` is a
placeholder that holds the DataFrame's relation, which the core builds at the call, as
PySpark's `plan` does, with its pickles, warnings and errors: the core builds the relation, and
Expand Down Expand Up @@ -626,6 +630,14 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan

## `pyspark.sql.connect.proto` and plugins

- **Pickles of `Any` and `Timestamp`** (intentional). The drop-in's protobuf runtime has classes
of its own of the well-known types, of its modules `pyspark.sql.connect.proto._rust.any_pb2` and
`timestamp_pb2`, as `protobuf`'s modules of them (`google.protobuf.any_pb2`) are `protobuf`'s,
which the drop-in leaves as they are. So the pickle of its `Any` or `Timestamp` names that module,
and loads where the drop-in is installed, with or without `protobuf`, but not in PySpark, such as
in the Python worker of the server that runs a UDF that holds one, where upb's names
`google.protobuf.any_pb2.Any`, which loads in the drop-in as a message of `protobuf`'s class,
which the runtime's fields take as they are.
- **Generated modules of a plugin's protos** (intentional; DESIGN). A module that protoc
generates of a proto3 file that imports the protos of Spark Connect, directly or through the
files it imports (Delta Lake's `delta/connect/relations.proto` and `commands.proto`), gets the
Expand Down
15 changes: 15 additions & 0 deletions docs/compatibility.md
Original file line number Diff line number Diff line change
Expand Up @@ -1688,6 +1688,13 @@ protobuf 6's differ), the methods of protobuf's `Any` and `Timestamp` (a Timesta
`pyspark.sql.connect.client.core` imports the package, so a pickle of PySpark's
`PlanObservedMetrics`, such as one in the `attrs` of a pandas DataFrame of `toPandas`, which
imports that module before it makes its messages, loads in a new process;
- the pickle of an `Any` or a `Timestamp` names its class of the runtime, of the module
`pyspark.sql.connect.proto._rust.any_pb2` or `timestamp_pb2`, which loads where the drop-in is
installed, with or without `protobuf`, where upb's names `protobuf`'s class
(`google.protobuf.any_pb2.Any`): so a pickle of the drop-in's does not load in PySpark, such as
in the Python worker of the server that runs a UDF that holds one, and one of PySpark's loads,
where `protobuf` is installed, as a message of `protobuf`'s class, which the runtime's fields
take as they are;
- a parse that fails raises upb's message (`Error parsing message with type 'spark.connect.Plan':
Wire format was corrupt`) of the runtime's class, `pyspark.sql.connect.proto._rust.DecodeError`,
where upb raises `google.protobuf.message.DecodeError` itself. Where `protobuf` is installed (of
Expand Down Expand Up @@ -1940,6 +1947,14 @@ PySpark serializes them when it builds each request. The classes of these differ
extensions and the global list are subclasses of `set` and `list`, of a plain `repr`, whose copies
and pickles are a plain set and plain lists; a set of tags refuses an object that is neither a str
nor bytes when it is added, of the `TypeError` that PySpark raises when it builds a request of it.
The lists of the extensions take any object, as PySpark's: an extension that is not an `Any`
(another message, or not a message), or an item that is not a pair, fails each call whose requests
carry the extensions, those of executions, analyses, configurations and interrupts, with upb's
`TypeError` (or the error of the unpacking of the item) before the request is sent, of the global
extensions first, as PySpark's requests raise it when they are built, until it is removed, while
the calls of the artifacts, `cloneSession` and `stop`, whose requests carry none, are made. The
error names the runtime's class of `Any` (`pyspark.sql.connect.proto._rust.any_pb2.Any`), where
upb's names `google.protobuf.any_pb2.Any`.
A `copy.copy` of the client has the attributes of its session, but a list that the copy's
attribute is set to is not one that the requests carry. The official tests of the client
(`sql/tests/connect/client/test_client*.py`) mock the gRPC stub and the reattachable iterator of
Expand Down
48 changes: 36 additions & 12 deletions python/pyspark/sql/connect/client/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -1901,7 +1901,7 @@ def __init__(self, session: Any, extensions: Iterable[Any] = ()) -> None:
super().__init__(extensions)
self._session_ref = weakref.ref(session)

def _sync(self) -> None:
def _sync(self, at_request: bool = False) -> None:
session = self._session_ref()
if session is None:
return
Expand All @@ -1910,7 +1910,7 @@ def _sync(self) -> None:
# that the requests read, as in PySpark, and neither is a copy's.
if state is None or state.get("global_user_context_extensions") is not self:
return
session._set_global_user_context_extensions(_serialized_extensions(self))
session._set_global_user_context_extensions(_serialized_extensions(self, at_request))

def __reduce_ex__(self, protocol: Any) -> Any:
return (list, (list(self),))
Expand Down Expand Up @@ -1948,8 +1948,8 @@ def _of(cls, thread_local: Any, extensions: Any) -> Any:
session._share_thread_user_context_extensions(extensions._handle)
return extensions

def _sync(self) -> None:
self._handle._replace(_serialized_extensions(self))
def _sync(self, at_request: bool = False) -> None:
self._handle._replace(_serialized_extensions(self, at_request))

def _copy(self) -> "_ThreadExtensions":
# A list of deep copies of the extensions, which no thread has yet.
Expand Down Expand Up @@ -2017,16 +2017,38 @@ def __deepcopy__(self, memo: Any) -> set[Any]:
return copy.deepcopy(set(self), memo)


def _serialized_extensions(extensions: Iterable[Any]) -> list[tuple[Any, bytes]]:
def _serialized_extensions(
extensions: Iterable[Any], at_request: bool = False
) -> list[tuple[Any, bytes]]:
# The IDs and the bytes of the google.protobuf.Any of `extensions`, as the Rust transport takes
# them for the user context of its requests.
return [(extension_id, extension.SerializeToString()) for extension_id, extension in extensions]
# them for the user context of its requests. Each is added to the extensions of a UserContext,
# as PySpark's `_update_request_with_user_context_extensions` adds it to the request that it
# builds, so an item that is not a pair, or an extension that is not an Any (another message,
# or not a message), fails as it fails there, of upb's TypeError. The lists take it, as
# PySpark's do: a change of a list leaves it out, and the serialization before each call whose
# requests carry the extensions (`at_request`) raises its error, as each of those requests of
# PySpark raises it.
from pyspark.sql.connect.proto import UserContext

serialized = []
for item in extensions:
try:
extension_id, extension = item
context = UserContext()
context.extensions.append(extension)
except Exception:
if at_request:
raise
continue
serialized.append((extension_id, context.extensions[0].SerializeToString()))
return serialized


def _synced(base: type, name: str) -> Any:
# A method of `base`, a list or a set, that changes it, after which the Rust transport takes
# the extensions or the tags: a change that it does not take, of an object that it cannot
# send, is undone, and its error raised.
# the extensions or the tags: a change that it does not take, of a tag that it cannot send,
# is undone, and its error raised. An extension that it cannot send stays in the list, and
# fails each call (see `_serialized_extensions`).
method = getattr(base, name)

def synced(self: Any, *args: Any, **kwargs: Any) -> Any:
Expand Down Expand Up @@ -2179,7 +2201,9 @@ def _refresh_user_context_extensions(session_ref: Any) -> None:
"""Serialize the user context extensions of the session of ``session_ref`` again for the
Rust transport, of its clients' global list and of the list of the current thread, before a
call of the session sends requests, as PySpark serializes them when it builds each request:
so a ``google.protobuf.Any`` of the lists that was changed in place is sent as it is now."""
so a ``google.protobuf.Any`` of the lists that was changed in place is sent as it is now, and
an extension that is not an Any raises upb's ``TypeError`` from the call, of the global list
first, as PySpark's request adds the global extensions first."""
session = session_ref()
if session is None:
return
Expand All @@ -2188,10 +2212,10 @@ def _refresh_user_context_extensions(session_ref: Any) -> None:
return
extensions = state.get("global_user_context_extensions")
if isinstance(extensions, _GlobalUserContextExtensions) and extensions:
extensions._sync()
extensions._sync(at_request=True)
own = getattr(state.get("thread_local"), "user_context_extensions", None)
if isinstance(own, _ThreadExtensions) and own:
own._sync()
own._sync(at_request=True)


# The client of each session, by its ID, while it is in use.
Expand Down
12 changes: 7 additions & 5 deletions python/pyspark/sql/connect/proto/_rust.py
Original file line number Diff line number Diff line change
Expand Up @@ -813,14 +813,16 @@ def install(package: dict[str, _Any]) -> None:
"""Export the messages of Spark Connect into ``package``, the globals of
``pyspark.sql.connect.proto``, as its ``from ..._pb2 import *`` of the generated modules does
(so its ``DESCRIPTOR`` is that of the last, ``pipelines_pb2``), make each generated module
(``pyspark.sql.connect.proto.base_pb2``, ...) the one of this runtime, and, when protobuf is
installed, add the files of the protos to its pool, give the modules that protoc generates of
other protos that import them the classes of the runtime, and make its ``UnknownFieldSet``
take the messages of the runtime."""
(``pyspark.sql.connect.proto.base_pb2``, ...) the one of this runtime, put the modules of the
well-known types of the runtime (``pyspark.sql.connect.proto._rust.any_pb2`` and
``timestamp_pb2``) in ``sys.modules``, so that their messages pickle, of their classes, as the
others do, and, when protobuf is installed, add the files of the protos to its pool, give the
modules that protoc generates of other protos that import them the classes of the runtime,
and make its ``UnknownFieldSet`` take the messages of the runtime."""
prefix = "pyspark.sql.connect.proto."
for module_name, module in _MODULES.items():
sys.modules[module_name] = module
if _is_exported(module_name):
sys.modules[module_name] = module
package[module_name[len(prefix) :]] = module
for stem in _EXPORTED:
for name, value in vars(_MODULES[f"{prefix}{stem}_pb2"]).items():
Expand Down
Loading
Loading