From 74db55426a7c93621f6ff4d3c2a2bcd98742f967 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Wed, 18 Nov 2026 22:41:30 +0000 Subject: [PATCH] [SPARK-62047][CONNECT][PYTHON] Pickle the Any and Timestamp messages of the drop-in's protobuf runtime, and fail each request of a user context extension that is not an Any, as PySpark Two gaps of the drop-in against PySpark 4.3.0: 1. The messages of the well-known types of the drop-in's protobuf runtime, `Any` and `Timestamp`, did not pickle: their classes are of the runtime's modules `pyspark.sql.connect.proto._rust.any_pb2` and `timestamp_pb2`, which were not in `sys.modules`, so `pickle.dumps` raised `Can't pickle : import of module ... failed`, and cloudpickle `cannot pickle 'classmethod_descriptor' object`. `install` now puts them in `sys.modules`, so their pickles load, in another process of the drop-in too, and cloudpickle pickles them by reference. 2. A user context extension that is not an `Any` (another message, such as `pb2.Relation()`, or an object that is not a message) was sent as the bytes of another message parsed as an `Any`, or raised `AttributeError` when it was added. PySpark's lists take any object, and each request of an execution, an analysis, a configuration or an interrupt raises upb's `TypeError` when it is built, of the global extensions first. The drop-in's lists now take it too, and the serialization of the extensions before each call that sends requests (which already re-serializes them, for an `Any` changed in place) adds each to a `UserContext` of the runtime, which raises upb's error (or that of the unpacking of an item that is not a pair) before the request is sent, until the extension is removed. The calls whose requests carry no extensions, as PySpark's requests of the artifacts, `CloneSession`, `ReleaseSession` and `GetStatus` carry none (`addArtifacts`, `copyFromLocalToFs`, `cloneSession`, `stop`, `client.release_session` and `client._get_operation_statuses`), no longer serialize them before they are made (`interruptible_without_user_context`), so they are made, as in PySpark. To close these differences from PySpark (the known-differences review). Yes: the messages of `Any` and `Timestamp` pickle, and a user context extension that is not an `Any` fails each call as in PySpark rather than being sent as an `Any` of other bytes. `docs/compatibility.md` and `dev/parity/known-differences.md` say that the pickle of the drop-in's `Any` names the runtime's module, which PySpark does not have, and that the `TypeError` names the runtime's class of `Any`. New tests in `python/tests/test_proto_rust_offline.py` (pickles of the well-known types, also in a new process and without `protobuf`) and `python/tests/test_dropin_offline.py` (extensions that are not an `Any`, against the mock server: no request is sent, the other thread's requests are, removal; the calls whose requests carry no extensions are made), which fail without the change. Probes of the extensions against a Spark 4.3.0 server give the outputs of PySpark 4.3.0 (upb) line for line, but for the class of `Any` that the error names (and the grpc error of `copyFromLocalToFs` that PySpark leaves unconverted, as before). Generated-by: Claude Code Co-authored-by: Isaac --- crates/pyspark-rs/src/interrupt.rs | 42 +++++- crates/pyspark-rs/src/pbmsg/message.rs | 5 +- crates/pyspark-rs/src/session.rs | 34 +++-- dev/parity/known-differences.md | 16 ++- docs/compatibility.md | 15 +++ python/pyspark/sql/connect/client/__init__.py | 48 +++++-- python/pyspark/sql/connect/proto/_rust.py | 12 +- python/tests/test_dropin_offline.py | 124 ++++++++++++++++++ python/tests/test_proto_rust_offline.py | 59 +++++++++ 9 files changed, 316 insertions(+), 39 deletions(-) diff --git a/crates/pyspark-rs/src/interrupt.rs b/crates/pyspark-rs/src/interrupt.rs index 7e37ff04..3a8d8ca5 100644 --- a/crates/pyspark-rs/src/interrupt.rs +++ b/crates/pyspark-rs/src/interrupt.rs @@ -224,7 +224,7 @@ pub(crate) fn interruptible_outcome>>( session: &SparkSession, f: impl FnOnce() -> Result + Send, ) -> PyResult>> { - run_interruptible(py, session, f, false) + run_interruptible(py, session, f, false, true) } /// As [`interruptible_outcome`], for a call that PySpark makes in a `try` whose `except Exception` @@ -235,7 +235,33 @@ pub(crate) fn interruptible_caught>>( session: &SparkSession, f: impl FnOnce() -> Result + Send, ) -> PyResult>> { - run_interruptible(py, session, f, true) + run_interruptible(py, session, f, true, true) +} + +/// As [`interruptible`], for a call whose requests carry no user context extensions, as PySpark's +/// requests of the artifacts, `CloneSession`, `ReleaseSession` and `GetStatus` carry none: the +/// extensions are not serialized again before it (see [`refresh_user_context`]), so that one that +/// is not an `Any`, which fails each call whose requests carry them, does not fail it, as in +/// PySpark. +pub(crate) fn interruptible_without_user_context>>( + py: Python<'_>, + session: &SparkSession, + f: impl FnOnce() -> Result + Send, +) -> PyResult { + run_interruptible(py, session, f, false, false)?.to_pyerr() +} + +/// As [`interruptible_outcome`], for a call whose requests carry no user context extensions (see +/// [`interruptible_without_user_context`]). +pub(crate) fn interruptible_outcome_without_user_context< + T: Send, + E: Send + Into>, +>( + py: Python<'_>, + session: &SparkSession, + f: impl FnOnce() -> Result + Send, +) -> PyResult>> { + run_interruptible(py, session, f, false, false) } /// Bring the user context extensions of `session` up to date before a call sends its requests, @@ -243,7 +269,9 @@ pub(crate) fn interruptible_caught>>( /// `PySparkSession::_set_user_context_refresh`), which serializes the `google.protobuf.Any` /// messages of its lists again, as PySpark serializes them when it builds each request, so that /// a change of a message in place is sent. Its error is raised, as PySpark raises that of the -/// serialization from the call. +/// serialization from the call, such as upb's `TypeError` of an extension that is not an `Any`: +/// so a call whose requests carry no extensions does not run it (see +/// [`interruptible_without_user_context`]). pub(crate) fn refresh_user_context(py: Python<'_>, session: &SparkSession) -> PyResult<()> { if let Some(refresher) = session.user_context_refresher() { if let Some(refresh) = refresher.downcast_ref::>() { @@ -254,14 +282,18 @@ pub(crate) fn refresh_user_context(py: Python<'_>, session: &SparkSession) -> Py } /// The wait of [`interruptible_outcome`], of whose results a failed interrupt is the inner one -/// when `catch_interrupt_failure`. +/// when `catch_interrupt_failure`, after the user context extensions are brought up to date when +/// `refresh_extensions`. fn run_interruptible>>( py: Python<'_>, session: &SparkSession, f: impl FnOnce() -> Result + Send, catch_interrupt_failure: bool, + refresh_extensions: bool, ) -> PyResult>> { - refresh_user_context(py, session)?; + if refresh_extensions { + refresh_user_context(py, session)?; + } let watch = Arc::new(SignalWatch { session: session.clone(), interrupted: AtomicBool::new(false), diff --git a/crates/pyspark-rs/src/pbmsg/message.rs b/crates/pyspark-rs/src/pbmsg/message.rs index 5af18339..7d98cec5 100644 --- a/crates/pyspark-rs/src/pbmsg/message.rs +++ b/crates/pyspark-rs/src/pbmsg/message.rs @@ -1881,7 +1881,10 @@ impl PyMessage { /// Pickle as protobuf's messages pickle: the class, and the state of the bytes of the /// message, which `__setstate__` parses. A nested class pickles by its qualified name, which - /// the generated modules of protobuf have too, so either runtime loads the pickle of the other. + /// the generated modules of protobuf have too, so either runtime loads the pickle of the + /// other, but for the well-known types (`Any`, `Timestamp`): their classes are of the modules + /// of the runtime (`pyspark.sql.connect.proto._rust.any_pb2`), which `install` puts in + /// `sys.modules`, where upb's are of protobuf's (`google.protobuf.any_pb2`). fn __reduce__<'py>( slf: &Bound<'py, Self>, ) -> PyResult<(Bound<'py, PyType>, Bound<'py, PyTuple>, Bound<'py, PyDict>)> { diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs index 3745552b..620c80f5 100644 --- a/crates/pyspark-rs/src/session.rs +++ b/crates/pyspark-rs/src/session.rs @@ -1591,7 +1591,7 @@ impl PySparkSession { return Err(unsupported_artifact_error(py, format)); } check_local_artifact(py, &name, &local, pyfile)?; - interruptible(py, &self.session, || { + crate::interrupt::interruptible_without_user_context(py, &self.session, || { self.session .add_artifacts_of_flags(&[p.as_str()], pyfile, archive, file) .map_err(Box::new) @@ -3198,7 +3198,7 @@ impl PySparkSession { local_path: String, dest_path: String, ) -> PyResult<()> { - interruptible(py, &self.session, || { + crate::interrupt::interruptible_without_user_context(py, &self.session, || { self.session .copy_from_local_to_fs(&local_path, &dest_path) .map_err(Box::new) @@ -3304,9 +3304,11 @@ impl PySparkSession { // session open, so nothing below runs. The core runs its stops of all sessions // one at a time too, for the stops that do not hold `SparkSession._lock`; the // resets below are idempotent, and run with the GIL, after the core's stop. - crate::interrupt::interruptible_outcome(py, &self.session, || { - self.session.stop().map_err(Box::new) - })? + crate::interrupt::interruptible_outcome_without_user_context( + py, + &self.session, + || self.session.stop().map_err(Box::new), + )? } else { // As PySpark's stop, which closes the client under its lock all the same. let _stopping = crate::interrupt::abortable_without_session( @@ -3598,7 +3600,9 @@ impl PySparkSession { /// Release the session on the server, without stopping it, for `client.release_session`. fn _release_session(&self, py: Python<'_>) -> PyResult<()> { let session = self.session.clone(); - interruptible(py, &session, || session.release_session().map_err(Box::new)) + crate::interrupt::interruptible_without_user_context(py, &session, || { + session.release_session().map_err(Box::new) + }) } /// Close the session without releasing it on the server, for `client.close`. @@ -3967,9 +3971,10 @@ impl PySparkSession { use prost::Message; let request = spark_connect_proto::GetStatusRequest::decode(request) .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?; - let response = interruptible(py, &self.session, || { - self.session.get_status_proto(request).map_err(Box::new) - })?; + let response = + crate::interrupt::interruptible_without_user_context(py, &self.session, || { + self.session.get_status_proto(request).map_err(Box::new) + })?; Ok(pyo3::types::PyBytes::new(py, &response.encode_to_vec())) } @@ -4356,11 +4361,12 @@ impl PySparkSession { py: Python<'_>, new_session_id: Option, ) -> PyResult> { - let session = interruptible(py, &self.session, || { - self.session - .clone_session_with_id(new_session_id.as_deref()) - .map_err(Box::new) - })?; + let session = + crate::interrupt::interruptible_without_user_context(py, &self.session, || { + self.session + .clone_session_with_id(new_session_id.as_deref()) + .map_err(Box::new) + })?; session_object(py, &session) } } diff --git a/dev/parity/known-differences.md b/dev/parity/known-differences.md index 4c333ee4..e7c95860 100644 --- a/dev/parity/known-differences.md +++ b/dev/parity/known-differences.md @@ -456,8 +456,12 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan transport's lists of them in step, where PySpark's is a `threading.local` of a plain set and list. The global list of the extensions is a subclass of `list` too. A set of tags refuses an object that is neither a str nor bytes when it is added, where PySpark's raises the `TypeError` - of the field of the tags when it builds a request. A list that the attribute of a `copy.copy` of - a client is set to is not one that the requests of the session carry. + of the field of the tags when it builds a request. An extension that is not an `Any` fails each + call whose requests carry the extensions, as PySpark's requests fail when they are built, of + upb's `TypeError`, which names the runtime's class of `Any` + (`pyspark.sql.connect.proto._rust.any_pb2.Any`) where PySpark's names + `google.protobuf.any_pb2.Any`. A list that the attribute of a `copy.copy` of a client is set to + is not one that the requests of the session carry. - **Plans of a DataFrame's relation** (intentional; DESIGN). `df._plan.plan(client)` is a placeholder that holds the DataFrame's relation, which the core builds at the call, as PySpark's `plan` does, with its pickles, warnings and errors: the core builds the relation, and @@ -626,6 +630,14 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan ## `pyspark.sql.connect.proto` and plugins +- **Pickles of `Any` and `Timestamp`** (intentional). The drop-in's protobuf runtime has classes + of its own of the well-known types, of its modules `pyspark.sql.connect.proto._rust.any_pb2` and + `timestamp_pb2`, as `protobuf`'s modules of them (`google.protobuf.any_pb2`) are `protobuf`'s, + which the drop-in leaves as they are. So the pickle of its `Any` or `Timestamp` names that module, + and loads where the drop-in is installed, with or without `protobuf`, but not in PySpark, such as + in the Python worker of the server that runs a UDF that holds one, where upb's names + `google.protobuf.any_pb2.Any`, which loads in the drop-in as a message of `protobuf`'s class, + which the runtime's fields take as they are. - **Generated modules of a plugin's protos** (intentional; DESIGN). A module that protoc generates of a proto3 file that imports the protos of Spark Connect, directly or through the files it imports (Delta Lake's `delta/connect/relations.proto` and `commands.proto`), gets the diff --git a/docs/compatibility.md b/docs/compatibility.md index b16a75ed..43d73557 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -1688,6 +1688,13 @@ protobuf 6's differ), the methods of protobuf's `Any` and `Timestamp` (a Timesta `pyspark.sql.connect.client.core` imports the package, so a pickle of PySpark's `PlanObservedMetrics`, such as one in the `attrs` of a pandas DataFrame of `toPandas`, which imports that module before it makes its messages, loads in a new process; +- the pickle of an `Any` or a `Timestamp` names its class of the runtime, of the module + `pyspark.sql.connect.proto._rust.any_pb2` or `timestamp_pb2`, which loads where the drop-in is + installed, with or without `protobuf`, where upb's names `protobuf`'s class + (`google.protobuf.any_pb2.Any`): so a pickle of the drop-in's does not load in PySpark, such as + in the Python worker of the server that runs a UDF that holds one, and one of PySpark's loads, + where `protobuf` is installed, as a message of `protobuf`'s class, which the runtime's fields + take as they are; - a parse that fails raises upb's message (`Error parsing message with type 'spark.connect.Plan': Wire format was corrupt`) of the runtime's class, `pyspark.sql.connect.proto._rust.DecodeError`, where upb raises `google.protobuf.message.DecodeError` itself. Where `protobuf` is installed (of @@ -1940,6 +1947,14 @@ PySpark serializes them when it builds each request. The classes of these differ extensions and the global list are subclasses of `set` and `list`, of a plain `repr`, whose copies and pickles are a plain set and plain lists; a set of tags refuses an object that is neither a str nor bytes when it is added, of the `TypeError` that PySpark raises when it builds a request of it. +The lists of the extensions take any object, as PySpark's: an extension that is not an `Any` +(another message, or not a message), or an item that is not a pair, fails each call whose requests +carry the extensions, those of executions, analyses, configurations and interrupts, with upb's +`TypeError` (or the error of the unpacking of the item) before the request is sent, of the global +extensions first, as PySpark's requests raise it when they are built, until it is removed, while +the calls of the artifacts, `cloneSession` and `stop`, whose requests carry none, are made. The +error names the runtime's class of `Any` (`pyspark.sql.connect.proto._rust.any_pb2.Any`), where +upb's names `google.protobuf.any_pb2.Any`. A `copy.copy` of the client has the attributes of its session, but a list that the copy's attribute is set to is not one that the requests carry. The official tests of the client (`sql/tests/connect/client/test_client*.py`) mock the gRPC stub and the reattachable iterator of diff --git a/python/pyspark/sql/connect/client/__init__.py b/python/pyspark/sql/connect/client/__init__.py index 4f65a97a..93aa2ba9 100644 --- a/python/pyspark/sql/connect/client/__init__.py +++ b/python/pyspark/sql/connect/client/__init__.py @@ -1901,7 +1901,7 @@ def __init__(self, session: Any, extensions: Iterable[Any] = ()) -> None: super().__init__(extensions) self._session_ref = weakref.ref(session) - def _sync(self) -> None: + def _sync(self, at_request: bool = False) -> None: session = self._session_ref() if session is None: return @@ -1910,7 +1910,7 @@ def _sync(self) -> None: # that the requests read, as in PySpark, and neither is a copy's. if state is None or state.get("global_user_context_extensions") is not self: return - session._set_global_user_context_extensions(_serialized_extensions(self)) + session._set_global_user_context_extensions(_serialized_extensions(self, at_request)) def __reduce_ex__(self, protocol: Any) -> Any: return (list, (list(self),)) @@ -1948,8 +1948,8 @@ def _of(cls, thread_local: Any, extensions: Any) -> Any: session._share_thread_user_context_extensions(extensions._handle) return extensions - def _sync(self) -> None: - self._handle._replace(_serialized_extensions(self)) + def _sync(self, at_request: bool = False) -> None: + self._handle._replace(_serialized_extensions(self, at_request)) def _copy(self) -> "_ThreadExtensions": # A list of deep copies of the extensions, which no thread has yet. @@ -2017,16 +2017,38 @@ def __deepcopy__(self, memo: Any) -> set[Any]: return copy.deepcopy(set(self), memo) -def _serialized_extensions(extensions: Iterable[Any]) -> list[tuple[Any, bytes]]: +def _serialized_extensions( + extensions: Iterable[Any], at_request: bool = False +) -> list[tuple[Any, bytes]]: # The IDs and the bytes of the google.protobuf.Any of `extensions`, as the Rust transport takes - # them for the user context of its requests. - return [(extension_id, extension.SerializeToString()) for extension_id, extension in extensions] + # them for the user context of its requests. Each is added to the extensions of a UserContext, + # as PySpark's `_update_request_with_user_context_extensions` adds it to the request that it + # builds, so an item that is not a pair, or an extension that is not an Any (another message, + # or not a message), fails as it fails there, of upb's TypeError. The lists take it, as + # PySpark's do: a change of a list leaves it out, and the serialization before each call whose + # requests carry the extensions (`at_request`) raises its error, as each of those requests of + # PySpark raises it. + from pyspark.sql.connect.proto import UserContext + + serialized = [] + for item in extensions: + try: + extension_id, extension = item + context = UserContext() + context.extensions.append(extension) + except Exception: + if at_request: + raise + continue + serialized.append((extension_id, context.extensions[0].SerializeToString())) + return serialized def _synced(base: type, name: str) -> Any: # A method of `base`, a list or a set, that changes it, after which the Rust transport takes - # the extensions or the tags: a change that it does not take, of an object that it cannot - # send, is undone, and its error raised. + # the extensions or the tags: a change that it does not take, of a tag that it cannot send, + # is undone, and its error raised. An extension that it cannot send stays in the list, and + # fails each call (see `_serialized_extensions`). method = getattr(base, name) def synced(self: Any, *args: Any, **kwargs: Any) -> Any: @@ -2179,7 +2201,9 @@ def _refresh_user_context_extensions(session_ref: Any) -> None: """Serialize the user context extensions of the session of ``session_ref`` again for the Rust transport, of its clients' global list and of the list of the current thread, before a call of the session sends requests, as PySpark serializes them when it builds each request: - so a ``google.protobuf.Any`` of the lists that was changed in place is sent as it is now.""" + so a ``google.protobuf.Any`` of the lists that was changed in place is sent as it is now, and + an extension that is not an Any raises upb's ``TypeError`` from the call, of the global list + first, as PySpark's request adds the global extensions first.""" session = session_ref() if session is None: return @@ -2188,10 +2212,10 @@ def _refresh_user_context_extensions(session_ref: Any) -> None: return extensions = state.get("global_user_context_extensions") if isinstance(extensions, _GlobalUserContextExtensions) and extensions: - extensions._sync() + extensions._sync(at_request=True) own = getattr(state.get("thread_local"), "user_context_extensions", None) if isinstance(own, _ThreadExtensions) and own: - own._sync() + own._sync(at_request=True) # The client of each session, by its ID, while it is in use. diff --git a/python/pyspark/sql/connect/proto/_rust.py b/python/pyspark/sql/connect/proto/_rust.py index faf008bc..5d22d380 100644 --- a/python/pyspark/sql/connect/proto/_rust.py +++ b/python/pyspark/sql/connect/proto/_rust.py @@ -813,14 +813,16 @@ def install(package: dict[str, _Any]) -> None: """Export the messages of Spark Connect into ``package``, the globals of ``pyspark.sql.connect.proto``, as its ``from ..._pb2 import *`` of the generated modules does (so its ``DESCRIPTOR`` is that of the last, ``pipelines_pb2``), make each generated module - (``pyspark.sql.connect.proto.base_pb2``, ...) the one of this runtime, and, when protobuf is - installed, add the files of the protos to its pool, give the modules that protoc generates of - other protos that import them the classes of the runtime, and make its ``UnknownFieldSet`` - take the messages of the runtime.""" + (``pyspark.sql.connect.proto.base_pb2``, ...) the one of this runtime, put the modules of the + well-known types of the runtime (``pyspark.sql.connect.proto._rust.any_pb2`` and + ``timestamp_pb2``) in ``sys.modules``, so that their messages pickle, of their classes, as the + others do, and, when protobuf is installed, add the files of the protos to its pool, give the + modules that protoc generates of other protos that import them the classes of the runtime, + and make its ``UnknownFieldSet`` take the messages of the runtime.""" prefix = "pyspark.sql.connect.proto." for module_name, module in _MODULES.items(): + sys.modules[module_name] = module if _is_exported(module_name): - sys.modules[module_name] = module package[module_name[len(prefix) :]] = module for stem in _EXPORTED: for name, value in vars(_MODULES[f"{prefix}{stem}_pb2"]).items(): diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 9f5b2afb..17c8412f 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -26134,6 +26134,130 @@ def inherited(): assert spark.getTags() == set() and run() == ([], []) +def test_user_context_extensions_that_are_not_anys_fail_each_request_as_pyspark(mock_connect): + # As PySpark's lists of the user context extensions, which take any object: an extension that + # is not an Any (another message, or not a message), or an item that is not a pair, stays in + # its list, and each call that sends a request raises upb's TypeError (or the error of the + # unpacking of the item) before the request is sent, as PySpark's + # `_update_request_with_user_context_extensions` raises it when it builds the request of an + # execution, an analysis, a configuration or an interrupt, of the global extensions first. The + # requests of another thread, which has a list of its own, are sent, and so are those of the + # thread once the extension is removed. + import re + import threading + + import pyspark.sql.connect.proto as pb2 + from pyspark.sql.connect.proto._rust import Any + + spark, service = mock_connect + client = spark.client + good = Any(type_url="type.googleapis.com/good") + other_class = re.escape( + "Parameter to MergeFrom() must be instance of same class: expected " + " got " + "." + ) + calls = [ + lambda: spark.range(1).collect(), + lambda: spark.range(1).filter("id > 0").schema, + lambda: spark.conf.get("spark.sql.shuffle.partitions"), + lambda: spark.interruptAll(), + ] + + def sent(): + return [ + len(service.execute_requests), + len(service.analyze_requests), + len(service.config_requests), + len(service.interrupt_requests), + ] + + def sent_extensions(): + spark.range(1).collect() + request = pb2.ExecutePlanRequest.FromString(service.execute_requests[-1]) + return [extension.type_url for extension in request.user_context.extensions] + + def fails(error, exception=TypeError): + before = sent() + for call in calls: + with pytest.raises(exception, match=f"^{error}$"): + call() + assert sent() == before + + for add in ( + client.add_threadlocal_user_context_extension, + client.add_global_user_context_extension, + ): + for bad, error in [ + (pb2.Relation(range=pb2.Range(end=1)), other_class), + ("x", re.escape("Expected a message object, but got 'x'.")), + (None, re.escape("Expected a message object, but got None.")), + ]: + good_id = add(good) + bad_id = add(bad) + if add == client.add_global_user_context_extension: + held = client.global_user_context_extensions + else: + held = client.thread_local.user_context_extensions + assert held == [(good_id, good), (bad_id, bad)] + fails(error) + assert held == [(good_id, good), (bad_id, bad)] + if add == client.add_threadlocal_user_context_extension: + other = [] + thread = threading.Thread(target=lambda: other.append(sent_extensions())) + thread.start() + thread.join() + assert other == [[]] + client.remove_user_context_extension(bad_id) + assert sent_extensions() == ["type.googleapis.com/good"] + client.clear_user_context_extensions() + # The global extensions are added first, as PySpark's request adds them. + client.add_threadlocal_user_context_extension(pb2.Relation()) + client.add_global_user_context_extension("x") + fails(re.escape("Expected a message object, but got 'x'.")) + client.clear_user_context_extensions() + # A list that the attributes are set to, and an item that is not a pair of a list. + client.global_user_context_extensions = [("global_x", "x")] + fails(re.escape("Expected a message object, but got 'x'.")) + client.global_user_context_extensions = [("global_x", good)] + client.thread_local.user_context_extensions = [("threadlocal_x", pb2.Relation())] + fails(other_class) + client.thread_local.user_context_extensions = [("threadlocal_x", good)] + assert sent_extensions() == ["type.googleapis.com/good", "type.googleapis.com/good"] + client.thread_local.user_context_extensions.append("abc") + fails(r"too many values to unpack \(expected 2(, got 3)?\)", ValueError) + client.clear_user_context_extensions() + assert sent_extensions() == [] + + +def test_calls_whose_requests_carry_no_user_context_extensions_take_one_that_is_not_an_any( + mock_connect, tmp_path +): + # As PySpark's requests of the artifacts, CloneSession, ReleaseSession and GetStatus, which + # carry no user context extensions: an extension that is not an Any, which fails the calls + # whose requests carry them, does not fail these. + import pyspark.sql.connect.proto as pb2 + + spark, service = mock_connect + client = spark.client + client.add_global_user_context_extension(pb2.Relation()) + client.add_threadlocal_user_context_extension("x") + path = tmp_path / "a.txt" + path.write_text("x") + spark.addArtifacts(str(path), file=True) + client.add_artifacts(str(path), pyfile=False, archive=False, file=True) + spark.copyFromLocalToFs(str(path), "/dest/a.txt") + client.copy_from_local_to_fs(str(path), "/dest/b.txt") + clone = spark.cloneSession() + client._get_operation_statuses() + assert len(service.clone_requests) == 1 and len(service.get_status_requests) == 1 + with pytest.raises(TypeError, match="^Expected a message object|^Parameter to MergeFrom"): + spark.range(1).collect() + clone.stop() + spark.stop() + assert spark.is_stopped + + def test_client_members_that_pysparks_tests_read_are_pysparks(mock_connect): # As PySpark Connect's SparkConnectClient, whose tests and libraries read these members: # _builder is the DefaultChannelBuilder of the connection; _session_id the session ID of the diff --git a/python/tests/test_proto_rust_offline.py b/python/tests/test_proto_rust_offline.py index eeda64fb..d4d9fa20 100644 --- a/python/tests/test_proto_rust_offline.py +++ b/python/tests/test_proto_rust_offline.py @@ -2455,6 +2455,10 @@ def find_spec(self, name, path, target=None): "str": [str(relation), str(context)], "pickle": pickle.loads(pickle.dumps(relation)) == relation, "pickle nested": pickle.loads(pickle.dumps(nested)) == nested, + "pickle well-known": [ + type(pickle.loads(pickle.dumps(m))) is type(m) and pickle.loads(pickle.dumps(m)) == m + for m in (context.extensions[0], event.timestamp) + ], "deepcopy": copy.deepcopy(context) == context, "numpy": np.array(values).tolist(), "sequence": isinstance(values, collections.abc.MutableSequence), @@ -2474,6 +2478,7 @@ def find_spec(self, name, path, target=None): assert result.returncode == 0, result.stderr out = json.loads(result.stdout) assert out["pickle"] and out["pickle nested"] and out["deepcopy"] + assert out["pickle well-known"] == [True, True] assert out["numpy"] == [1.5, 2.5] and out["sequence"] and out["mapping"] assert out["enum"] == 4 assert out["decode error"] == ( @@ -3062,6 +3067,60 @@ def test_pickles_of_nested_messages_load_as_upbs(upb): assert type(loaded) is ours(full_name) and loaded == mine, full_name +def test_pickles_of_the_well_known_types_load_of_the_classes_of_the_runtime(): + # The classes of the runtime of the well-known types, Any and Timestamp, are of its modules + # pyspark.sql.connect.proto._rust.any_pb2 and timestamp_pb2, which are in sys.modules, so + # their messages pickle of their classes, as the others do, cloudpickle pickles them by + # reference, and their pickles load in another process of the drop-in, with or without + # protobuf. protobuf's are of its modules (google.protobuf.any_pb2), so, where it is + # installed, their pickles load as protobuf's classes, which the fields of the runtime take as + # they are. + from pyspark import cloudpickle + + packed = _rust.Any() + packed.Pack(proto.Relation(range=proto.Range(end=3))) + stamp = _rust.Timestamp(seconds=7, nanos=8) + assert sys.modules["pyspark.sql.connect.proto._rust.any_pb2"].Any is _rust.Any + assert sys.modules["pyspark.sql.connect.proto._rust.timestamp_pb2"].Timestamp is _rust.Timestamp + for message in (packed, stamp, proto.Relation(extension=packed).extension): + for dumps in (pickle.dumps, cloudpickle.dumps): + loaded = pickle.loads(dumps(message)) + assert type(loaded) is type(message) and loaded == message + assert pickle.loads(cloudpickle.dumps(lambda: packed))() == packed + try: + from google.protobuf import any_pb2 + except ImportError: # the drop-in runs without protobuf + any_pb2 = None + if any_pb2 is not None: + theirs = pickle.loads(pickle.dumps(any_pb2.Any.FromString(packed.SerializeToString()))) + assert type(theirs) is any_pb2.Any + assert proto.Relation(extension=theirs).extension == packed + script = textwrap.dedent( + """ + import json, pickle, sys + + packed, stamp = pickle.loads(sys.stdin.buffer.read()) + from pyspark.sql.connect import proto + from pyspark.sql.connect.proto import _rust + + relation = proto.Relation() + out = [type(packed) is _rust.Any, packed.Unpack(relation), relation.range.end] + out += [type(stamp) is _rust.Timestamp, stamp.ToJsonString()] + print(json.dumps(out)) + """ + ) + root = str(pathlib.Path(__file__).resolve().parent.parent) + result = subprocess.run( + [sys.executable, "-c", script], + input=pickle.dumps([packed, stamp]), + capture_output=True, + timeout=120, + cwd=root, + ) + assert result.returncode == 0, result.stderr.decode()[-3000:] + assert json.loads(result.stdout) == [True, True, 3, True, "1970-01-01T00:00:07.000000008Z"] + + # Unknown fields of each wire type: a varint, a fixed64, bytes, a group of a varint and bytes, a # fixed32 and bytes of a message. _UNKNOWN_FIELDS = (