From 8517f598635e937e382dd14c413dc97ad7b36416 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Thu, 19 Nov 2026 01:55:18 +0000 Subject: [PATCH] [SPARK-62061][CONNECT][PYTHON] Read a path of addArtifacts that is not a str as PySpark's ArtifactManager, and define the client's results in pyspark.sql.connect.client.core with PySpark's import * of its package PySpark's SparkConnectClient.add_artifacts (connect/client/core.py:2461-2470) reads, checks and uploads its paths one after the other, and its ArtifactManager parses each with urlparse (artifact.py:184-203), after SparkSession.addArtifacts has checked its flags (connect/session.py:1110-1121). So a path that is not a str raises urlparse's error: the AttributeError of the `decode` that a pathlib.Path or an int does not have, or, of bytes or of a falsy object, which urlparse parses into a scheme of bytes, UNSUPPORTED_OPERATION of that scheme. The drop-in's addArtifacts, addArtifact and client.add_artifacts took the paths as a list of str, so any other object raised PyO3's TypeError before the flags were checked and before the paths before it were uploaded. PySpark's pyspark.sql.connect.client package is `from pyspark.sql.connect.client.core import *` and its getLogLevel, with no `__all__`: its `import *` gives the four classes of the `__all__` of its core, getLogLevel and its modules core, artifact, reattach and retries. AnalyzeResult, ConfigResult and PlanObservedMetrics are of its core only. The drop-in's package, which is the module of its client, defined those three classes and exported only the four. - The binding reads each path of addArtifacts and _add_artifacts_of_flags as PySpark's ArtifactManager does: a str as it is, a Windows absolute path of another object as the file URI that PureWindowsPath and Path.resolve make of it, and urlparse's error, or UNSUPPORTED_OPERATION of a scheme of bytes, otherwise. Each path is uploaded before the next is read. - AnalyzeResult, ConfigResult and PlanObservedMetrics are defined in pyspark.sql.connect.client.core, which has the metrics classes of PySpark's core too, and the package imports them as private names. Its `__all__` is the names of PySpark's `import *` of the package but its gRPC modules: the four classes, core and getLogLevel. The package sets the four classes in core once it has defined them; the binding reads PlanObservedMetrics there. - The mock server of the offline suite records the first request of each AddArtifacts. - docs/compatibility.md describes the modules, and that an error that names the type of a singleton type class, as the NOT_EXPECTED_TYPE of udf(f, IntegerType), names `type`, the metaclass of the native class. Co-authored-by: Isaac --- crates/pyspark-rs/src/dataframe.rs | 2 +- crates/pyspark-rs/src/session.rs | 77 ++++++- docs/compatibility.md | 13 +- python/pyspark/sql/connect/client/__init__.py | 214 +++--------------- python/pyspark/sql/connect/client/core.py | 200 ++++++++++++++-- python/tests/_mock_connect_server.py | 4 + python/tests/test_dropin_offline.py | 74 +++++- 7 files changed, 374 insertions(+), 210 deletions(-) diff --git a/crates/pyspark-rs/src/dataframe.rs b/crates/pyspark-rs/src/dataframe.rs index 3cc7c2f1..dbccbe2e 100644 --- a/crates/pyspark-rs/src/dataframe.rs +++ b/crates/pyspark-rs/src/dataframe.rs @@ -7826,7 +7826,7 @@ pub(crate) fn execution_metrics<'py>( } } let observed_cls = py - .import("pyspark.sql.connect.client")? + .import("pyspark.sql.connect.client.core")? .getattr("PlanObservedMetrics")?; // The metrics as PySpark's client keeps them, the `Expression.Literal` messages that the // server sent, of `pyspark.sql.connect.proto`, which `PlanObservedMetrics.pairs` converts diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs index 620c80f5..e643f2bb 100644 --- a/crates/pyspark-rs/src/session.rs +++ b/crates/pyspark-rs/src/session.rs @@ -1567,6 +1567,23 @@ impl PySparkSession { /// a path is raised once the paths before it are uploaded, which the session keeps. The flags, /// `pyfile`, `archive` and `file`, are any objects, of which PySpark's `_parse_artifacts` /// tests the truth (`if pyfile and ...`), as this does once there is a path. + /// Upload the artifacts of `paths`, of the flags `flags`, one path after the other, as + /// PySpark's `SparkConnectClient.add_artifacts` reads, checks and uploads each path before the + /// next (see `artifact_path_of`): so a path that is not supported raises once the paths before + /// it are uploaded. + fn upload_artifact_paths( + &self, + py: Python<'_>, + paths: &[Bound<'_, PyAny>], + flags: &[Bound<'_, PyAny>; 3], + ) -> PyResult<()> { + for path in paths { + let path = artifact_path_of(py, path)?; + self.upload_artifacts(py, vec![path], flags)?; + } + Ok(()) + } + fn upload_artifacts( &self, py: Python<'_>, @@ -3773,17 +3790,18 @@ impl PySparkSession { /// Add local files, or `file:` URIs, as artifacts of the session, as PySpark's /// `SparkConnectClient.add_artifacts(*path, pyfile, archive, file)`: as `addArtifacts`, but of /// any of the flags, each path named by the first flag that it matches, as PySpark's client - /// does not check them: it only tests their truth. + /// does not check them: it only tests their truth. Its paths are read as those of + /// `addArtifacts`. #[pyo3(signature = (*path, pyfile, archive, file))] fn _add_artifacts_of_flags( &self, py: Python<'_>, - path: Vec, + path: Vec>, pyfile: Bound<'_, PyAny>, archive: Bound<'_, PyAny>, file: Bound<'_, PyAny>, ) -> PyResult<()> { - self.upload_artifacts(py, path, &[pyfile, archive, file]) + self.upload_artifact_paths(py, &path, &[pyfile, archive, file]) } /// The retry policy of the RPCs of the session, as the keyword arguments of PySpark's @@ -4311,7 +4329,9 @@ impl PySparkSession { /// the flags and its extension, and the errors for a path that is not supported are /// PySpark's (see `spark_connect_core::artifact::parse_artifact`). A class file is refused, /// as PySpark's ArtifactManager does not upload one. The flags are any objects, as PySpark's: - /// `sum([file, pyfile, archive]) > 1` refuses them first, and each path tests their truth. + /// `sum([file, pyfile, archive]) > 1` refuses them first, and each path tests their truth. The + /// paths are uploaded one after the other, and one that is not a str is read as PySpark's + /// ArtifactManager reads it, of the error of its `urlparse` (see `artifact_path_of`). #[pyo3( name = "addArtifacts", signature = (*path, pyfile=DefaultedOption::Omitted, archive=DefaultedOption::Omitted, file=DefaultedOption::Omitted), @@ -4320,13 +4340,13 @@ impl PySparkSession { fn add_artifacts( &self, py: Python<'_>, - path: Vec, + path: Vec>, pyfile: DefaultedOption, archive: DefaultedOption, file: DefaultedOption, ) -> PyResult<()> { let flags = session_artifact_flags(py, &pyfile, &archive, &file)?; - self.upload_artifacts(py, path, &flags) + self.upload_artifact_paths(py, &path, &flags) } /// `addArtifact` is `addArtifacts`, as in PySpark. @@ -4338,7 +4358,7 @@ impl PySparkSession { fn add_artifact( &self, py: Python<'_>, - path: Vec, + path: Vec>, pyfile: DefaultedOption, archive: DefaultedOption, file: DefaultedOption, @@ -4395,6 +4415,49 @@ fn unsupported_artifact_error(py: Python<'_>, unsupported: UnsupportedArtifact) } } +/// The path or URI of `path`, a path of `addArtifacts` or of `client.add_artifacts`, as PySpark's +/// ArtifactManager reads it (`_parse_artifacts`): a str as it is. Of another object, the file URI +/// that `Path(path).resolve().as_uri()` makes of a Windows absolute path, as PySpark converts one +/// of `PureWindowsPath`, whose `Exception` it ignores; else `urlparse` raises its error of the +/// object, as the `AttributeError` of the `decode` that a `pathlib.Path` or an int does not have, +/// or parses it, as bytes or a falsy object, into a scheme of bytes, which is neither empty nor +/// `file` as PySpark compares it, so not supported. +fn artifact_path_of(py: Python<'_>, path: &Bound<'_, PyAny>) -> PyResult { + if path.is_instance_of::() { + return path.extract(); + } + let pathlib = py.import("pathlib")?; + let windows_uri = || -> PyResult> { + let windows = pathlib.getattr("PureWindowsPath")?.call1((path,))?; + if !(windows.call_method0("is_absolute")?.is_truthy()? + && windows.getattr("drive")?.is_truthy()?) + { + return Ok(None); + } + let uri = pathlib + .getattr("Path")? + .call1((path,))? + .call_method0("resolve")? + .call_method0("as_uri")?; + Ok(Some(uri.extract()?)) + }; + match windows_uri() { + Ok(Some(uri)) => return Ok(uri), + Ok(None) => {} + Err(error) if error.is_instance_of::(py) => {} + Err(error) => return Err(error), + } + let parsed = py + .import("urllib.parse")? + .getattr("urlparse")? + .call1((path,))?; + let scheme = parsed.getattr("scheme")?.str()?.to_string(); + Err(unsupported_artifact_error( + py, + UnsupportedArtifact::Scheme(scheme), + )) +} + /// The flags `pyfile`, `archive` and `file` of `SparkSession.addArtifacts`, of `False` when they /// are omitted, as PySpark's checks them before anything else: `sum([file, pyfile, archive]) > 1`, /// as Python adds and compares them, raises `INVALID_MULTIPLE_ARGUMENT_CONDITIONS`, so a single diff --git a/docs/compatibility.md b/docs/compatibility.md index 43d73557..43820e3c 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -2282,7 +2282,9 @@ their class. PySpark's singleton types, from `NullType` and `IntegerType` to `CalendarIntervalType` and `IntegralType`, are classes of the metaclass `DataTypeSingleton`, which `type(IntegerType)` shows. The drop-in's are native classes, whose metaclass is `type`: PyO3 has no option of a metaclass of -a class, and CPython refuses to assign the `__class__` of a class of `type` after it is made. +a class, and CPython refuses to assign the `__class__` of a class of `type` after it is made. So an +error that names the type of such a class, as the `NOT_EXPECTED_TYPE` of `udf(f, IntegerType)` (of +the class, not of an instance), names `type` where PySpark's names `DataTypeSingleton`. `pyspark.sql.types` has PySpark's `DataTypeSingleton`, of which a class is made as PySpark's makes it, such as a subclass of a singleton type of `__main__`, which PySpark's cloudpickle sends by value of its metaclass: it loads, of one instance, as in PySpark. The constructors of the native @@ -2751,9 +2753,12 @@ SparkSession, its `conf`, `catalog`, `read`, `readStream`, `streams`, `udf`, `ud its windows, and an `Observation`. So are `SparkSession.Hook`, nested in the session class, and the classes that `pyspark.sql.connect.client` names, which are of `pyspark.sql.connect.client.core`: its channel builders, `SparkConnectClient` and `RpcDeadlines`, which PySpark's module names too, of -the `__all__` of its `core`, and `PlanObservedMetrics`, `AnalyzeResult` and `ConfigResult`, classes -of PySpark's `core` that only the drop-in's module names, with its own helpers and the names that -it imports, as the drop-in's client is that module. +the `__all__` of its `core`. `PlanObservedMetrics`, `AnalyzeResult` and `ConfigResult` are of that +`core` only, as PySpark's. The drop-in's `pyspark.sql.connect.client` has its own helpers and the +names that it imports too, as the drop-in's client is that module, and an `__all__`, of the names +that PySpark's `from pyspark.sql.connect.client import *` gives of its package, which has none: +those classes, `getLogLevel` and `core`, but for its modules `artifact` and `reattach`, which the +transport replaces, and `retries`, which imports grpc, and is imported when it is read. The functions of `pyspark.sql.connect.functions` are of its modules (`builtin` and `partitioning`), and so are those of `pyspark.sql.connect.avro.functions` and `pyspark.sql.connect.protobuf.functions`: they are not those of `pyspark.sql.functions`, diff --git a/python/pyspark/sql/connect/client/__init__.py b/python/pyspark/sql/connect/client/__init__.py index 93aa2ba9..c0f8fe72 100644 --- a/python/pyspark/sql/connect/client/__init__.py +++ b/python/pyspark/sql/connect/client/__init__.py @@ -24,7 +24,19 @@ policies, the reattachable executions, the result chunking and the RPC deadlines of the client are those of the transport.""" -__all__ = ["ChannelBuilder", "DefaultChannelBuilder", "RpcDeadlines", "SparkConnectClient"] +# The names that `from pyspark.sql.connect.client import *` gives of PySpark's package, which has +# no `__all__`: the classes of the `__all__` of its core, its `getLogLevel` and its core module. +# PySpark's gives its modules `artifact`, `reattach` and `retries` too, which its core imports: +# the transport replaces the first two, and `retries` is imported once it is read, as it imports +# grpc, which the client does not need otherwise. +__all__ = [ + "ChannelBuilder", + "DefaultChannelBuilder", + "RpcDeadlines", + "SparkConnectClient", + "core", + "getLogLevel", +] import copy @@ -53,11 +65,19 @@ from pyspark.sql.connect.logging import logger # As PySpark's package, which exports the log level of Spark Connect's logger. from pyspark.sql.connect.logging import getLogLevel # noqa: F401 -from pyspark.sql.metrics import ExecutionInfo, ObservedMetrics, PlanMetrics +from pyspark.sql.metrics import ExecutionInfo, PlanMetrics from pyspark.sql.types import DataType, StructType from pyspark.util import PythonEvalType from pyspark.version import __version__ +# The results of the analyses, the configurations and the observed metrics of the client, which +# PySpark's core defines, and its package does not have. +from pyspark.sql.connect.client.core import ( + AnalyzeResult as _AnalyzeResult, + ConfigResult as _ConfigResult, + PlanObservedMetrics as _PlanObservedMetrics, +) + if TYPE_CHECKING: import pandas as pd import pyarrow as pa @@ -66,7 +86,6 @@ from pyspark.sql.connect.client.retries import RetryPolicy from pyspark.sql.connect.observation import Observation from pyspark.sql.datasource import DataSource - from pyspark.storagelevel import StorageLevel @dataclass(frozen=True) @@ -439,174 +458,6 @@ def __repr__(self) -> str: return f"_UnsupportedLiteral({self.literal})" -class PlanObservedMetrics(ObservedMetrics): - """Observed metrics of one ``observe(...)`` node, as carried by - ``DataFrame.executionInfo.flows``, as PySpark Connect's: its ``metrics`` are the - ``Expression.Literal`` messages (of ``pyspark.sql.connect.proto``) that the server sent, - which ``pairs`` converts with ``LiteralExpression._to_value``, and so raises PySpark's - ``UNSUPPORTED_LITERAL`` of a literal that it does not convert.""" - - # As PySpark Connect's, of pyspark.sql.connect.client.core, whose name this class has too. - __module__ = "pyspark.sql.connect.client.core" - - def __init__(self, name: str, metrics: list["pb2.Expression.Literal"], keys: list[str]): - self._name = name - self._metrics = metrics - self._keys = keys if keys else [f"observed_metric_{i}" for i in range(len(self.metrics))] - - def __repr__(self) -> str: - return f"Plan observed({self._name}={self._metrics})" - - @property - def name(self) -> str: - return self._name - - @property - def metrics(self) -> list["pb2.Expression.Literal"]: - return self._metrics - - @property - def pairs(self) -> dict[str, Any]: - from pyspark.sql.connect.expressions import LiteralExpression - - result = {} - for x in range(len(self._metrics)): - result[self.keys[x]] = LiteralExpression._to_value(self.metrics[x]) - return result - - @property - def keys(self) -> list[str]: - return self._keys - - def to_dict(self) -> dict[str, Any]: - """Return a JSON-serializable dictionary representation of this observed metrics, - with the keys ``name``, ``keys`` and ``pairs``, as PySpark Connect's.""" - return { - "name": self._name, - "keys": self._keys, - "pairs": self.pairs, - } - - -class AnalyzeResult: - """The result of an analysis of SparkConnectClient's ``_analyze``, as PySpark Connect's: the - one field of the result of the analysis, and None for the others.""" - - # As PySpark Connect's, of pyspark.sql.connect.client.core, whose name this class has too. - __module__ = "pyspark.sql.connect.client.core" - - def __init__( - self, - schema: Optional[DataType], - explain_string: Optional[str], - tree_string: Optional[str], - is_local: Optional[bool], - is_streaming: Optional[bool], - input_files: Optional[list[str]], - spark_version: Optional[str], - parsed: Optional[DataType], - is_same_semantics: Optional[bool], - semantic_hash: Optional[int], - storage_level: Optional["StorageLevel"], - ddl_string: Optional[str], - ): - self.schema = schema - self.explain_string = explain_string - self.tree_string = tree_string - self.is_local = is_local - self.is_streaming = is_streaming - self.input_files = input_files - self.spark_version = spark_version - self.parsed = parsed - self.is_same_semantics = is_same_semantics - self.semantic_hash = semantic_hash - self.storage_level = storage_level - self.ddl_string = ddl_string - - @classmethod - def fromProto(cls, pb: Any) -> "AnalyzeResult": - from pyspark.sql.connect import types - from pyspark.sql.connect.conversion import proto_to_storage_level - - schema: Optional[DataType] = None - explain_string: Optional[str] = None - tree_string: Optional[str] = None - is_local: Optional[bool] = None - is_streaming: Optional[bool] = None - input_files: Optional[list[str]] = None - spark_version: Optional[str] = None - parsed: Optional[DataType] = None - is_same_semantics: Optional[bool] = None - semantic_hash: Optional[int] = None - storage_level: Optional[StorageLevel] = None - ddl_string: Optional[str] = None - - if pb.HasField("schema"): - schema = types.proto_schema_to_pyspark_data_type(pb.schema.schema) - elif pb.HasField("explain"): - explain_string = pb.explain.explain_string - elif pb.HasField("tree_string"): - tree_string = pb.tree_string.tree_string - elif pb.HasField("is_local"): - is_local = pb.is_local.is_local - elif pb.HasField("is_streaming"): - is_streaming = pb.is_streaming.is_streaming - elif pb.HasField("input_files"): - input_files = pb.input_files.files - elif pb.HasField("spark_version"): - spark_version = pb.spark_version.version - elif pb.HasField("ddl_parse"): - parsed = types.proto_schema_to_pyspark_data_type(pb.ddl_parse.parsed) - elif pb.HasField("same_semantics"): - is_same_semantics = pb.same_semantics.result - elif pb.HasField("semantic_hash"): - semantic_hash = pb.semantic_hash.result - elif pb.HasField("persist"): - pass - elif pb.HasField("unpersist"): - pass - elif pb.HasField("get_storage_level"): - storage_level = proto_to_storage_level(pb.get_storage_level.storage_level) - elif pb.HasField("json_to_ddl"): - ddl_string = pb.json_to_ddl.ddl_string - else: - raise SparkConnectException("No analyze result found!") - - return AnalyzeResult( - schema, - explain_string, - tree_string, - is_local, - is_streaming, - input_files, - spark_version, - parsed, - is_same_semantics, - semantic_hash, - storage_level, - ddl_string, - ) - - -class ConfigResult: - """The result of SparkConnectClient's ``config``, as PySpark Connect's: the pairs of a key and - its value, None for a key without one, and the warnings of the server.""" - - # As PySpark Connect's, of pyspark.sql.connect.client.core, whose name this class has too. - __module__ = "pyspark.sql.connect.client.core" - - def __init__(self, pairs: list[tuple[str, Optional[str]]], warnings: list[str]): - self.pairs = pairs - self.warnings = warnings - - @classmethod - def fromProto(cls, pb: "pb2.ConfigResponse") -> "ConfigResult": - return ConfigResult( - pairs=[(pair.key, pair.value if pair.HasField("value") else None) for pair in pb.pairs], - warnings=list(pb.warnings), - ) - - def _self_destruct_table(table: Any) -> Any: """``table``, of the record batches of a result, each a chunk of its columns, with each column of each batch copied into its own allocation, as ``SparkConnectClient._execute_and_fetch`` @@ -1026,7 +877,7 @@ def get_config_with_defaults( ``get_config_with_defaults``, of which pandas-on-Spark reads its ANSI mode.""" return tuple(self._session.conf._get_with_defaults(list(pairs))) - def config(self, operation: "pb2.ConfigRequest.Operation") -> ConfigResult: + def config(self, operation: "pb2.ConfigRequest.Operation") -> _ConfigResult: """Send ``operation``, a ``ConfigRequest.Operation``, as SparkConnectClient's ``config``: the result of the configuration request of the session.""" import pyspark.sql.connect.proto as pb2 @@ -1034,9 +885,9 @@ def config(self, operation: "pb2.ConfigRequest.Operation") -> ConfigResult: req = pb2.ConfigRequest() req.operation.CopyFrom(operation) response = pb2.ConfigResponse.FromString(self._session._config(req.SerializeToString())) - return ConfigResult.fromProto(response) + return _ConfigResult.fromProto(response) - def _analyze(self, method: str, **kwargs: Any) -> AnalyzeResult: + def _analyze(self, method: str, **kwargs: Any) -> _AnalyzeResult: """The result of the analysis ``method`` of the arguments ``kwargs``, as SparkConnectClient's ``_analyze``: an AnalyzePlanRequest of the plans or the relations it is given, as they are.""" @@ -1107,7 +958,7 @@ def _analyze(self, method: str, **kwargs: Any) -> AnalyzeResult: }, ) data = self._session._analyze_plan(req.SerializeToString()) - return AnalyzeResult.fromProto(pb2.AnalyzePlanResponse.FromString(data)) + return _AnalyzeResult.fromProto(pb2.AnalyzePlanResponse.FromString(data)) def schema(self, plan: "pb2.Plan") -> StructType: """The schema of ``plan``, a ``spark.connect.Plan``, as SparkConnectClient's ``schema``: @@ -1240,7 +1091,7 @@ def _execute_and_fetch( its plan metrics, its observed metrics, its properties, and its operation ID.""" import pyarrow as pa - observed_metrics: list[PlanObservedMetrics] = [] + observed_metrics: list[_PlanObservedMetrics] = [] metrics: list[PlanMetrics] = [] batches: list[pa.RecordBatch] = [] schema: Optional[StructType] = None @@ -1254,7 +1105,7 @@ def _execute_and_fetch( batches.append(response) elif isinstance(response, PlanMetrics): metrics.append(response) - elif isinstance(response, PlanObservedMetrics): + elif isinstance(response, _PlanObservedMetrics): observed_metrics.append(response) elif isinstance(response, dict): properties.update(**response) @@ -2437,3 +2288,12 @@ def _log_grpc_error(file: str, line: int, message: str) -> None: except OSError: # As gRPC's core, which goes on when stderr is closed. pass + + +# The classes of the `__all__` of PySpark's core, which official code and pickles read there: the +# core module was imported before they were defined. +from pyspark.sql.connect.client import core as _core # noqa: E402 + +for _name in _core.__all__: + setattr(_core, _name, globals()[_name]) +del _core, _name diff --git a/python/pyspark/sql/connect/client/core.py b/python/pyspark/sql/connect/client/core.py index 9fd9512b..a6e36a67 100644 --- a/python/pyspark/sql/connect/client/core.py +++ b/python/pyspark/sql/connect/client/core.py @@ -14,29 +14,44 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Thin re-export of the client classes of ``pyspark.sql.connect.client.core``, which official -PySpark code and tests import from there, and of which these classes are, as PySpark's: so a -pickle names them here. The gRPC client itself is the Rust transport: the drop-in does not -provide the rest of ``core`` (the gRPC stub of the client).""" +"""The module of PySpark's ``pyspark.sql.connect.client.core``, which official PySpark code and +tests import, and of which its classes are, as PySpark's: so a pickle names them here. The results +of the analyses, the configurations and the observed metrics of the client are defined here, as +PySpark's are; the classes of ``__all__`` are those of the package, which sets them here once it +has defined them. The gRPC client itself is the Rust transport: the drop-in does not provide the +rest of ``core`` (the gRPC stub of the client).""" __all__ = ["ChannelBuilder", "DefaultChannelBuilder", "RpcDeadlines", "SparkConnectClient"] -from typing import Any +from typing import TYPE_CHECKING, Any, Optional # As PySpark's core, which imports the messages of Spark Connect, so that protobuf's default pool # has their files once this module is imported: a pickle of PySpark's PlanObservedMetrics, which # imports this module before it makes the Expression.Literal messages of its metrics of their full # names (protobuf's `_InternalConstructMessage`), loads in a new process, as with PySpark. import pyspark.sql.connect.proto # noqa: F401 -from pyspark.sql.connect.client import ( - AnalyzeResult, # noqa: F401 - ChannelBuilder, - ConfigResult, # noqa: F401 - DefaultChannelBuilder, - PlanObservedMetrics, # noqa: F401 - RpcDeadlines, - SparkConnectClient, +from pyspark.errors.exceptions.connect import SparkConnectException + +# As PySpark's core, which has the metrics classes of the execution info of its results. +from pyspark.sql.metrics import ( # noqa: F401 + ExecutionInfo, + MetricValue, + ObservedMetrics, + PlanMetrics, ) +from pyspark.sql.types import DataType + +if TYPE_CHECKING: + import pyspark.sql.connect.proto as pb2 + + # The classes of `__all__`, which the package sets here once it has defined them. + from pyspark.sql.connect.client import ( + ChannelBuilder, + DefaultChannelBuilder, + RpcDeadlines, + SparkConnectClient, + ) + from pyspark.storagelevel import StorageLevel # The names of pyspark.sql.connect.client.retries that PySpark's core imports, and so has too. _RETRIES_NAMES = ( @@ -55,3 +70,162 @@ def __getattr__(name: str) -> Any: return getattr(retries, name) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +class PlanObservedMetrics(ObservedMetrics): + """Observed metrics of one ``observe(...)`` node, as carried by + ``DataFrame.executionInfo.flows``, as PySpark Connect's: its ``metrics`` are the + ``Expression.Literal`` messages (of ``pyspark.sql.connect.proto``) that the server sent, + which ``pairs`` converts with ``LiteralExpression._to_value``, and so raises PySpark's + ``UNSUPPORTED_LITERAL`` of a literal that it does not convert.""" + + def __init__(self, name: str, metrics: list["pb2.Expression.Literal"], keys: list[str]): + self._name = name + self._metrics = metrics + self._keys = keys if keys else [f"observed_metric_{i}" for i in range(len(self.metrics))] + + def __repr__(self) -> str: + return f"Plan observed({self._name}={self._metrics})" + + @property + def name(self) -> str: + return self._name + + @property + def metrics(self) -> list["pb2.Expression.Literal"]: + return self._metrics + + @property + def pairs(self) -> dict[str, Any]: + from pyspark.sql.connect.expressions import LiteralExpression + + result = {} + for x in range(len(self._metrics)): + result[self.keys[x]] = LiteralExpression._to_value(self.metrics[x]) + return result + + @property + def keys(self) -> list[str]: + return self._keys + + def to_dict(self) -> dict[str, Any]: + """Return a JSON-serializable dictionary representation of this observed metrics, + with the keys ``name``, ``keys`` and ``pairs``, as PySpark Connect's.""" + return { + "name": self._name, + "keys": self._keys, + "pairs": self.pairs, + } + + +class AnalyzeResult: + """The result of an analysis of SparkConnectClient's ``_analyze``, as PySpark Connect's: the + one field of the result of the analysis, and None for the others.""" + + def __init__( + self, + schema: Optional[DataType], + explain_string: Optional[str], + tree_string: Optional[str], + is_local: Optional[bool], + is_streaming: Optional[bool], + input_files: Optional[list[str]], + spark_version: Optional[str], + parsed: Optional[DataType], + is_same_semantics: Optional[bool], + semantic_hash: Optional[int], + storage_level: Optional["StorageLevel"], + ddl_string: Optional[str], + ): + self.schema = schema + self.explain_string = explain_string + self.tree_string = tree_string + self.is_local = is_local + self.is_streaming = is_streaming + self.input_files = input_files + self.spark_version = spark_version + self.parsed = parsed + self.is_same_semantics = is_same_semantics + self.semantic_hash = semantic_hash + self.storage_level = storage_level + self.ddl_string = ddl_string + + @classmethod + def fromProto(cls, pb: Any) -> "AnalyzeResult": + from pyspark.sql.connect import types + from pyspark.sql.connect.conversion import proto_to_storage_level + + schema: Optional[DataType] = None + explain_string: Optional[str] = None + tree_string: Optional[str] = None + is_local: Optional[bool] = None + is_streaming: Optional[bool] = None + input_files: Optional[list[str]] = None + spark_version: Optional[str] = None + parsed: Optional[DataType] = None + is_same_semantics: Optional[bool] = None + semantic_hash: Optional[int] = None + storage_level: Optional[StorageLevel] = None + ddl_string: Optional[str] = None + + if pb.HasField("schema"): + schema = types.proto_schema_to_pyspark_data_type(pb.schema.schema) + elif pb.HasField("explain"): + explain_string = pb.explain.explain_string + elif pb.HasField("tree_string"): + tree_string = pb.tree_string.tree_string + elif pb.HasField("is_local"): + is_local = pb.is_local.is_local + elif pb.HasField("is_streaming"): + is_streaming = pb.is_streaming.is_streaming + elif pb.HasField("input_files"): + input_files = pb.input_files.files + elif pb.HasField("spark_version"): + spark_version = pb.spark_version.version + elif pb.HasField("ddl_parse"): + parsed = types.proto_schema_to_pyspark_data_type(pb.ddl_parse.parsed) + elif pb.HasField("same_semantics"): + is_same_semantics = pb.same_semantics.result + elif pb.HasField("semantic_hash"): + semantic_hash = pb.semantic_hash.result + elif pb.HasField("persist"): + pass + elif pb.HasField("unpersist"): + pass + elif pb.HasField("get_storage_level"): + storage_level = proto_to_storage_level(pb.get_storage_level.storage_level) + elif pb.HasField("json_to_ddl"): + ddl_string = pb.json_to_ddl.ddl_string + else: + raise SparkConnectException("No analyze result found!") + + return AnalyzeResult( + schema, + explain_string, + tree_string, + is_local, + is_streaming, + input_files, + spark_version, + parsed, + is_same_semantics, + semantic_hash, + storage_level, + ddl_string, + ) + + +class ConfigResult: + """The result of SparkConnectClient's ``config``, as PySpark Connect's: the pairs of a key and + its value, None for a key without one, and the warnings of the server.""" + + def __init__(self, pairs: list[tuple[str, Optional[str]]], warnings: list[str]): + self.pairs = pairs + self.warnings = warnings + + @classmethod + def fromProto(cls, pb: "pb2.ConfigResponse") -> "ConfigResult": + return ConfigResult( + pairs=[(pair.key, pair.value if pair.HasField("value") else None) for pair in pb.pairs], + warnings=list(pb.warnings), + ) diff --git a/python/tests/_mock_connect_server.py b/python/tests/_mock_connect_server.py index 62f2441e..37e41bda 100644 --- a/python/tests/_mock_connect_server.py +++ b/python/tests/_mock_connect_server.py @@ -389,6 +389,8 @@ def reset(self): self.clone_requests = [] # The GetStatusRequests of each GetStatus. self.get_status_requests = [] + # The session and the first request of each AddArtifacts. + self.add_artifacts_requests = [] # The DDL of the JSON of each JsonToDDL. self.ddl_string = "" with self.listener_condition: @@ -593,6 +595,8 @@ def add_artifacts(self, requests, context): session_id = b"" for request in requests: session_id = parse_message(request).get(1, [b""])[0] + # The first request of each AddArtifacts, of the session that sent it. + self.add_artifacts_requests.append((session_id.decode(), bytes(request))) break if self.block_add_artifacts and not self.add_artifacts_gate.wait(15): context.abort( diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 17c8412f..eec2275c 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -4060,22 +4060,37 @@ def test_channel_builder_parses_the_connection_string_and_the_session_honors_it_ def test_the_classes_of_the_client_are_of_its_core_module_as_pyspark(): - # As PySpark Connect's ChannelBuilder, DefaultChannelBuilder, PlanObservedMetrics and - # SparkConnectClient, of pyspark.sql.connect.client.core, which names them: their repr and a - # pickle by reference name them there. + # As PySpark Connect's ChannelBuilder, DefaultChannelBuilder, RpcDeadlines and + # SparkConnectClient, of pyspark.sql.connect.client.core, which names them and whose `import *` + # its package makes, and its PlanObservedMetrics, AnalyzeResult and ConfigResult, which only + # that module names: their repr and a pickle by reference name them there. The package's + # `import *` gives the names that PySpark's gives, those classes, its getLogLevel and its core + # module, but its gRPC modules. + import pyspark.sql.connect.logging from pyspark.sql.connect import client from pyspark.sql.connect.client import core - for name in ["ChannelBuilder", "DefaultChannelBuilder", "PlanObservedMetrics", - "SparkConnectClient", "RpcDeadlines", "AnalyzeResult", "ConfigResult"]: + for name in ["ChannelBuilder", "DefaultChannelBuilder", "SparkConnectClient", "RpcDeadlines"]: cls = getattr(client, name) assert (cls.__module__, cls.__qualname__) == ("pyspark.sql.connect.client.core", name) assert getattr(core, name) is cls assert pickle.loads(pickle.dumps(cls)) is cls + for name in ["PlanObservedMetrics", "AnalyzeResult", "ConfigResult"]: + cls = getattr(core, name) + assert (cls.__module__, cls.__qualname__) == ("pyspark.sql.connect.client.core", name) + assert not hasattr(client, name) + assert pickle.loads(pickle.dumps(cls)) is cls assert repr(client.DefaultChannelBuilder("sc://localhost")).startswith( "