diff --git a/AGENTS.md b/AGENTS.md index eea5515b..717dea71 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -214,14 +214,18 @@ history, and update via `git subtree pull`. Rigorously classify before touching anything (diff our dir vs the version tag): - **Rust-backed shims — KEEP OURS** (anything importing `pyspark._pyspark`, or an - adapter): `sql/*` top-level, `ml/connect/*`, `sql/streaming/{query,readwriter}.py`, - `errors/*`, `pyspark/__init__.py`, `util.py` (Connect-only trim), `storagelevel.py`. + adapter): most of `sql/*` top-level, `sql/connect/*` (the classes of the objects of a + session, as PySpark Connect's), `ml/connect/*`, `errors/*`, `pyspark/__init__.py`, `util.py` + (Connect-only trim), `storagelevel.py`. - **Pure upstream — VENDOR via subtree**: `cloudpickle`, `pandas` (see caveat), `_globals.py`, `loose_version.py`, `serializers.py`, `resource/*` and `sql/connect/resource/*` (PySpark's resource profiles, whose profile id the connect `ResourceProfile` creates through `SparkSession.client`), and audit for more. A directory of - shims can hold vendored files too, such as most of `errors/*` and `sql/session.py`: - `VENDORED_FILES` of `scripts/check_vendored_upstream.sh` lists every vendored file. + shims can hold vendored files too, such as most of `errors/*`, `sql/session.py` and the + classic modules of the objects of a session (`sql/{catalog,conf,readwriter,udf,udtf,group}.py`, + `sql/streaming/{query,readwriter}.py`, `sql/pandas/group_ops.py` and others), whose objects on + Spark Connect are of the classes of `sql/connect/*`, as in PySpark: `VENDORED_FILES` of + `scripts/check_vendored_upstream.sh` lists every vendored file. Subtree conversion (serial, needs a **clean working tree**, one dir at a time): ``` diff --git a/crates/pyspark-rs/src/lib.rs b/crates/pyspark-rs/src/lib.rs index 2c5e0282..2859f084 100644 --- a/crates/pyspark-rs/src/lib.rs +++ b/crates/pyspark-rs/src/lib.rs @@ -51,7 +51,6 @@ mod transport; mod tvf; mod types; mod udf_command; -mod udtf_analyze; mod values; mod window; @@ -245,11 +244,6 @@ fn _pyspark(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { // Register TVF + Observation m.add_class::()?; m.add_class::()?; - m.add( - "SkipRestOfInputTableException", - m.py() - .get_type::(), - )?; m.add_class::()?; diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs index 5ec176c9..c1c6497f 100644 --- a/crates/pyspark-rs/src/session.rs +++ b/crates/pyspark-rs/src/session.rs @@ -3448,11 +3448,13 @@ impl PySparkSession { } /// Get the UDF registration API (`spark.udf`). Returns the - /// `pyspark.sql.udf.UDFRegistration` bound to this session, so it can register - /// Python UDFs and Java UDFs/UDAFs (registerJavaFunction / registerJavaUDAF). + /// `pyspark.sql.connect.udf.UDFRegistration` bound to this session, as PySpark Connect's, so + /// it can register Python UDFs and Java UDFs/UDAFs (registerJavaFunction / registerJavaUDAF). #[getter(udf)] fn get_udf<'a>(slf: &Bound<'a, Self>, py: Python<'a>) -> PyResult> { - let cls = py.import("pyspark.sql.udf")?.getattr("UDFRegistration")?; + let cls = py + .import("pyspark.sql.connect.udf")? + .getattr("UDFRegistration")?; cls.call1((slf,)) } @@ -4062,7 +4064,7 @@ impl PySparkSession { #[getter] fn udtf<'a>(slf: &Bound<'a, Self>) -> PyResult> { slf.py() - .import("pyspark.sql.udtf")? + .import("pyspark.sql.connect.udtf")? .getattr("UDTFRegistration")? .call1((slf,)) } diff --git a/crates/pyspark-rs/src/streaming.rs b/crates/pyspark-rs/src/streaming.rs index 0b919f28..a964333d 100644 --- a/crates/pyspark-rs/src/streaming.rs +++ b/crates/pyspark-rs/src/streaming.rs @@ -57,7 +57,7 @@ fn post_listener_event<'a>( event: &StreamingQueryListenerEvent, started: bool, ) -> std::result::Result<(), ListenerDispatchError> { - let module = py.import("pyspark.sql.streaming.query")?; + let module = py.import("pyspark.sql.connect.streaming.query")?; let kwargs = PyDict::new(py); kwargs.set_item("started", started)?; let decoded = module.call_method( @@ -1153,7 +1153,7 @@ impl PyDataStreamReader { /// Python wrapper for Trigger. #[pyclass( name = "Trigger", - module = "pyspark.sql.streaming.readwriter", + module = "pyspark.sql.connect.streaming.readwriter", from_py_object )] #[derive(Clone)] @@ -1740,7 +1740,7 @@ impl PyDataStreamWriter { f: &Bound<'_, PyAny>, ) -> PyResult> { let func = py - .import("pyspark.sql.streaming.readwriter")? + .import("pyspark.sql.connect.streaming.readwriter")? .getattr("_construct_foreach_function")? .call1((f,))?; let serializers = py.import("pyspark.serializers")?; @@ -2125,7 +2125,7 @@ fn warn(py: Python<'_>, line: u32, message: String) -> PyResult<()> { /// which Apache Spark's tests read to remove them. #[pyclass( name = "StreamingQueryListenerBus", - module = "pyspark.sql.streaming.query", + module = "pyspark.sql.connect.streaming.query", frozen )] pub struct PyStreamingQueryListenerBus { @@ -2381,7 +2381,10 @@ impl PyStreamingQueryManager { /// Python wrapper for ListenerEventStream. /// Implements `__iter__` and `__next__` to yield (event_type: i32, event_json: String) tuples. -#[pyclass(name = "ListenerEventStream", module = "pyspark.sql.streaming.query")] +#[pyclass( + name = "ListenerEventStream", + module = "pyspark.sql.connect.streaming.query" +)] pub struct PyListenerEventStream { inner: Option, /// The session of the stream, whose waits a signal of the process ends. diff --git a/crates/pyspark-rs/src/udtf_analyze.rs b/crates/pyspark-rs/src/udtf_analyze.rs deleted file mode 100644 index 625193a9..00000000 --- a/crates/pyspark-rs/src/udtf_analyze.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! `pyspark.sql.udtf.SkipRestOfInputTableException`. The data classes of the `analyze` of a -//! polymorphic UDTF (`AnalyzeArgument`, `PartitioningColumn`, `OrderingColumn`, -//! `SelectedColumn` and `AnalyzeResult`) are PySpark's dataclasses, of the shim -//! `pyspark.sql.udtf`: a handler that captures them pickles with them, and the Python worker -//! of the server unpickles them as PySpark's. - -// The module argument sets `__module__`; use the reference path (upstream defines this -// in pyspark/sql/udtf.py) rather than the extension's own name. -pyo3::create_exception!( - pyspark.sql.udtf, - SkipRestOfInputTableException, - pyo3::exceptions::PyException -); diff --git a/dev/parity/known-differences.md b/dev/parity/known-differences.md index bfbbcba8..1f6b8d88 100644 --- a/dev/parity/known-differences.md +++ b/dev/parity/known-differences.md @@ -565,10 +565,11 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan which their copies and pickles keep as PySpark's do, but not their state, which PySpark's holds there (`vars(spark.streams)` is empty, where PySpark's has `_session` and `_sqlb`). - **Subclasses** (intentional; the state is native). PySpark's classes are plain classes that a - user subclasses. The drop-in's DataFrame, `Observation`, `SparkSession`, `Catalog`, - `DataSourceRegistration`, `DataFrameReader`, `DataStreamReader`, `SparkSession.Hook`, - `StorageLevel`, `PythonEvalType` and the data types are subclassed as PySpark's: a subclass - constructs of the arguments of its `__init__`, and its copies and pickles are of the subclass + user subclasses. The drop-in's DataFrame, `Observation`, the `SparkSession`, `Catalog`, + `DataSourceRegistration`, `DataFrameReader` and `DataStreamReader` of `pyspark.sql.connect`, + `SparkSession.Hook`, `StorageLevel`, `PythonEvalType` and the data types are subclassed as + PySpark's: a subclass constructs of the arguments of its `__init__`, and its copies and pickles + are of the subclass (but those of a session, which are the active session of its ID, as PySpark's), and PySpark's pickle of a reader, a catalog or a registration, which makes it of its class without arguments, loads in the drop-in. The `Window` of `pyspark.sql.window` and of @@ -694,15 +695,17 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan mypy and pyright type an application as with `pyspark-client` (see `docs/compatibility.md`), but: - **Imports of PySpark's modules** (intentional). A name that a module of PySpark imports, such as - `DataFrame` of `pyspark.sql.catalog`, is an attribute of the module, which mypy takes by default - with `pyspark-client`. The stubs declare it only when the drop-in's module has it at run time, so - that it is an error, as `from pyspark.sql.catalog import DataFrame` fails with the drop-in. + `DataFrame` of `pyspark.sql.connect.catalog`, is an attribute of the module, which mypy takes by + default with `pyspark-client`. The stubs declare it only when the drop-in's module has it at run + time, so that it is an error, as `from pyspark.sql.connect.catalog import DataFrame` fails with + the drop-in. pyright reports it as a private import with `pyspark-client` too. - **Names that only PySpark's annotations need** (intentional). `TYPE_ONLY` of `python/tests/test_type_stubs_offline.py` lists the names that the stubs declare and the drop-in does not have, with their reasons: the classic-only `DataFrame.rdd`, as type checkers of - `pyspark-client` see it, and the bases, metaclass, decorators and internal classes of PySpark's - classes that the drop-in implements natively. + `pyspark-client` see it, the type variables of the annotations of `pyspark.sql.types`, and the + base and the internal class of PySpark Connect's classes that the drop-in implements natively, + the `OptionUtils` of its readers and writers and its `StreamingQueryListenerBus`. - **`SparkSession.client`** (intentional; the drop-in's own class). The client has PySpark Connect's public API, of the class `pyspark.sql.connect.client.SparkConnectClient` rather than `pyspark.sql.connect.client.core`'s. diff --git a/docs/compatibility.md b/docs/compatibility.md index caf6041b..e639a74d 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -2082,16 +2082,16 @@ differences: re-exported when the drop-in's module has it too, so that `F.partitioning`, which pyright reports as a private import with `pyspark-client`, and `F.PandasUDFType`, which mypy with `--no-implicit-reexport` reports, are not reported. One that the drop-in's module does not - have, such as `pyspark.sql.catalog.DataFrame`, is an error, where mypy takes it with + have, such as `pyspark.sql.connect.catalog.DataFrame`, is an error, where mypy takes it with `pyspark-client` by default, as it fails at run time. So is the plumbing of PySpark's modules that the drop-in does not have, such as the py4j converters of `pyspark.sql.types`. - A few names that PySpark's annotations need are declared, which the drop-in does not have at run time: `DataFrame.rdd`, which PySpark defines when not `is_remote_only()`, so that type checkers - of `pyspark-client` see it too, and neither client has it; and the bases and the decorators of - PySpark's classes that the drop-in implements natively, such as `OptionUtils` of the readers and - writers, `PandasGroupedOpsMixin` of `GroupedData` and `dfapi` (`TYPE_ONLY` of the test lists - them with their reasons). `pyspark.sql.types` has `DataTypeSingleton`, the metaclass of the - singleton types of the stubs, at run time too, of which the native classes are not. The + of `pyspark-client` see it too, and neither client has it; and the classes of PySpark Connect + that the drop-in implements natively in another way, `OptionUtils`, the base of its readers and + writers, and the `StreamingQueryListenerBus` of its `StreamingQueryManager` (`TYPE_ONLY` of the + test lists them with their reasons). `pyspark.sql.types` has `DataTypeSingleton`, the metaclass + of the singleton types of the stubs, at run time too, of which the native classes are not. The `SparkContext`, `RDD` and the other names of classic PySpark that PySpark's `pyspark` module imports of `pyspark.core` when not `is_remote_only()`, which `pyspark-client` does not ship, the drop-in's module imports so too, in a block that does not run either: mypy takes them as @@ -2110,9 +2110,9 @@ differences: pyright does not see in PySpark's source, as its modules copy them there at run time: a hover over `col` of `pyspark.sql.connect.functions` or the `builder` of its `SparkSession` shows its documentation with the drop-in, and none with `pyspark-client`. -- The names that only the drop-in has, such as `Trigger` of `pyspark.sql.streaming.readwriter` and - `FrameBound` of `pyspark.sql.window`, are typed with the drop-in and are errors with - `pyspark-client`, as at run time. +- The names that only the drop-in has, such as `Trigger` of + `pyspark.sql.connect.streaming.readwriter` and `FrameBound` of `pyspark.sql.window`, are typed + with the drop-in and are errors with `pyspark-client`, as at run time. PySpark's `pyspark.sql.functions` is a package of the modules `builtin` and `partitioning`, where the drop-in's is a module, with the stubs of the three in the directory `pyspark/sql/functions/`, @@ -2767,11 +2767,12 @@ built then, as PySpark's sends the protos that `_build_merge_action` built for t value that a function reads, changed after the clause, is not that of the merge, also of a copy of the builder, and a window of no partitions of a clause warns once, at the clause. -The drop-in has one class for each UDF and UDTF, the Connect class of `pyspark.sql.connect.udf` -and `pyspark.sql.connect.udtf`, which the classic names `pyspark.sql.udf.UserDefinedFunction` and -`pyspark.sql.udtf.UserDefinedTableFunction` also name. So `isinstance` of a UDTF that `udtf` builds -is True against either name, where PySpark 4.3.0 gives False for the classic name, whose class -needs a SparkContext. `udf` gives a function in both clients, which is of neither class. +A UDF and a UDTF are of PySpark Connect's classes, `UserDefinedFunction` of +`pyspark.sql.connect.udf` and `UserDefinedTableFunction` of `pyspark.sql.connect.udtf`, as in +PySpark. The classic `pyspark.sql.udf` and `pyspark.sql.udtf` are PySpark's modules, of Apache +Spark as they are, whose classes, which `pyspark.sql.functions` names too, need a SparkContext: so +`isinstance` of a UDTF that `udtf` builds against the classic `UserDefinedTableFunction` is False, +as in PySpark. `udf` gives a function in both clients, which is of neither class. So it is for the objects of a session: each is of PySpark Connect's class, of its module of `pyspark.sql.connect`, as `repr`, `type(x).__module__` and pickle show it. These are the @@ -2807,26 +2808,36 @@ without a URL or a channel builder, returns the active session from `getOrCreate `spark.sql.catalogImplementation`. The classic classes of `pyspark.sql.conf`, `pyspark.sql.tvf`, `pyspark.sql.datasource`, -`pyspark.sql.merge` and `pyspark.sql.observation` are PySpark's, of those modules of Apache Spark -as they are, so `isinstance` of a session's `conf`, `tvf`, `dataSource` and `mergeInto` against -them is False, as in PySpark. `Observation(...)` makes an Observation of -`pyspark.sql.connect.observation`, which is not an instance of `pyspark.sql.Observation`, as -PySpark's `__new__` makes one. `pyspark.sql.connect.window.Window` is a subclass of -`pyspark.sql.window.Window`, as in PySpark. For the other objects of a session, the classic name -names PySpark Connect's class, so `isinstance` against it is True, where PySpark gives False: -`Catalog`, `DataFrameReader`, `DataFrameWriter`, `DataFrameWriterV2`, `DataStreamReader`, -`DataStreamWriter`, `StreamingQueryManager`, `StreamingQuery`, `UDFRegistration`, -`UDTFRegistration`, `GroupedData` and `PandasCogroupedOps`, and the UDTFs above. The classic -`DataFrameNaFunctions`, `DataFrameStatFunctions`, `WindowSpec` and `TableArg` name PySpark -Connect's classes too, which in PySpark are their subclasses: `isinstance` is True in both, and +`pyspark.sql.merge`, `pyspark.sql.observation`, `pyspark.sql.catalog`, `pyspark.sql.readwriter`, +`pyspark.sql.udf`, `pyspark.sql.udtf`, `pyspark.sql.group`, `pyspark.sql.pandas.group_ops`, +`pyspark.sql.streaming.readwriter` and `pyspark.sql.streaming.query` are PySpark's, of those +modules of Apache Spark as they are, which `pyspark.sql` and `pyspark.sql.streaming` name as +PySpark's do. So `isinstance` against them is False, as in PySpark, of a session's `conf`, `tvf`, +`dataSource`, `catalog`, `read`, `readStream`, `streams`, `udf` and `udtf`, of a DataFrame's +`mergeInto`, `write`, `writeTo`, `writeStream`, `groupBy` and `cogroup`, of a StreamingQuery and of +the UDTFs above. Their constructors and methods are those of Spark Classic, as PySpark's: +`pyspark.sql.DataFrameReader(spark)` and `pyspark.sql.Catalog(spark)` raise the +`JVM_ATTRIBUTE_NOT_SUPPORTED` of `_jsparkSession`, and so does the `register` of a +`pyspark.sql.udf.UDFRegistration(spark)`, as in PySpark, where the reader of `spark.read` and +`pyspark.sql.connect.readwriter.DataFrameReader(spark)` are PySpark Connect's. +`Observation(...)` makes an Observation of `pyspark.sql.connect.observation`, which is not an +instance of `pyspark.sql.Observation`, as PySpark's `__new__` makes one. +`pyspark.sql.connect.window.Window` is a subclass of `pyspark.sql.window.Window`, as in PySpark. +The classic `DataFrameNaFunctions`, `DataFrameStatFunctions`, `WindowSpec` and `TableArg` name +PySpark Connect's classes, which in PySpark are their subclasses: `isinstance` is True in both, and the MRO of such an object is its native class and `object`. The concrete DataFrame and Column are not subclasses of the public `pyspark.sql.DataFrame` and `pyspark.sql.Column`: `isinstance` holds by registration, so the MRO of a DataFrame is its native class and `object`, and -`pyspark.sql.Column` names the concrete class. +`pyspark.sql.Column` names the concrete class, where PySpark's is the abstract class of the +concrete one of `pyspark.sql.connect.column`. The readers and writers of PySpark Connect are +subclasses of the `OptionUtils` of `pyspark.sql.connect.readwriter`, a class of private methods +only, of which the drop-in's native readers and writers are not: their MRO is their class and +`object`, and `pyspark.sql.connect.readwriter` has no `OptionUtils`. PySpark's classes are plain Python classes, which a user subclasses. So are these of the drop-in: -the DataFrame, `Observation`, `SparkSession`, `Catalog`, `DataSourceRegistration`, -`DataFrameReader` and `DataStreamReader`, which construct of their PySpark arguments +the DataFrame, `Observation`, and the `SparkSession`, `Catalog`, `DataSourceRegistration`, +`DataFrameReader` and `DataStreamReader` of `pyspark.sql.connect`, which construct of their PySpark +arguments (`SparkSession(connection=...)`, `Catalog(spark)`, `DataFrameReader(spark)`), also a subclass of an `__init__` of its own, and whose setters, copies and pickles are of the subclass (but those of a session, which are the active session of its ID, as PySpark's); diff --git a/pyproject.toml b/pyproject.toml index 7358fc29..1741deb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,7 +92,6 @@ extend-exclude = [ "python/pyspark/ml/connect/tuning.pyi", "python/pyspark/ml/stat.pyi", "python/pyspark/sql/avro/functions.pyi", - "python/pyspark/sql/catalog.pyi", "python/pyspark/sql/column.pyi", "python/pyspark/sql/connect/avro/functions.pyi", "python/pyspark/sql/connect/catalog.pyi", @@ -121,16 +120,9 @@ extend-exclude = [ "python/pyspark/sql/functions/__init__.pyi", "python/pyspark/sql/functions/builtin.pyi", "python/pyspark/sql/functions/partitioning.pyi", - "python/pyspark/sql/group.pyi", - "python/pyspark/sql/pandas/group_ops.pyi", "python/pyspark/sql/protobuf/functions.pyi", - "python/pyspark/sql/readwriter.pyi", - "python/pyspark/sql/streaming/query.pyi", - "python/pyspark/sql/streaming/readwriter.pyi", "python/pyspark/sql/table_arg.pyi", "python/pyspark/sql/types.pyi", - "python/pyspark/sql/udf.pyi", - "python/pyspark/sql/udtf.pyi", "python/pyspark/sql/window.pyi", "python/pyspark/storagelevel.pyi", "python/pyspark/util.pyi", @@ -209,6 +201,14 @@ extend-exclude = [ "python/pyspark/sql/merge.py", "python/pyspark/sql/observation.py", "python/pyspark/sql/context.py", + "python/pyspark/sql/catalog.py", + "python/pyspark/sql/readwriter.py", + "python/pyspark/sql/udf.py", + "python/pyspark/sql/udtf.py", + "python/pyspark/sql/group.py", + "python/pyspark/sql/pandas/group_ops.py", + "python/pyspark/sql/streaming/readwriter.py", + "python/pyspark/sql/streaming/query.py", "python/pyspark/_typing.pyi", "python/pyspark/sql/_typing.pyi", "python/pyspark/accumulators.py", @@ -366,7 +366,7 @@ ignore = ["E501"] # line length handled by the formatter, not enforced hard "python/pyspark/sql/functions.py" = ["F822", "F401", "UP035", "E402"] # Re-export surfaces: the names are imported so that they resolve under PySpark's import # paths, not because this module uses them (F401), and from typing as there (UP035). -"python/pyspark/sql/{types,streaming/query}.py" = ["F401", "UP035"] +"python/pyspark/sql/types.py" = ["F401", "UP035"] # Fork-adapted copies of Apache Spark modules (upstream code with the Rust-backed parts # swapped in). Only the correctness rules (F, E, W) apply; the import order, the typing and # string-formatting idioms and the bugbear style stay as upstream writes them, so a diff @@ -375,8 +375,10 @@ ignore = ["E501"] # line length handled by the formatter, not enforced hard "python/pyspark/errors/exceptions/connect.py" = ["I", "UP", "B"] "python/pyspark/mllib/stat/__init__.py" = ["I", "UP", "B"] "python/pyspark/pipelines/{api,spark_connect_graph_element_registry,spark_connect_pipeline}.py" = ["I", "UP", "B", "F401"] -"python/pyspark/sql/{__init__,tvf_argument,types,udf,udtf}.py" = ["I", "UP", "B"] -"python/pyspark/sql/connect/{context,datasource,expressions,sql_formatter,types,utils}.py" = ["I", "UP", "B"] +"python/pyspark/sql/{__init__,tvf_argument,types}.py" = ["I", "UP", "B"] +# connect/udf.py and connect/udtf.py adapt PySpark's classic udf.py and udtf.py, whose UDF and UDTF +# objects they create as PySpark Connect's modules do. +"python/pyspark/sql/connect/{context,datasource,expressions,sql_formatter,types,udf,udtf,utils}.py" = ["I", "UP", "B"] "python/pyspark/sql/streaming/{stateful_processor_api_client,transform_with_state_driver_worker}.py" = ["I", "UP", "B"] "python/pyspark/streaming/{context,dstream,util}.py" = ["I", "UP", "B"] "python/pyspark/testing/{connectutils,sqlutils}.py" = ["I", "UP", "B"] diff --git a/python/.coveragerc b/python/.coveragerc index 74a220ad..8a7dac23 100644 --- a/python/.coveragerc +++ b/python/.coveragerc @@ -34,6 +34,9 @@ omit = */pyspark/errors/* */pyspark/messages/* */pyspark/sql/streaming/* + # The drop-in's streaming code of the modules of PySpark Connect (the listener bus and the + # functions of foreach), as that of pyspark.sql.streaming: the subsystem has its own tests. + */pyspark/sql/connect/streaming/* */pyspark/sql/worker/* */pyspark/sql/plot/* */pyspark/sql/pandas/* @@ -52,6 +55,14 @@ omit = */pyspark/sql/merge.py */pyspark/sql/observation.py */pyspark/sql/context.py + */pyspark/sql/catalog.py + */pyspark/sql/readwriter.py + */pyspark/sql/udf.py + */pyspark/sql/udtf.py + */pyspark/sql/group.py + */pyspark/sql/pandas/group_ops.py + */pyspark/sql/streaming/readwriter.py + */pyspark/sql/streaming/query.py */pyspark/serializers.py */pyspark/loose_version.py */pyspark/_globals.py diff --git a/python/pyspark/sql/catalog.py b/python/pyspark/sql/catalog.py index 8d4ed9dd..d698c458 100644 --- a/python/pyspark/sql/catalog.py +++ b/python/pyspark/sql/catalog.py @@ -14,18 +14,20 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""Catalog + its metadata result classes, mirroring ``pyspark.sql.catalog``. -The Catalog is implemented natively in the Rust core and re-exported here. Its methods return -the NamedTuples of PySpark's ``pyspark.sql.catalog`` below (Database, Table, Column, Function, -CatalogMetadata and TablePartition), which are tuples, as PySpark's are.""" +import sys +import warnings +from typing import Any, Callable, Dict, NamedTuple, List, Optional, TYPE_CHECKING -from typing import NamedTuple, Optional +from pyspark.errors import PySparkTypeError +from pyspark.storagelevel import StorageLevel +from pyspark.sql.dataframe import DataFrame +from pyspark.sql.session import SparkSession +from pyspark.sql.types import StructType -from pyspark._pyspark import Catalog - -# Of PySpark's pyspark.sql.catalog, whose examples use it (Catalog.cacheTable). -from pyspark.storagelevel import StorageLevel # noqa: F401 +if TYPE_CHECKING: + from pyspark.sql._typing import UserDefinedFunctionLike + from pyspark.sql._typing import DataTypeOrString class CatalogMetadata(NamedTuple): @@ -43,7 +45,7 @@ class Database(NamedTuple): class Table(NamedTuple): name: str catalog: Optional[str] - namespace: Optional[list[str]] + namespace: Optional[List[str]] description: Optional[str] tableType: str isTemporary: bool @@ -69,7 +71,7 @@ class Column(NamedTuple): class Function(NamedTuple): name: str catalog: Optional[str] - namespace: Optional[list[str]] + namespace: Optional[List[str]] description: Optional[str] className: str isTemporary: bool @@ -79,12 +81,1517 @@ class TablePartition(NamedTuple): partition: str -__all__ = [ - "Catalog", - "CatalogMetadata", - "Database", - "Table", - "Column", - "Function", - "TablePartition", -] +class Catalog: + """Spark SQL catalog interface. + + Use :attr:`~pyspark.sql.SparkSession.catalog` on an active session. This class is a thin + wrapper around ``org.apache.spark.sql.catalog.Catalog``. + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + """ + + def __init__(self, sparkSession: SparkSession) -> None: + """Create a new Catalog that wraps the underlying JVM object.""" + self._sparkSession = sparkSession + self._jsparkSession = sparkSession._jsparkSession + self._sc = sparkSession._sc + self._jcatalog = sparkSession._jsparkSession.catalog() + + def currentCatalog(self) -> str: + """Returns the current catalog in this session. + + .. versionadded:: 3.4.0 + + Returns + ------- + str + The current catalog name. + + Examples + -------- + >>> spark.catalog.currentCatalog() + 'spark_catalog' + """ + return self._jcatalog.currentCatalog() + + def setCurrentCatalog(self, catalogName: str) -> None: + """Sets the current catalog in this session. + + .. versionadded:: 3.4.0 + + Parameters + ---------- + catalogName : str + Name of the catalog to set. + + Examples + -------- + >>> spark.catalog.setCurrentCatalog("spark_catalog") + """ + return self._jcatalog.setCurrentCatalog(catalogName) + + def listCatalogs(self, pattern: Optional[str] = None) -> List[CatalogMetadata]: + """Returns a list of catalogs available in this session. + + With ``pattern``, returns only catalogs whose name matches that pattern. + + .. versionadded:: 3.4.0 + + Parameters + ---------- + pattern : str, optional + Pattern that catalog names must match. + + .. versionadded:: 3.5.0 + + Returns + ------- + list + A list of :class:`CatalogMetadata`. + + Examples + -------- + >>> spark.catalog.listCatalogs() + [CatalogMetadata(name='spark_catalog', description=None)] + + >>> spark.catalog.listCatalogs("spark*") + [CatalogMetadata(name='spark_catalog', description=None)] + + >>> spark.catalog.listCatalogs("hive*") + [] + """ + if pattern is None: + iter = self._jcatalog.listCatalogs().toLocalIterator() + else: + iter = self._jcatalog.listCatalogs(pattern).toLocalIterator() + catalogs = [] + while iter.hasNext(): + jcatalog = iter.next() + catalogs.append( + CatalogMetadata(name=jcatalog.name(), description=jcatalog.description()) + ) + return catalogs + + def dropTable(self, tableName: str, ifExists: bool = False, purge: bool = False) -> None: + """Drops a persistent table. + + This does not remove temp views; use :meth:`Catalog.dropTempView`. + + .. versionadded:: 4.2.0 + + Parameters + ---------- + tableName : str + Name of the table to drop. May be qualified with catalog and database (namespace). + ifExists : bool, optional + If ``True``, do not fail when the table does not exist. + purge : bool, optional + If ``True``, skip moving data to a trash directory when the catalog supports it. + + Examples + -------- + >>> _ = spark.sql("DROP TABLE IF EXISTS tbl_drop_doc") + >>> _ = spark.sql("CREATE TABLE tbl_drop_doc (id INT) USING parquet") + >>> spark.catalog.dropTable("tbl_drop_doc") + >>> spark.catalog.tableExists("tbl_drop_doc") + False + """ + self._jcatalog.dropTable(tableName, ifExists, purge) + + def dropView(self, viewName: str, ifExists: bool = False) -> None: + """Drops a persistent view. + + .. versionadded:: 4.2.0 + + Parameters + ---------- + viewName : str + Name of the view to drop. May be qualified with catalog and database (namespace). + ifExists : bool, optional + If ``True``, do not fail when the view does not exist. + + Examples + -------- + >>> _ = spark.sql("DROP VIEW IF EXISTS view_drop_doc") + >>> _ = spark.sql("CREATE VIEW view_drop_doc AS SELECT 1 AS c") + >>> spark.catalog.dropView("view_drop_doc") + >>> spark.catalog.tableExists("view_drop_doc") + False + """ + self._jcatalog.dropView(viewName, ifExists) + + def createDatabase( + self, dbName: str, ifNotExists: bool = False, properties: Optional[Dict[str, str]] = None + ) -> None: + """Creates a namespace (database/schema). + + ``dbName`` may be a multi-part identifier (for example ``catalog.schema``). Optional + ``properties`` follow ``CREATE NAMESPACE`` (for example comment or location keys). + + .. versionadded:: 4.2.0 + + Parameters + ---------- + dbName : str + Name of the namespace to create. + ifNotExists : bool, optional + If ``True``, do not fail when the namespace already exists. + properties : dict, optional + Map of namespace property keys to string values. + + Examples + -------- + >>> spark.catalog.dropDatabase("db_create_doc", ifExists=True, cascade=True) + >>> spark.catalog.createDatabase("db_create_doc") + >>> spark.catalog.databaseExists("db_create_doc") + True + >>> spark.catalog.dropDatabase("db_create_doc", cascade=True) + >>> spark.catalog.databaseExists("db_create_doc") + False + """ + sc = self._sc + assert sc is not None + assert sc._gateway is not None + ju = sc._gateway.jvm.java.util + m = ju.HashMap() + if properties: + for k, v in properties.items(): + m.put(k, v) + self._jcatalog.createDatabase(dbName, ifNotExists, m) + + def dropDatabase(self, dbName: str, ifExists: bool = False, cascade: bool = False) -> None: + """Drops a namespace. + + .. versionadded:: 4.2.0 + + Parameters + ---------- + dbName : str + Name of the namespace to drop. May be qualified with catalog name. + ifExists : bool, optional + If ``True``, do not fail when the namespace does not exist. + cascade : bool, optional + If ``True``, also drop tables and functions in the namespace. + + Examples + -------- + >>> spark.catalog.dropDatabase("db_drop_doc", ifExists=True, cascade=True) + >>> spark.catalog.createDatabase("db_drop_doc") + >>> spark.catalog.dropDatabase("db_drop_doc", cascade=True) + >>> spark.catalog.databaseExists("db_drop_doc") + False + """ + self._jcatalog.dropDatabase(dbName, ifExists, cascade) + + def listPartitions(self, tableName: str) -> List[TablePartition]: + """Lists partition value strings for a table (same as ``SHOW PARTITIONS``). + + .. versionadded:: 4.2.0 + + Parameters + ---------- + tableName : str + Name of the partitioned table. May be qualified with catalog and database (namespace). + + Returns + ------- + list + A list of :class:`TablePartition` (each ``partition`` field is a spec string). + + Examples + -------- + >>> _ = spark.sql("DROP TABLE IF EXISTS tbl_part_doc") + >>> _ = spark.sql( + ... "CREATE TABLE tbl_part_doc (id INT, p INT) USING parquet PARTITIONED BY (p)" + ... ) + >>> _ = spark.sql("INSERT INTO tbl_part_doc PARTITION (p = 1) SELECT 1") + >>> parts = [x.partition for x in spark.catalog.listPartitions("tbl_part_doc")] + >>> any("p=1" in s for s in parts) + True + >>> _ = spark.sql("DROP TABLE tbl_part_doc") + """ + iter = self._jcatalog.listPartitions(tableName).toLocalIterator() + out: List[TablePartition] = [] + while iter.hasNext(): + j = iter.next() + out.append(TablePartition(partition=j.partition())) + return out + + def listViews(self, dbName: Optional[str] = None, pattern: Optional[str] = None) -> List[Table]: + """Lists views in a namespace. + + With no arguments, lists views in the current namespace. With ``dbName`` only, lists + views in that namespace (may be catalog-qualified). With ``pattern``, filters view names + with a SQL ``LIKE`` string; if ``pattern`` is given without ``dbName``, the current + database is used as the namespace. + + .. versionadded:: 4.2.0 + + Parameters + ---------- + dbName : str, optional + Namespace to list views from. May be qualified with catalog name. + pattern : str, optional + SQL ``LIKE`` pattern for view names. + + Returns + ------- + list + A list of :class:`Table` (same row shape as :meth:`Catalog.listTables`). + + Notes + ----- + Raises :class:`AnalysisException` if ``dbName`` names a namespace that does not exist. + + Examples + -------- + >>> _ = spark.sql("DROP VIEW IF EXISTS view_list_doc") + >>> _ = spark.sql("CREATE VIEW view_list_doc AS SELECT 1 AS c") + >>> "view_list_doc" in [v.name for v in spark.catalog.listViews()] + True + >>> "view_list_doc" in [v.name for v in spark.catalog.listViews(pattern="view_list_*")] + True + >>> _ = spark.sql("DROP VIEW view_list_doc") + """ + if pattern is not None and dbName is None: + dbName = self.currentDatabase() + if dbName is None: + iter = self._jcatalog.listViews().toLocalIterator() + elif pattern is None: + iter = self._jcatalog.listViews(dbName).toLocalIterator() + else: + iter = self._jcatalog.listViews(dbName, pattern).toLocalIterator() + views = [] + while iter.hasNext(): + jtable = iter.next() + jnamespace = jtable.namespace() + namespace = list(jnamespace) if jnamespace is not None else None + views.append( + Table( + name=jtable.name(), + catalog=jtable.catalog(), + namespace=namespace, + description=jtable.description(), + tableType=jtable.tableType(), + isTemporary=jtable.isTemporary(), + ) + ) + return views + + def getTableProperties(self, tableName: str) -> Dict[str, str]: + """Returns all table properties as a dict (same as ``SHOW TBLPROPERTIES``). + + .. versionadded:: 4.2.0 + + Parameters + ---------- + tableName : str + Table or view name. May be qualified with catalog and database (namespace). + + Returns + ------- + dict + Map of property key to value. + + Examples + -------- + >>> _ = spark.sql("DROP TABLE IF EXISTS tbl_props_doc") + >>> _ = spark.sql( + ... "CREATE TABLE tbl_props_doc (id INT) USING parquet " + ... "TBLPROPERTIES ('doc_prop_key' = 'doc_prop_val')" + ... ) + >>> spark.catalog.getTableProperties("tbl_props_doc")["doc_prop_key"] + 'doc_prop_val' + >>> _ = spark.sql("DROP TABLE tbl_props_doc") + """ + jm = self._jcatalog.getTableProperties(tableName) + return {k: jm.get(k) for k in jm.keySet()} + + def getCreateTableString(self, tableName: str, asSerde: bool = False) -> str: + """Returns the ``SHOW CREATE TABLE`` DDL string for a relation. + + .. versionadded:: 4.2.0 + + Parameters + ---------- + tableName : str + Table or view name. May be qualified with catalog and database (namespace). + asSerde : bool, optional + If ``True``, request Hive serde DDL when applicable. + + Returns + ------- + str + DDL string from ``SHOW CREATE TABLE``. + + Examples + -------- + >>> _ = spark.sql("DROP TABLE IF EXISTS tbl_ddl_doc") + >>> _ = spark.sql("CREATE TABLE tbl_ddl_doc (id INT) USING parquet") + >>> "CREATE" in spark.catalog.getCreateTableString("tbl_ddl_doc").upper() + True + >>> _ = spark.sql("DROP TABLE tbl_ddl_doc") + """ + return self._jcatalog.getCreateTableString(tableName, asSerde) + + def truncateTable(self, tableName: str) -> None: + """Truncates a table (removes all data from the table; not supported for views). + + .. versionadded:: 4.2.0 + + Parameters + ---------- + tableName : str + Name of the table to truncate. May be qualified with catalog and database (namespace). + + Examples + -------- + >>> _ = spark.sql("DROP TABLE IF EXISTS tbl_tr_doc") + >>> _ = spark.sql("CREATE TABLE tbl_tr_doc (id INT) USING csv") + >>> _ = spark.sql("INSERT INTO tbl_tr_doc VALUES (1), (2)") + >>> spark.table("tbl_tr_doc").count() + 2 + >>> spark.catalog.truncateTable("tbl_tr_doc") + >>> spark.table("tbl_tr_doc").count() + 0 + >>> _ = spark.sql("DROP TABLE tbl_tr_doc") + """ + self._jcatalog.truncateTable(tableName) + + def analyzeTable(self, tableName: str, noScan: bool = False) -> None: + """Computes table statistics (same as SQL ``ANALYZE TABLE COMPUTE STATISTICS``). + + .. versionadded:: 4.2.0 + + Parameters + ---------- + tableName : str + Table or view name. May be qualified with catalog and database (namespace). + noScan : bool, optional + If ``True``, use ``NOSCAN`` mode (reuse existing column statistics where possible). + + Examples + -------- + >>> _ = spark.sql("DROP TABLE IF EXISTS tbl_an_doc") + >>> _ = spark.sql("CREATE TABLE tbl_an_doc (id INT) USING csv") + >>> _ = spark.sql("INSERT INTO tbl_an_doc VALUES (1)") + >>> spark.catalog.analyzeTable("tbl_an_doc") + >>> spark.catalog.analyzeTable("tbl_an_doc", noScan=True) + >>> _ = spark.sql("DROP TABLE tbl_an_doc") + """ + self._jcatalog.analyzeTable(tableName, noScan) + + def currentDatabase(self) -> str: + """ + Returns the current database (namespace) in this session. + + .. versionadded:: 2.0.0 + + Returns + ------- + str + The current database (namespace) name. + + Examples + -------- + >>> spark.catalog.currentDatabase() + 'default' + """ + return self._jcatalog.currentDatabase() + + def setCurrentDatabase(self, dbName: str) -> None: + """ + Sets the current database (namespace) in this session. + + .. versionadded:: 2.0.0 + + Parameters + ---------- + dbName : str + Name of the database (namespace) to set. + + Examples + -------- + >>> spark.catalog.setCurrentDatabase("default") + """ + return self._jcatalog.setCurrentDatabase(dbName) + + def listDatabases(self, pattern: Optional[str] = None) -> List[Database]: + """ + Returns a list of databases (namespaces) available within the current catalog. + + With ``pattern``, returns only databases whose name matches that pattern. + + .. versionadded:: 2.0.0 + + Parameters + ---------- + pattern : str, optional + Pattern that database names must match. + + .. versionadded:: 3.5.0 + + Returns + ------- + list + A list of :class:`Database`. + + Examples + -------- + >>> spark.catalog.listDatabases() + [Database(name='default', catalog='spark_catalog', description='default database', ... + + >>> spark.catalog.listDatabases("def*") + [Database(name='default', catalog='spark_catalog', description='default database', ... + + >>> spark.catalog.listDatabases("def2*") + [] + """ + if pattern is None: + iter = self._jcatalog.listDatabases().toLocalIterator() + else: + iter = self._jcatalog.listDatabases(pattern).toLocalIterator() + databases = [] + while iter.hasNext(): + jdb = iter.next() + databases.append( + Database( + name=jdb.name(), + catalog=jdb.catalog(), + description=jdb.description(), + locationUri=jdb.locationUri(), + ) + ) + return databases + + def getDatabase(self, dbName: str) -> Database: + """Get the database with the specified name. + This throws an :class:`AnalysisException` when the database cannot be found. + + .. versionadded:: 3.4.0 + + Parameters + ---------- + dbName : str + name of the database to get. + + Returns + ------- + :class:`Database` + The database found by the name. + + Examples + -------- + >>> spark.catalog.getDatabase("default") + Database(name='default', catalog='spark_catalog', description='default database', ... + + Using the fully qualified name with the catalog name. + + >>> spark.catalog.getDatabase("spark_catalog.default") + Database(name='default', catalog='spark_catalog', description='default database', ... + """ + jdb = self._jcatalog.getDatabase(dbName) + return Database( + name=jdb.name(), + catalog=jdb.catalog(), + description=jdb.description(), + locationUri=jdb.locationUri(), + ) + + def databaseExists(self, dbName: str) -> bool: + """Check if the database with the specified name exists. + + .. versionadded:: 3.3.0 + + Parameters + ---------- + dbName : str + name of the database to check existence + + .. versionchanged:: 3.4.0 + Allow ``dbName`` to be qualified with catalog name. + + Returns + ------- + bool + Indicating whether the database exists + + Examples + -------- + Check if 'test_new_database' database exists + + >>> spark.catalog.databaseExists("test_new_database") + False + >>> _ = spark.sql("CREATE DATABASE test_new_database") + >>> spark.catalog.databaseExists("test_new_database") + True + + Using the fully qualified name with the catalog name. + + >>> spark.catalog.databaseExists("spark_catalog.test_new_database") + True + >>> _ = spark.sql("DROP DATABASE test_new_database") + """ + return self._jcatalog.databaseExists(dbName) + + def listTables( + self, dbName: Optional[str] = None, pattern: Optional[str] = None + ) -> List[Table]: + """Returns a list of tables/views in the current database (namespace), or in the database + given by ``dbName`` when provided (the name may be qualified with catalog). + + With ``pattern``, returns only tables and views whose name matches the pattern. This + includes all temporary views. + + .. versionadded:: 2.0.0 + + Parameters + ---------- + dbName : str, optional + Database (namespace) to list tables from. If omitted, the current database is used. + + .. versionchanged:: 3.4.0 + Allow ``dbName`` to be qualified with catalog name. + + pattern : str, optional + Pattern that table names must match. + + .. versionadded:: 3.5.0 + + Returns + ------- + list + A list of :class:`Table`. + + Examples + -------- + >>> spark.range(1).createTempView("test_view") + >>> spark.catalog.listTables() + [Table(name='test_view', catalog=None, namespace=[], description=None, ... + + >>> spark.catalog.listTables(pattern="test*") + [Table(name='test_view', catalog=None, namespace=[], description=None, ... + + >>> spark.catalog.listTables(pattern="table*") + [] + + >>> _ = spark.catalog.dropTempView("test_view") + >>> spark.catalog.listTables() + [] + """ + if dbName is None: + dbName = self.currentDatabase() + + if pattern is None: + iter = self._jcatalog.listTables(dbName).toLocalIterator() + else: + iter = self._jcatalog.listTables(dbName, pattern).toLocalIterator() + tables = [] + while iter.hasNext(): + jtable = iter.next() + + jnamespace = jtable.namespace() + namespace = list(jnamespace) if jnamespace is not None else None + + tables.append( + Table( + name=jtable.name(), + catalog=jtable.catalog(), + namespace=namespace, + description=jtable.description(), + tableType=jtable.tableType(), + isTemporary=jtable.isTemporary(), + ) + ) + return tables + + def getTable(self, tableName: str) -> Table: + """Get the table or view with the specified name. This table can be a temporary view or a + table/view. This throws an :class:`AnalysisException` when no Table can be found. + + .. versionadded:: 3.4.0 + + Parameters + ---------- + tableName : str + name of the table to get. + + .. versionchanged:: 3.4.0 + Allow `tableName` to be qualified with catalog name. + + Returns + ------- + :class:`Table` + The table found by the name. + + Examples + -------- + >>> _ = spark.sql("DROP TABLE IF EXISTS tbl1") + >>> _ = spark.sql("CREATE TABLE tbl1 (name STRING, age INT) USING parquet") + >>> spark.catalog.getTable("tbl1") + Table(name='tbl1', catalog='spark_catalog', namespace=['default'], ... + + Using the fully qualified name with the catalog name. + + >>> spark.catalog.getTable("default.tbl1") + Table(name='tbl1', catalog='spark_catalog', namespace=['default'], ... + >>> spark.catalog.getTable("spark_catalog.default.tbl1") + Table(name='tbl1', catalog='spark_catalog', namespace=['default'], ... + >>> _ = spark.sql("DROP TABLE tbl1") + + Throw an analysis exception when the table does not exist. + + >>> spark.catalog.getTable("tbl1") + Traceback (most recent call last): + ... + AnalysisException: ... + """ + jtable = self._jcatalog.getTable(tableName) + jnamespace = jtable.namespace() + namespace = list(jnamespace) if jnamespace is not None else None + return Table( + name=jtable.name(), + catalog=jtable.catalog(), + namespace=namespace, + description=jtable.description(), + tableType=jtable.tableType(), + isTemporary=jtable.isTemporary(), + ) + + def listFunctions( + self, dbName: Optional[str] = None, pattern: Optional[str] = None + ) -> List[Function]: + """ + Returns a list of functions registered in the current database (namespace), or in the + database given by ``dbName`` when provided (the name may be qualified with catalog). + + With ``pattern``, returns only functions whose name matches the pattern. This includes all + built-in and temporary functions. + + .. versionadded:: 3.4.0 + + Parameters + ---------- + dbName : str, optional + Database (namespace) to list functions from. If omitted, the current database is used. + May be qualified with catalog name. + pattern : str, optional + Pattern that function names must match. + + .. versionadded:: 3.5.0 + + Returns + ------- + list + A list of :class:`Function`. + + Examples + -------- + >>> spark.catalog.listFunctions() + [Function(name=... + + >>> spark.catalog.listFunctions(pattern="to_*") + [Function(name=... + + >>> spark.catalog.listFunctions(pattern="*not_existing_func*") + [] + """ + if dbName is None: + dbName = self.currentDatabase() + if pattern is None: + iter = self._jcatalog.listFunctions(dbName).toLocalIterator() + else: + iter = self._jcatalog.listFunctions(dbName, pattern).toLocalIterator() + functions = [] + while iter.hasNext(): + jfunction = iter.next() + jnamespace = jfunction.namespace() + namespace = list(jnamespace) if jnamespace is not None else None + + functions.append( + Function( + name=jfunction.name(), + catalog=jfunction.catalog(), + namespace=namespace, + description=jfunction.description(), + className=jfunction.className(), + isTemporary=jfunction.isTemporary(), + ) + ) + return functions + + def functionExists(self, functionName: str, dbName: Optional[str] = None) -> bool: + """Check if the function with the specified name exists. + This can either be a temporary function or a function. + + .. versionadded:: 3.3.0 + + Parameters + ---------- + functionName : str + name of the function to check existence + + .. versionchanged:: 3.4.0 + Allow ``functionName`` to be qualified with catalog name + + dbName : str, optional + name of the database to check function existence in. + + Returns + ------- + bool + Indicating whether the function exists + + Notes + ----- + If no database is specified, the current database and catalog + are used. This API includes all temporary functions. + + Examples + -------- + >>> spark.catalog.functionExists("count") + True + + Using the fully qualified name for function name. + + >>> spark.catalog.functionExists("default.unexisting_function") + False + >>> spark.catalog.functionExists("spark_catalog.default.unexisting_function") + False + """ + if dbName is None: + return self._jcatalog.functionExists(functionName) + else: + warnings.warn( + "`dbName` has been deprecated since Spark 3.4 and might be removed in " + "a future version. Use functionExists(`dbName.tableName`) instead.", + FutureWarning, + ) + return self._jcatalog.functionExists(dbName, functionName) + + def getFunction(self, functionName: str) -> Function: + """Get the function with the specified name. This function can be a temporary function or a + function. This throws an :class:`AnalysisException` when the function cannot be found. + + .. versionadded:: 3.4.0 + + Parameters + ---------- + functionName : str + name of the function to check existence. + + Returns + ------- + :class:`Function` + The function found by the name. + + Examples + -------- + >>> _ = spark.sql( + ... "CREATE FUNCTION my_func1 AS 'test.org.apache.spark.sql.MyDoubleAvg'") + >>> spark.catalog.getFunction("my_func1") + Function(name='my_func1', catalog='spark_catalog', namespace=['default'], ... + + Using the fully qualified name for function name. + + >>> spark.catalog.getFunction("default.my_func1") + Function(name='my_func1', catalog='spark_catalog', namespace=['default'], ... + >>> spark.catalog.getFunction("spark_catalog.default.my_func1") + Function(name='my_func1', catalog='spark_catalog', namespace=['default'], ... + + Throw an analysis exception when the function does not exists. + + >>> spark.catalog.getFunction("my_func2") + Traceback (most recent call last): + ... + AnalysisException: ... + """ + jfunction = self._jcatalog.getFunction(functionName) + jnamespace = jfunction.namespace() + namespace = list(jnamespace) if jnamespace is not None else None + return Function( + name=jfunction.name(), + catalog=jfunction.catalog(), + namespace=namespace, + description=jfunction.description(), + className=jfunction.className(), + isTemporary=jfunction.isTemporary(), + ) + + def listColumns(self, tableName: str, dbName: Optional[str] = None) -> List[Column]: + """Returns a list of columns for the given table/view in the specified database. + + .. versionadded:: 2.0.0 + + Parameters + ---------- + tableName : str + name of the table to list columns. + + .. versionchanged:: 3.4.0 + Allow ``tableName`` to be qualified with catalog name when ``dbName`` is None. + + dbName : str, optional + name of the database to find the table to list columns. + + Returns + ------- + list + A list of :class:`Column`. + + Notes + ----- + The order of arguments here is different from that of its JVM counterpart + because Python does not support method overloading. + + If no database is specified, the current database and catalog + are used. This API includes all temporary views. + + Examples + -------- + >>> _ = spark.sql("DROP TABLE IF EXISTS tbl1") + >>> _ = spark.sql("CREATE TABLE tblA (name STRING, age INT) USING parquet") + >>> spark.catalog.listColumns("tblA") + [Column(name='name', description=None, dataType='string', nullable=True, ... + >>> _ = spark.sql("DROP TABLE tblA") + """ + if dbName is None: + iter = self._jcatalog.listColumns(tableName).toLocalIterator() + else: + warnings.warn( + "`dbName` has been deprecated since Spark 3.4 and might be removed in " + "a future version. Use listColumns(`dbName.tableName`) instead.", + FutureWarning, + ) + iter = self._jcatalog.listColumns(dbName, tableName).toLocalIterator() + + columns = [] + while iter.hasNext(): + jcolumn = iter.next() + columns.append( + Column( + name=jcolumn.name(), + description=jcolumn.description(), + dataType=jcolumn.dataType(), + nullable=jcolumn.nullable(), + isPartition=jcolumn.isPartition(), + isBucket=jcolumn.isBucket(), + isCluster=jcolumn.isCluster(), + ) + ) + return columns + + def tableExists(self, tableName: str, dbName: Optional[str] = None) -> bool: + """Check if the table or view with the specified name exists. + This can either be a temporary view or a table/view. + + .. versionadded:: 3.3.0 + + Parameters + ---------- + tableName : str + name of the table to check existence. + If no database is specified, first try to treat ``tableName`` as a + multi-layer-namespace identifier, then try ``tableName`` as a normal table + name in the current database if necessary. + + .. versionchanged:: 3.4.0 + Allow ``tableName`` to be qualified with catalog name when ``dbName`` is None. + + dbName : str, optional + name of the database to check table existence in. + + Returns + ------- + bool + Indicating whether the table/view exists + + Examples + -------- + This function can check if a table is defined or not: + + >>> spark.catalog.tableExists("unexisting_table") + False + >>> _ = spark.sql("DROP TABLE IF EXISTS tbl1") + >>> _ = spark.sql("CREATE TABLE tbl1 (name STRING, age INT) USING parquet") + >>> spark.catalog.tableExists("tbl1") + True + + Using the fully qualified names for tables. + + >>> spark.catalog.tableExists("default.tbl1") + True + >>> spark.catalog.tableExists("spark_catalog.default.tbl1") + True + >>> spark.catalog.tableExists("tbl1", "default") + True + >>> _ = spark.sql("DROP TABLE tbl1") + + Check if views exist: + + >>> spark.catalog.tableExists("view1") + False + >>> _ = spark.sql("CREATE VIEW view1 AS SELECT 1") + >>> spark.catalog.tableExists("view1") + True + + Using the fully qualified names for views. + + >>> spark.catalog.tableExists("default.view1") + True + >>> spark.catalog.tableExists("spark_catalog.default.view1") + True + >>> spark.catalog.tableExists("view1", "default") + True + >>> _ = spark.sql("DROP VIEW view1") + + Check if temporary views exist: + + >>> _ = spark.sql("CREATE TEMPORARY VIEW view1 AS SELECT 1") + >>> spark.catalog.tableExists("view1") + True + >>> df = spark.sql("DROP VIEW view1") + >>> spark.catalog.tableExists("view1") + False + """ + if dbName is None: + return self._jcatalog.tableExists(tableName) + else: + warnings.warn( + "`dbName` has been deprecated since Spark 3.4 and might be removed in " + "a future version. Use tableExists(`dbName.tableName`) instead.", + FutureWarning, + ) + return self._jcatalog.tableExists(dbName, tableName) + + def createExternalTable( + self, + tableName: str, + path: Optional[str] = None, + source: Optional[str] = None, + schema: Optional[StructType] = None, + **options: str, + ) -> "DataFrame": + """Creates a table based on the dataset in a data source. + + It returns the DataFrame associated with the external table. + + The data source is specified by the ``source`` and a set of ``options``. + If ``source`` is not specified, the default data source configured by + ``spark.sql.sources.default`` will be used. + + Optionally, a schema can be provided as the schema of the returned :class:`DataFrame` and + created external table. + + .. versionadded:: 2.0.0 + + Returns + ------- + :class:`DataFrame` + """ + warnings.warn( + "createExternalTable is deprecated since Spark 2.2, please use createTable instead.", + FutureWarning, + ) + return self.createTable(tableName, path, source, schema, **options) + + def createTable( + self, + tableName: str, + path: Optional[str] = None, + source: Optional[str] = None, + schema: Optional[StructType] = None, + description: Optional[str] = None, + **options: str, + ) -> "DataFrame": + """Creates a table based on the dataset in a data source. + + .. versionadded:: 2.2.0 + + Parameters + ---------- + tableName : str + name of the table to create. + + .. versionchanged:: 3.4.0 + Allow ``tableName`` to be qualified with catalog name. + + path : str, optional + the path in which the data for this table exists. + When ``path`` is specified, an external table is + created from the data at the given path. Otherwise a managed table is created. + source : str, optional + the source of this table such as 'parquet, 'orc', etc. + If ``source`` is not specified, the default data source configured by + ``spark.sql.sources.default`` will be used. + schema : class:`StructType`, optional + the schema for this table. + description : str, optional + the description of this table. + + .. versionchanged:: 3.1.0 + Added the ``description`` parameter. + + **options : dict, optional + extra options to specify in the table. + + Returns + ------- + :class:`DataFrame` + The DataFrame associated with the table. + + Examples + -------- + Creating a managed table. + + >>> _ = spark.catalog.createTable("tbl1", schema=spark.range(1).schema, source='parquet') + >>> _ = spark.sql("DROP TABLE tbl1") + + Creating an external table + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="createTable") as d: + ... _ = spark.catalog.createTable( + ... "tbl2", schema=spark.range(1).schema, path=d, source='parquet') + >>> _ = spark.sql("DROP TABLE tbl2") + """ + if path is not None: + options["path"] = path + if source is None: + c = self._sparkSession._jconf + source = c.defaultDataSourceName() + if description is None: + description = "" + if schema is None: + df = self._jcatalog.createTable(tableName, source, description, options) + else: + if not isinstance(schema, StructType): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "schema", + "expected_type": "struct type", + "arg_type": type(schema).__name__, + }, + ) + scala_datatype = self._jsparkSession.parseDataType(schema.json()) + df = self._jcatalog.createTable(tableName, source, scala_datatype, description, options) + return DataFrame(df, self._sparkSession) + + def dropTempView(self, viewName: str) -> bool: + """Drops the local temporary view with the given view name in the catalog. + If the view has been cached before, then it will also be uncached. + Returns true if this view is dropped successfully, false otherwise. + + .. versionadded:: 2.0.0 + + Parameters + ---------- + viewName : str + name of the temporary view to drop. + + Returns + ------- + bool + If the temporary view was successfully dropped or not. + + .. versionadded:: 2.1.0 + The return type of this method was ``None`` in Spark 2.0, but changed to ``bool`` + in Spark 2.1. + + Examples + -------- + >>> spark.createDataFrame([(1, 1)]).createTempView("my_table") + + Dropping the temporary view. + + >>> spark.catalog.dropTempView("my_table") + True + + Throw an exception if the temporary view does not exists. + + >>> spark.table("my_table") + Traceback (most recent call last): + ... + AnalysisException: ... + """ + return self._jcatalog.dropTempView(viewName) + + def dropGlobalTempView(self, viewName: str) -> bool: + """Drops the global temporary view with the given view name in the catalog. + + .. versionadded:: 2.1.0 + + Parameters + ---------- + viewName : str + name of the global view to drop. + + Returns + ------- + bool + If the global view was successfully dropped or not. + + Notes + ----- + If the view has been cached before, then it will also be uncached. + + Examples + -------- + >>> spark.createDataFrame([(1, 1)]).createGlobalTempView("my_table") + + Dropping the global view. + + >>> spark.catalog.dropGlobalTempView("my_table") + True + + Throw an exception if the global view does not exists. + + >>> spark.table("global_temp.my_table") + Traceback (most recent call last): + ... + AnalysisException: ... + """ + return self._jcatalog.dropGlobalTempView(viewName) + + def registerFunction( + self, name: str, f: Callable[..., Any], returnType: Optional["DataTypeOrString"] = None + ) -> "UserDefinedFunctionLike": + """An alias for :func:`spark.udf.register`. + See :meth:`pyspark.sql.UDFRegistration.register`. + + .. versionadded:: 2.0.0 + + .. deprecated:: 2.3.0 + Use :func:`spark.udf.register` instead. + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + """ + warnings.warn("Deprecated in 2.3.0. Use spark.udf.register instead.", FutureWarning) + return self._sparkSession.udf.register(name, f, returnType) + + def isCached(self, tableName: str) -> bool: + """ + Returns true if the table is currently cached in-memory. + + .. versionadded:: 2.0.0 + + Parameters + ---------- + tableName : str + name of the table to get. + + .. versionchanged:: 3.4.0 + Allow ``tableName`` to be qualified with catalog name. + + Returns + ------- + bool + + Examples + -------- + >>> _ = spark.sql("DROP TABLE IF EXISTS tbl1") + >>> _ = spark.sql("CREATE TABLE tbl1 (name STRING, age INT) USING parquet") + >>> spark.catalog.cacheTable("tbl1") + >>> spark.catalog.isCached("tbl1") + True + + Throw an analysis exception when the table does not exist. + + >>> spark.catalog.isCached("not_existing_table") + Traceback (most recent call last): + ... + AnalysisException: ... + + Using the fully qualified name for the table. + + >>> spark.catalog.isCached("spark_catalog.default.tbl1") + True + >>> spark.catalog.uncacheTable("tbl1") + >>> _ = spark.sql("DROP TABLE tbl1") + """ + return self._jcatalog.isCached(tableName) + + def cacheTable(self, tableName: str, storageLevel: Optional[StorageLevel] = None) -> None: + """Caches the specified table in-memory or with given storage level. + Default MEMORY_AND_DISK. + + .. versionadded:: 2.0.0 + + Parameters + ---------- + tableName : str + name of the table to get. + + .. versionchanged:: 3.4.0 + Allow ``tableName`` to be qualified with catalog name. + + storageLevel : :class:`StorageLevel`, optional + storage level to set for persistence. + + .. versionchanged:: 3.5.0 + Allow to specify storage level. + + Notes + ----- + Cached data is shared across all Spark sessions on the cluster. + + Examples + -------- + >>> _ = spark.sql("DROP TABLE IF EXISTS tbl1") + >>> _ = spark.sql("CREATE TABLE tbl1 (name STRING, age INT) USING parquet") + >>> spark.catalog.cacheTable("tbl1") + + or + + >>> spark.catalog.cacheTable("tbl1", StorageLevel.OFF_HEAP) + + Throw an analysis exception when the table does not exist. + + >>> spark.catalog.cacheTable("not_existing_table") + Traceback (most recent call last): + ... + AnalysisException: ... + + Using the fully qualified name for the table. + + >>> spark.catalog.cacheTable("spark_catalog.default.tbl1") + >>> spark.catalog.uncacheTable("tbl1") + >>> _ = spark.sql("DROP TABLE tbl1") + """ + if storageLevel: + javaStorageLevel = self._sc._getJavaStorageLevel(storageLevel) + self._jcatalog.cacheTable(tableName, javaStorageLevel) + else: + self._jcatalog.cacheTable(tableName) + + def uncacheTable(self, tableName: str) -> None: + """Removes the specified table from the in-memory cache. + + .. versionadded:: 2.0.0 + + Parameters + ---------- + tableName : str + name of the table to get. + + .. versionchanged:: 3.4.0 + Allow ``tableName`` to be qualified with catalog name. + + Notes + ----- + Cached data is shared across all Spark sessions on the cluster, so uncaching it + affects all sessions. + + Examples + -------- + >>> _ = spark.sql("DROP TABLE IF EXISTS tbl1") + >>> _ = spark.sql("CREATE TABLE tbl1 (name STRING, age INT) USING parquet") + >>> spark.catalog.cacheTable("tbl1") + >>> spark.catalog.uncacheTable("tbl1") + >>> spark.catalog.isCached("tbl1") + False + + Throw an analysis exception when the table does not exist. + + >>> spark.catalog.uncacheTable("not_existing_table") + Traceback (most recent call last): + ... + AnalysisException: ... + + Using the fully qualified name for the table. + + >>> spark.catalog.uncacheTable("spark_catalog.default.tbl1") + >>> spark.catalog.isCached("tbl1") + False + >>> _ = spark.sql("DROP TABLE tbl1") + """ + self._jcatalog.uncacheTable(tableName) + + def clearCache(self) -> None: + """Removes all cached tables from the in-memory cache. + + .. versionadded:: 2.0.0 + + Notes + ----- + Cached data is shared across all Spark sessions on the cluster, so clearing + the cache affects all sessions. + + Examples + -------- + >>> _ = spark.sql("DROP TABLE IF EXISTS tbl1") + >>> _ = spark.sql("CREATE TABLE tbl1 (name STRING, age INT) USING parquet") + >>> spark.catalog.clearCache() + >>> spark.catalog.isCached("tbl1") + False + >>> _ = spark.sql("DROP TABLE tbl1") + """ + self._jcatalog.clearCache() + + def refreshTable(self, tableName: str) -> None: + """Invalidates and refreshes all the cached data and metadata of the given table. + + .. versionadded:: 2.0.0 + + Parameters + ---------- + tableName : str + name of the table to get. + + .. versionchanged:: 3.4.0 + Allow ``tableName`` to be qualified with catalog name. + + Examples + -------- + The example below caches a table, and then removes the data. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="refreshTable") as d: + ... _ = spark.sql("DROP TABLE IF EXISTS tbl1") + ... _ = spark.sql( + ... "CREATE TABLE tbl1 (col STRING) USING TEXT LOCATION '{}'".format(d)) + ... _ = spark.sql("INSERT INTO tbl1 SELECT 'abc'") + ... spark.catalog.cacheTable("tbl1") + ... spark.table("tbl1").show() + +---+ + |col| + +---+ + |abc| + +---+ + + Because the table is cached, it computes from the cached data as below. + + >>> spark.table("tbl1").count() + 1 + + After refreshing the table, it shows 0 because the data does not exist anymore. + + >>> spark.catalog.refreshTable("tbl1") + >>> spark.table("tbl1").count() + 0 + + Using the fully qualified name for the table. + + >>> spark.catalog.refreshTable("spark_catalog.default.tbl1") + >>> _ = spark.sql("DROP TABLE tbl1") + """ + self._jcatalog.refreshTable(tableName) + + def recoverPartitions(self, tableName: str) -> None: + """Recovers all the partitions in the directory of a table and updates the catalog. + + Only works with a partitioned table, and not a view. + + .. versionadded:: 2.1.1 + + Parameters + ---------- + tableName : str + Table name; may be qualified with catalog and database (namespace). If no database + identifier is provided, the name refers to a table in the current database. + + Examples + -------- + The example below creates a partitioned table against the existing directory of + the partitioned table. After that, it recovers the partitions. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="recoverPartitions") as d: + ... _ = spark.sql("DROP TABLE IF EXISTS tbl1") + ... p = d.replace("'", "''") + ... spark.range(1).selectExpr( + ... "id as key", "id as value").write.partitionBy( + ... "key").mode("overwrite").format("csv").save(d) + ... _ = spark.sql( + ... "CREATE TABLE tbl1 (key LONG, value LONG) USING csv OPTIONS (header false, path '{}') " + ... "PARTITIONED BY (key)".format(p)) + ... spark.table("tbl1").show() + ... spark.catalog.recoverPartitions("tbl1") + ... spark.table("tbl1").show() + +-----+---+ + |value|key| + +-----+---+ + +-----+---+ + +-----+---+ + |value|key| + +-----+---+ + | 0| 0| + +-----+---+ + >>> _ = spark.sql("DROP TABLE tbl1") + """ + self._jcatalog.recoverPartitions(tableName) + + def refreshByPath(self, path: str) -> None: + """Invalidates and refreshes all the cached data (and the associated metadata) for any + DataFrame that contains the given data source path. + + .. versionadded:: 2.2.0 + + Parameters + ---------- + path : str + the path to refresh the cache. + + Examples + -------- + The example below caches a table, and then removes the data. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="refreshByPath") as d: + ... _ = spark.sql("DROP TABLE IF EXISTS tbl1") + ... _ = spark.sql( + ... "CREATE TABLE tbl1 (col STRING) USING TEXT LOCATION '{}'".format(d)) + ... _ = spark.sql("INSERT INTO tbl1 SELECT 'abc'") + ... spark.catalog.cacheTable("tbl1") + ... spark.table("tbl1").show() + +---+ + |col| + +---+ + |abc| + +---+ + + Because the table is cached, it computes from the cached data as below. + + >>> spark.table("tbl1").count() + 1 + + After refreshing the table by path, it shows 0 because the data does not exist anymore. + + >>> spark.catalog.refreshByPath(d) + >>> spark.table("tbl1").count() + 0 + + >>> _ = spark.sql("DROP TABLE tbl1") + """ + self._jcatalog.refreshByPath(path) + + +def _test() -> None: + import os + import doctest + from pyspark.sql import SparkSession + import pyspark.sql.catalog + + os.chdir(os.environ["SPARK_HOME"]) + + globs = pyspark.sql.catalog.__dict__.copy() + globs["spark"] = ( + SparkSession.builder.master("local[4]").appName("sql.catalog tests").getOrCreate() + ) + failure_count, test_count = doctest.testmod( + pyspark.sql.catalog, + globs=globs, + optionflags=doctest.ELLIPSIS + | doctest.NORMALIZE_WHITESPACE + | doctest.IGNORE_EXCEPTION_DETAIL, + ) + globs["spark"].stop() + if failure_count: + sys.exit(-1) + + +if __name__ == "__main__": + _test() diff --git a/python/pyspark/sql/catalog.pyi b/python/pyspark/sql/catalog.pyi deleted file mode 100644 index dd61af41..00000000 --- a/python/pyspark/sql/catalog.pyi +++ /dev/null @@ -1,1246 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Generated by scripts/gen_type_stubs.py from python/pyspark/sql/catalog.py of Apache Spark v4.3.0: -# do not edit. Type checkers read this stub instead of the module of the drop-in. - -from typing import Any, Callable, Dict, NamedTuple, List, Optional -from pyspark.storagelevel import StorageLevel as StorageLevel -from pyspark.sql.dataframe import DataFrame -from pyspark.sql.session import SparkSession -from pyspark.sql.types import StructType -from pyspark.sql._typing import UserDefinedFunctionLike -from pyspark.sql._typing import DataTypeOrString - -class CatalogMetadata(NamedTuple): - name: str - description: Optional[str] - -class Database(NamedTuple): - name: str - catalog: Optional[str] - description: Optional[str] - locationUri: str - -class Table(NamedTuple): - name: str - catalog: Optional[str] - namespace: Optional[List[str]] - description: Optional[str] - tableType: str - isTemporary: bool - @property - def database(self) -> Optional[str]: ... - -class Column(NamedTuple): - name: str - description: Optional[str] - dataType: str - nullable: bool - isPartition: bool - isBucket: bool - isCluster: bool - -class Function(NamedTuple): - name: str - catalog: Optional[str] - namespace: Optional[List[str]] - description: Optional[str] - className: str - isTemporary: bool - -class TablePartition(NamedTuple): - partition: str - -class Catalog: - """Spark SQL catalog interface. - - Use :attr:`~pyspark.sql.SparkSession.catalog` on an active session. This class is a thin - wrapper around ``org.apache.spark.sql.catalog.Catalog``. - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - """ - def __init__(self, sparkSession: SparkSession) -> None: - """Create a new Catalog that wraps the underlying JVM object.""" - self._sparkSession = sparkSession - self._jsparkSession = sparkSession._jsparkSession - self._sc = sparkSession._sc - self._jcatalog = sparkSession._jsparkSession.catalog() - def currentCatalog(self) -> str: - """Returns the current catalog in this session. - - .. versionadded:: 3.4.0 - - Returns - ------- - str - The current catalog name. - - Examples - -------- - >>> spark.catalog.currentCatalog() - 'spark_catalog' - """ - def setCurrentCatalog(self, catalogName: str) -> None: - """Sets the current catalog in this session. - - .. versionadded:: 3.4.0 - - Parameters - ---------- - catalogName : str - Name of the catalog to set. - - Examples - -------- - >>> spark.catalog.setCurrentCatalog("spark_catalog") - """ - def listCatalogs(self, pattern: Optional[str]=None) -> List[CatalogMetadata]: - """Returns a list of catalogs available in this session. - - With ``pattern``, returns only catalogs whose name matches that pattern. - - .. versionadded:: 3.4.0 - - Parameters - ---------- - pattern : str, optional - Pattern that catalog names must match. - - .. versionadded:: 3.5.0 - - Returns - ------- - list - A list of :class:`CatalogMetadata`. - - Examples - -------- - >>> spark.catalog.listCatalogs() - [CatalogMetadata(name='spark_catalog', description=None)] - - >>> spark.catalog.listCatalogs("spark*") - [CatalogMetadata(name='spark_catalog', description=None)] - - >>> spark.catalog.listCatalogs("hive*") - [] - """ - def dropTable(self, tableName: str, ifExists: bool=False, purge: bool=False) -> None: - """Drops a persistent table. - - This does not remove temp views; use :meth:`Catalog.dropTempView`. - - .. versionadded:: 4.2.0 - - Parameters - ---------- - tableName : str - Name of the table to drop. May be qualified with catalog and database (namespace). - ifExists : bool, optional - If ``True``, do not fail when the table does not exist. - purge : bool, optional - If ``True``, skip moving data to a trash directory when the catalog supports it. - - Examples - -------- - >>> _ = spark.sql("DROP TABLE IF EXISTS tbl_drop_doc") - >>> _ = spark.sql("CREATE TABLE tbl_drop_doc (id INT) USING parquet") - >>> spark.catalog.dropTable("tbl_drop_doc") - >>> spark.catalog.tableExists("tbl_drop_doc") - False - """ - def dropView(self, viewName: str, ifExists: bool=False) -> None: - """Drops a persistent view. - - .. versionadded:: 4.2.0 - - Parameters - ---------- - viewName : str - Name of the view to drop. May be qualified with catalog and database (namespace). - ifExists : bool, optional - If ``True``, do not fail when the view does not exist. - - Examples - -------- - >>> _ = spark.sql("DROP VIEW IF EXISTS view_drop_doc") - >>> _ = spark.sql("CREATE VIEW view_drop_doc AS SELECT 1 AS c") - >>> spark.catalog.dropView("view_drop_doc") - >>> spark.catalog.tableExists("view_drop_doc") - False - """ - def createDatabase(self, dbName: str, ifNotExists: bool=False, properties: Optional[Dict[str, str]]=None) -> None: - """Creates a namespace (database/schema). - - ``dbName`` may be a multi-part identifier (for example ``catalog.schema``). Optional - ``properties`` follow ``CREATE NAMESPACE`` (for example comment or location keys). - - .. versionadded:: 4.2.0 - - Parameters - ---------- - dbName : str - Name of the namespace to create. - ifNotExists : bool, optional - If ``True``, do not fail when the namespace already exists. - properties : dict, optional - Map of namespace property keys to string values. - - Examples - -------- - >>> spark.catalog.dropDatabase("db_create_doc", ifExists=True, cascade=True) - >>> spark.catalog.createDatabase("db_create_doc") - >>> spark.catalog.databaseExists("db_create_doc") - True - >>> spark.catalog.dropDatabase("db_create_doc", cascade=True) - >>> spark.catalog.databaseExists("db_create_doc") - False - """ - def dropDatabase(self, dbName: str, ifExists: bool=False, cascade: bool=False) -> None: - """Drops a namespace. - - .. versionadded:: 4.2.0 - - Parameters - ---------- - dbName : str - Name of the namespace to drop. May be qualified with catalog name. - ifExists : bool, optional - If ``True``, do not fail when the namespace does not exist. - cascade : bool, optional - If ``True``, also drop tables and functions in the namespace. - - Examples - -------- - >>> spark.catalog.dropDatabase("db_drop_doc", ifExists=True, cascade=True) - >>> spark.catalog.createDatabase("db_drop_doc") - >>> spark.catalog.dropDatabase("db_drop_doc", cascade=True) - >>> spark.catalog.databaseExists("db_drop_doc") - False - """ - def listPartitions(self, tableName: str) -> List[TablePartition]: - """Lists partition value strings for a table (same as ``SHOW PARTITIONS``). - - .. versionadded:: 4.2.0 - - Parameters - ---------- - tableName : str - Name of the partitioned table. May be qualified with catalog and database (namespace). - - Returns - ------- - list - A list of :class:`TablePartition` (each ``partition`` field is a spec string). - - Examples - -------- - >>> _ = spark.sql("DROP TABLE IF EXISTS tbl_part_doc") - >>> _ = spark.sql( - ... "CREATE TABLE tbl_part_doc (id INT, p INT) USING parquet PARTITIONED BY (p)" - ... ) - >>> _ = spark.sql("INSERT INTO tbl_part_doc PARTITION (p = 1) SELECT 1") - >>> parts = [x.partition for x in spark.catalog.listPartitions("tbl_part_doc")] - >>> any("p=1" in s for s in parts) - True - >>> _ = spark.sql("DROP TABLE tbl_part_doc") - """ - def listViews(self, dbName: Optional[str]=None, pattern: Optional[str]=None) -> List[Table]: - """Lists views in a namespace. - - With no arguments, lists views in the current namespace. With ``dbName`` only, lists - views in that namespace (may be catalog-qualified). With ``pattern``, filters view names - with a SQL ``LIKE`` string; if ``pattern`` is given without ``dbName``, the current - database is used as the namespace. - - .. versionadded:: 4.2.0 - - Parameters - ---------- - dbName : str, optional - Namespace to list views from. May be qualified with catalog name. - pattern : str, optional - SQL ``LIKE`` pattern for view names. - - Returns - ------- - list - A list of :class:`Table` (same row shape as :meth:`Catalog.listTables`). - - Notes - ----- - Raises :class:`AnalysisException` if ``dbName`` names a namespace that does not exist. - - Examples - -------- - >>> _ = spark.sql("DROP VIEW IF EXISTS view_list_doc") - >>> _ = spark.sql("CREATE VIEW view_list_doc AS SELECT 1 AS c") - >>> "view_list_doc" in [v.name for v in spark.catalog.listViews()] - True - >>> "view_list_doc" in [v.name for v in spark.catalog.listViews(pattern="view_list_*")] - True - >>> _ = spark.sql("DROP VIEW view_list_doc") - """ - def getTableProperties(self, tableName: str) -> Dict[str, str]: - """Returns all table properties as a dict (same as ``SHOW TBLPROPERTIES``). - - .. versionadded:: 4.2.0 - - Parameters - ---------- - tableName : str - Table or view name. May be qualified with catalog and database (namespace). - - Returns - ------- - dict - Map of property key to value. - - Examples - -------- - >>> _ = spark.sql("DROP TABLE IF EXISTS tbl_props_doc") - >>> _ = spark.sql( - ... "CREATE TABLE tbl_props_doc (id INT) USING parquet " - ... "TBLPROPERTIES ('doc_prop_key' = 'doc_prop_val')" - ... ) - >>> spark.catalog.getTableProperties("tbl_props_doc")["doc_prop_key"] - 'doc_prop_val' - >>> _ = spark.sql("DROP TABLE tbl_props_doc") - """ - def getCreateTableString(self, tableName: str, asSerde: bool=False) -> str: - """Returns the ``SHOW CREATE TABLE`` DDL string for a relation. - - .. versionadded:: 4.2.0 - - Parameters - ---------- - tableName : str - Table or view name. May be qualified with catalog and database (namespace). - asSerde : bool, optional - If ``True``, request Hive serde DDL when applicable. - - Returns - ------- - str - DDL string from ``SHOW CREATE TABLE``. - - Examples - -------- - >>> _ = spark.sql("DROP TABLE IF EXISTS tbl_ddl_doc") - >>> _ = spark.sql("CREATE TABLE tbl_ddl_doc (id INT) USING parquet") - >>> "CREATE" in spark.catalog.getCreateTableString("tbl_ddl_doc").upper() - True - >>> _ = spark.sql("DROP TABLE tbl_ddl_doc") - """ - def truncateTable(self, tableName: str) -> None: - """Truncates a table (removes all data from the table; not supported for views). - - .. versionadded:: 4.2.0 - - Parameters - ---------- - tableName : str - Name of the table to truncate. May be qualified with catalog and database (namespace). - - Examples - -------- - >>> _ = spark.sql("DROP TABLE IF EXISTS tbl_tr_doc") - >>> _ = spark.sql("CREATE TABLE tbl_tr_doc (id INT) USING csv") - >>> _ = spark.sql("INSERT INTO tbl_tr_doc VALUES (1), (2)") - >>> spark.table("tbl_tr_doc").count() - 2 - >>> spark.catalog.truncateTable("tbl_tr_doc") - >>> spark.table("tbl_tr_doc").count() - 0 - >>> _ = spark.sql("DROP TABLE tbl_tr_doc") - """ - def analyzeTable(self, tableName: str, noScan: bool=False) -> None: - """Computes table statistics (same as SQL ``ANALYZE TABLE COMPUTE STATISTICS``). - - .. versionadded:: 4.2.0 - - Parameters - ---------- - tableName : str - Table or view name. May be qualified with catalog and database (namespace). - noScan : bool, optional - If ``True``, use ``NOSCAN`` mode (reuse existing column statistics where possible). - - Examples - -------- - >>> _ = spark.sql("DROP TABLE IF EXISTS tbl_an_doc") - >>> _ = spark.sql("CREATE TABLE tbl_an_doc (id INT) USING csv") - >>> _ = spark.sql("INSERT INTO tbl_an_doc VALUES (1)") - >>> spark.catalog.analyzeTable("tbl_an_doc") - >>> spark.catalog.analyzeTable("tbl_an_doc", noScan=True) - >>> _ = spark.sql("DROP TABLE tbl_an_doc") - """ - def currentDatabase(self) -> str: - """Returns the current database (namespace) in this session. - - .. versionadded:: 2.0.0 - - Returns - ------- - str - The current database (namespace) name. - - Examples - -------- - >>> spark.catalog.currentDatabase() - 'default' - """ - def setCurrentDatabase(self, dbName: str) -> None: - """Sets the current database (namespace) in this session. - - .. versionadded:: 2.0.0 - - Parameters - ---------- - dbName : str - Name of the database (namespace) to set. - - Examples - -------- - >>> spark.catalog.setCurrentDatabase("default") - """ - def listDatabases(self, pattern: Optional[str]=None) -> List[Database]: - """Returns a list of databases (namespaces) available within the current catalog. - - With ``pattern``, returns only databases whose name matches that pattern. - - .. versionadded:: 2.0.0 - - Parameters - ---------- - pattern : str, optional - Pattern that database names must match. - - .. versionadded:: 3.5.0 - - Returns - ------- - list - A list of :class:`Database`. - - Examples - -------- - >>> spark.catalog.listDatabases() - [Database(name='default', catalog='spark_catalog', description='default database', ... - - >>> spark.catalog.listDatabases("def*") - [Database(name='default', catalog='spark_catalog', description='default database', ... - - >>> spark.catalog.listDatabases("def2*") - [] - """ - def getDatabase(self, dbName: str) -> Database: - """Get the database with the specified name. - This throws an :class:`AnalysisException` when the database cannot be found. - - .. versionadded:: 3.4.0 - - Parameters - ---------- - dbName : str - name of the database to get. - - Returns - ------- - :class:`Database` - The database found by the name. - - Examples - -------- - >>> spark.catalog.getDatabase("default") - Database(name='default', catalog='spark_catalog', description='default database', ... - - Using the fully qualified name with the catalog name. - - >>> spark.catalog.getDatabase("spark_catalog.default") - Database(name='default', catalog='spark_catalog', description='default database', ... - """ - def databaseExists(self, dbName: str) -> bool: - """Check if the database with the specified name exists. - - .. versionadded:: 3.3.0 - - Parameters - ---------- - dbName : str - name of the database to check existence - - .. versionchanged:: 3.4.0 - Allow ``dbName`` to be qualified with catalog name. - - Returns - ------- - bool - Indicating whether the database exists - - Examples - -------- - Check if 'test_new_database' database exists - - >>> spark.catalog.databaseExists("test_new_database") - False - >>> _ = spark.sql("CREATE DATABASE test_new_database") - >>> spark.catalog.databaseExists("test_new_database") - True - - Using the fully qualified name with the catalog name. - - >>> spark.catalog.databaseExists("spark_catalog.test_new_database") - True - >>> _ = spark.sql("DROP DATABASE test_new_database") - """ - def listTables(self, dbName: Optional[str]=None, pattern: Optional[str]=None) -> List[Table]: - """Returns a list of tables/views in the current database (namespace), or in the database - given by ``dbName`` when provided (the name may be qualified with catalog). - - With ``pattern``, returns only tables and views whose name matches the pattern. This - includes all temporary views. - - .. versionadded:: 2.0.0 - - Parameters - ---------- - dbName : str, optional - Database (namespace) to list tables from. If omitted, the current database is used. - - .. versionchanged:: 3.4.0 - Allow ``dbName`` to be qualified with catalog name. - - pattern : str, optional - Pattern that table names must match. - - .. versionadded:: 3.5.0 - - Returns - ------- - list - A list of :class:`Table`. - - Examples - -------- - >>> spark.range(1).createTempView("test_view") - >>> spark.catalog.listTables() - [Table(name='test_view', catalog=None, namespace=[], description=None, ... - - >>> spark.catalog.listTables(pattern="test*") - [Table(name='test_view', catalog=None, namespace=[], description=None, ... - - >>> spark.catalog.listTables(pattern="table*") - [] - - >>> _ = spark.catalog.dropTempView("test_view") - >>> spark.catalog.listTables() - [] - """ - def getTable(self, tableName: str) -> Table: - """Get the table or view with the specified name. This table can be a temporary view or a - table/view. This throws an :class:`AnalysisException` when no Table can be found. - - .. versionadded:: 3.4.0 - - Parameters - ---------- - tableName : str - name of the table to get. - - .. versionchanged:: 3.4.0 - Allow `tableName` to be qualified with catalog name. - - Returns - ------- - :class:`Table` - The table found by the name. - - Examples - -------- - >>> _ = spark.sql("DROP TABLE IF EXISTS tbl1") - >>> _ = spark.sql("CREATE TABLE tbl1 (name STRING, age INT) USING parquet") - >>> spark.catalog.getTable("tbl1") - Table(name='tbl1', catalog='spark_catalog', namespace=['default'], ... - - Using the fully qualified name with the catalog name. - - >>> spark.catalog.getTable("default.tbl1") - Table(name='tbl1', catalog='spark_catalog', namespace=['default'], ... - >>> spark.catalog.getTable("spark_catalog.default.tbl1") - Table(name='tbl1', catalog='spark_catalog', namespace=['default'], ... - >>> _ = spark.sql("DROP TABLE tbl1") - - Throw an analysis exception when the table does not exist. - - >>> spark.catalog.getTable("tbl1") - Traceback (most recent call last): - ... - AnalysisException: ... - """ - def listFunctions(self, dbName: Optional[str]=None, pattern: Optional[str]=None) -> List[Function]: - """Returns a list of functions registered in the current database (namespace), or in the - database given by ``dbName`` when provided (the name may be qualified with catalog). - - With ``pattern``, returns only functions whose name matches the pattern. This includes all - built-in and temporary functions. - - .. versionadded:: 3.4.0 - - Parameters - ---------- - dbName : str, optional - Database (namespace) to list functions from. If omitted, the current database is used. - May be qualified with catalog name. - pattern : str, optional - Pattern that function names must match. - - .. versionadded:: 3.5.0 - - Returns - ------- - list - A list of :class:`Function`. - - Examples - -------- - >>> spark.catalog.listFunctions() - [Function(name=... - - >>> spark.catalog.listFunctions(pattern="to_*") - [Function(name=... - - >>> spark.catalog.listFunctions(pattern="*not_existing_func*") - [] - """ - def functionExists(self, functionName: str, dbName: Optional[str]=None) -> bool: - """Check if the function with the specified name exists. - This can either be a temporary function or a function. - - .. versionadded:: 3.3.0 - - Parameters - ---------- - functionName : str - name of the function to check existence - - .. versionchanged:: 3.4.0 - Allow ``functionName`` to be qualified with catalog name - - dbName : str, optional - name of the database to check function existence in. - - Returns - ------- - bool - Indicating whether the function exists - - Notes - ----- - If no database is specified, the current database and catalog - are used. This API includes all temporary functions. - - Examples - -------- - >>> spark.catalog.functionExists("count") - True - - Using the fully qualified name for function name. - - >>> spark.catalog.functionExists("default.unexisting_function") - False - >>> spark.catalog.functionExists("spark_catalog.default.unexisting_function") - False - """ - def getFunction(self, functionName: str) -> Function: - """Get the function with the specified name. This function can be a temporary function or a - function. This throws an :class:`AnalysisException` when the function cannot be found. - - .. versionadded:: 3.4.0 - - Parameters - ---------- - functionName : str - name of the function to check existence. - - Returns - ------- - :class:`Function` - The function found by the name. - - Examples - -------- - >>> _ = spark.sql( - ... "CREATE FUNCTION my_func1 AS 'test.org.apache.spark.sql.MyDoubleAvg'") - >>> spark.catalog.getFunction("my_func1") - Function(name='my_func1', catalog='spark_catalog', namespace=['default'], ... - - Using the fully qualified name for function name. - - >>> spark.catalog.getFunction("default.my_func1") - Function(name='my_func1', catalog='spark_catalog', namespace=['default'], ... - >>> spark.catalog.getFunction("spark_catalog.default.my_func1") - Function(name='my_func1', catalog='spark_catalog', namespace=['default'], ... - - Throw an analysis exception when the function does not exists. - - >>> spark.catalog.getFunction("my_func2") - Traceback (most recent call last): - ... - AnalysisException: ... - """ - def listColumns(self, tableName: str, dbName: Optional[str]=None) -> List[Column]: - """Returns a list of columns for the given table/view in the specified database. - - .. versionadded:: 2.0.0 - - Parameters - ---------- - tableName : str - name of the table to list columns. - - .. versionchanged:: 3.4.0 - Allow ``tableName`` to be qualified with catalog name when ``dbName`` is None. - - dbName : str, optional - name of the database to find the table to list columns. - - Returns - ------- - list - A list of :class:`Column`. - - Notes - ----- - The order of arguments here is different from that of its JVM counterpart - because Python does not support method overloading. - - If no database is specified, the current database and catalog - are used. This API includes all temporary views. - - Examples - -------- - >>> _ = spark.sql("DROP TABLE IF EXISTS tbl1") - >>> _ = spark.sql("CREATE TABLE tblA (name STRING, age INT) USING parquet") - >>> spark.catalog.listColumns("tblA") - [Column(name='name', description=None, dataType='string', nullable=True, ... - >>> _ = spark.sql("DROP TABLE tblA") - """ - def tableExists(self, tableName: str, dbName: Optional[str]=None) -> bool: - """Check if the table or view with the specified name exists. - This can either be a temporary view or a table/view. - - .. versionadded:: 3.3.0 - - Parameters - ---------- - tableName : str - name of the table to check existence. - If no database is specified, first try to treat ``tableName`` as a - multi-layer-namespace identifier, then try ``tableName`` as a normal table - name in the current database if necessary. - - .. versionchanged:: 3.4.0 - Allow ``tableName`` to be qualified with catalog name when ``dbName`` is None. - - dbName : str, optional - name of the database to check table existence in. - - Returns - ------- - bool - Indicating whether the table/view exists - - Examples - -------- - This function can check if a table is defined or not: - - >>> spark.catalog.tableExists("unexisting_table") - False - >>> _ = spark.sql("DROP TABLE IF EXISTS tbl1") - >>> _ = spark.sql("CREATE TABLE tbl1 (name STRING, age INT) USING parquet") - >>> spark.catalog.tableExists("tbl1") - True - - Using the fully qualified names for tables. - - >>> spark.catalog.tableExists("default.tbl1") - True - >>> spark.catalog.tableExists("spark_catalog.default.tbl1") - True - >>> spark.catalog.tableExists("tbl1", "default") - True - >>> _ = spark.sql("DROP TABLE tbl1") - - Check if views exist: - - >>> spark.catalog.tableExists("view1") - False - >>> _ = spark.sql("CREATE VIEW view1 AS SELECT 1") - >>> spark.catalog.tableExists("view1") - True - - Using the fully qualified names for views. - - >>> spark.catalog.tableExists("default.view1") - True - >>> spark.catalog.tableExists("spark_catalog.default.view1") - True - >>> spark.catalog.tableExists("view1", "default") - True - >>> _ = spark.sql("DROP VIEW view1") - - Check if temporary views exist: - - >>> _ = spark.sql("CREATE TEMPORARY VIEW view1 AS SELECT 1") - >>> spark.catalog.tableExists("view1") - True - >>> df = spark.sql("DROP VIEW view1") - >>> spark.catalog.tableExists("view1") - False - """ - def createExternalTable(self, tableName: str, path: Optional[str]=None, source: Optional[str]=None, schema: Optional[StructType]=None, **options: str) -> 'DataFrame': - """Creates a table based on the dataset in a data source. - - It returns the DataFrame associated with the external table. - - The data source is specified by the ``source`` and a set of ``options``. - If ``source`` is not specified, the default data source configured by - ``spark.sql.sources.default`` will be used. - - Optionally, a schema can be provided as the schema of the returned :class:`DataFrame` and - created external table. - - .. versionadded:: 2.0.0 - - Returns - ------- - :class:`DataFrame` - """ - def createTable(self, tableName: str, path: Optional[str]=None, source: Optional[str]=None, schema: Optional[StructType]=None, description: Optional[str]=None, **options: str) -> 'DataFrame': - """Creates a table based on the dataset in a data source. - - .. versionadded:: 2.2.0 - - Parameters - ---------- - tableName : str - name of the table to create. - - .. versionchanged:: 3.4.0 - Allow ``tableName`` to be qualified with catalog name. - - path : str, optional - the path in which the data for this table exists. - When ``path`` is specified, an external table is - created from the data at the given path. Otherwise a managed table is created. - source : str, optional - the source of this table such as 'parquet, 'orc', etc. - If ``source`` is not specified, the default data source configured by - ``spark.sql.sources.default`` will be used. - schema : class:`StructType`, optional - the schema for this table. - description : str, optional - the description of this table. - - .. versionchanged:: 3.1.0 - Added the ``description`` parameter. - - **options : dict, optional - extra options to specify in the table. - - Returns - ------- - :class:`DataFrame` - The DataFrame associated with the table. - - Examples - -------- - Creating a managed table. - - >>> _ = spark.catalog.createTable("tbl1", schema=spark.range(1).schema, source='parquet') - >>> _ = spark.sql("DROP TABLE tbl1") - - Creating an external table - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="createTable") as d: - ... _ = spark.catalog.createTable( - ... "tbl2", schema=spark.range(1).schema, path=d, source='parquet') - >>> _ = spark.sql("DROP TABLE tbl2") - """ - def dropTempView(self, viewName: str) -> bool: - """Drops the local temporary view with the given view name in the catalog. - If the view has been cached before, then it will also be uncached. - Returns true if this view is dropped successfully, false otherwise. - - .. versionadded:: 2.0.0 - - Parameters - ---------- - viewName : str - name of the temporary view to drop. - - Returns - ------- - bool - If the temporary view was successfully dropped or not. - - .. versionadded:: 2.1.0 - The return type of this method was ``None`` in Spark 2.0, but changed to ``bool`` - in Spark 2.1. - - Examples - -------- - >>> spark.createDataFrame([(1, 1)]).createTempView("my_table") - - Dropping the temporary view. - - >>> spark.catalog.dropTempView("my_table") - True - - Throw an exception if the temporary view does not exists. - - >>> spark.table("my_table") - Traceback (most recent call last): - ... - AnalysisException: ... - """ - def dropGlobalTempView(self, viewName: str) -> bool: - """Drops the global temporary view with the given view name in the catalog. - - .. versionadded:: 2.1.0 - - Parameters - ---------- - viewName : str - name of the global view to drop. - - Returns - ------- - bool - If the global view was successfully dropped or not. - - Notes - ----- - If the view has been cached before, then it will also be uncached. - - Examples - -------- - >>> spark.createDataFrame([(1, 1)]).createGlobalTempView("my_table") - - Dropping the global view. - - >>> spark.catalog.dropGlobalTempView("my_table") - True - - Throw an exception if the global view does not exists. - - >>> spark.table("global_temp.my_table") - Traceback (most recent call last): - ... - AnalysisException: ... - """ - def registerFunction(self, name: str, f: Callable[..., Any], returnType: Optional['DataTypeOrString']=None) -> 'UserDefinedFunctionLike': - """An alias for :func:`spark.udf.register`. - See :meth:`pyspark.sql.UDFRegistration.register`. - - .. versionadded:: 2.0.0 - - .. deprecated:: 2.3.0 - Use :func:`spark.udf.register` instead. - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - """ - def isCached(self, tableName: str) -> bool: - """Returns true if the table is currently cached in-memory. - - .. versionadded:: 2.0.0 - - Parameters - ---------- - tableName : str - name of the table to get. - - .. versionchanged:: 3.4.0 - Allow ``tableName`` to be qualified with catalog name. - - Returns - ------- - bool - - Examples - -------- - >>> _ = spark.sql("DROP TABLE IF EXISTS tbl1") - >>> _ = spark.sql("CREATE TABLE tbl1 (name STRING, age INT) USING parquet") - >>> spark.catalog.cacheTable("tbl1") - >>> spark.catalog.isCached("tbl1") - True - - Throw an analysis exception when the table does not exist. - - >>> spark.catalog.isCached("not_existing_table") - Traceback (most recent call last): - ... - AnalysisException: ... - - Using the fully qualified name for the table. - - >>> spark.catalog.isCached("spark_catalog.default.tbl1") - True - >>> spark.catalog.uncacheTable("tbl1") - >>> _ = spark.sql("DROP TABLE tbl1") - """ - def cacheTable(self, tableName: str, storageLevel: Optional[StorageLevel]=None) -> None: - """Caches the specified table in-memory or with given storage level. - Default MEMORY_AND_DISK. - - .. versionadded:: 2.0.0 - - Parameters - ---------- - tableName : str - name of the table to get. - - .. versionchanged:: 3.4.0 - Allow ``tableName`` to be qualified with catalog name. - - storageLevel : :class:`StorageLevel`, optional - storage level to set for persistence. - - .. versionchanged:: 3.5.0 - Allow to specify storage level. - - Notes - ----- - Cached data is shared across all Spark sessions on the cluster. - - Examples - -------- - >>> _ = spark.sql("DROP TABLE IF EXISTS tbl1") - >>> _ = spark.sql("CREATE TABLE tbl1 (name STRING, age INT) USING parquet") - >>> spark.catalog.cacheTable("tbl1") - - or - - >>> spark.catalog.cacheTable("tbl1", StorageLevel.OFF_HEAP) - - Throw an analysis exception when the table does not exist. - - >>> spark.catalog.cacheTable("not_existing_table") - Traceback (most recent call last): - ... - AnalysisException: ... - - Using the fully qualified name for the table. - - >>> spark.catalog.cacheTable("spark_catalog.default.tbl1") - >>> spark.catalog.uncacheTable("tbl1") - >>> _ = spark.sql("DROP TABLE tbl1") - """ - def uncacheTable(self, tableName: str) -> None: - """Removes the specified table from the in-memory cache. - - .. versionadded:: 2.0.0 - - Parameters - ---------- - tableName : str - name of the table to get. - - .. versionchanged:: 3.4.0 - Allow ``tableName`` to be qualified with catalog name. - - Notes - ----- - Cached data is shared across all Spark sessions on the cluster, so uncaching it - affects all sessions. - - Examples - -------- - >>> _ = spark.sql("DROP TABLE IF EXISTS tbl1") - >>> _ = spark.sql("CREATE TABLE tbl1 (name STRING, age INT) USING parquet") - >>> spark.catalog.cacheTable("tbl1") - >>> spark.catalog.uncacheTable("tbl1") - >>> spark.catalog.isCached("tbl1") - False - - Throw an analysis exception when the table does not exist. - - >>> spark.catalog.uncacheTable("not_existing_table") - Traceback (most recent call last): - ... - AnalysisException: ... - - Using the fully qualified name for the table. - - >>> spark.catalog.uncacheTable("spark_catalog.default.tbl1") - >>> spark.catalog.isCached("tbl1") - False - >>> _ = spark.sql("DROP TABLE tbl1") - """ - def clearCache(self) -> None: - """Removes all cached tables from the in-memory cache. - - .. versionadded:: 2.0.0 - - Notes - ----- - Cached data is shared across all Spark sessions on the cluster, so clearing - the cache affects all sessions. - - Examples - -------- - >>> _ = spark.sql("DROP TABLE IF EXISTS tbl1") - >>> _ = spark.sql("CREATE TABLE tbl1 (name STRING, age INT) USING parquet") - >>> spark.catalog.clearCache() - >>> spark.catalog.isCached("tbl1") - False - >>> _ = spark.sql("DROP TABLE tbl1") - """ - def refreshTable(self, tableName: str) -> None: - """Invalidates and refreshes all the cached data and metadata of the given table. - - .. versionadded:: 2.0.0 - - Parameters - ---------- - tableName : str - name of the table to get. - - .. versionchanged:: 3.4.0 - Allow ``tableName`` to be qualified with catalog name. - - Examples - -------- - The example below caches a table, and then removes the data. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="refreshTable") as d: - ... _ = spark.sql("DROP TABLE IF EXISTS tbl1") - ... _ = spark.sql( - ... "CREATE TABLE tbl1 (col STRING) USING TEXT LOCATION '{}'".format(d)) - ... _ = spark.sql("INSERT INTO tbl1 SELECT 'abc'") - ... spark.catalog.cacheTable("tbl1") - ... spark.table("tbl1").show() - +---+ - |col| - +---+ - |abc| - +---+ - - Because the table is cached, it computes from the cached data as below. - - >>> spark.table("tbl1").count() - 1 - - After refreshing the table, it shows 0 because the data does not exist anymore. - - >>> spark.catalog.refreshTable("tbl1") - >>> spark.table("tbl1").count() - 0 - - Using the fully qualified name for the table. - - >>> spark.catalog.refreshTable("spark_catalog.default.tbl1") - >>> _ = spark.sql("DROP TABLE tbl1") - """ - def recoverPartitions(self, tableName: str) -> None: - """Recovers all the partitions in the directory of a table and updates the catalog. - - Only works with a partitioned table, and not a view. - - .. versionadded:: 2.1.1 - - Parameters - ---------- - tableName : str - Table name; may be qualified with catalog and database (namespace). If no database - identifier is provided, the name refers to a table in the current database. - - Examples - -------- - The example below creates a partitioned table against the existing directory of - the partitioned table. After that, it recovers the partitions. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="recoverPartitions") as d: - ... _ = spark.sql("DROP TABLE IF EXISTS tbl1") - ... p = d.replace("'", "''") - ... spark.range(1).selectExpr( - ... "id as key", "id as value").write.partitionBy( - ... "key").mode("overwrite").format("csv").save(d) - ... _ = spark.sql( - ... "CREATE TABLE tbl1 (key LONG, value LONG) USING csv OPTIONS (header false, path '{}') " - ... "PARTITIONED BY (key)".format(p)) - ... spark.table("tbl1").show() - ... spark.catalog.recoverPartitions("tbl1") - ... spark.table("tbl1").show() - +-----+---+ - |value|key| - +-----+---+ - +-----+---+ - +-----+---+ - |value|key| - +-----+---+ - | 0| 0| - +-----+---+ - >>> _ = spark.sql("DROP TABLE tbl1") - """ - def refreshByPath(self, path: str) -> None: - """Invalidates and refreshes all the cached data (and the associated metadata) for any - DataFrame that contains the given data source path. - - .. versionadded:: 2.2.0 - - Parameters - ---------- - path : str - the path to refresh the cache. - - Examples - -------- - The example below caches a table, and then removes the data. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="refreshByPath") as d: - ... _ = spark.sql("DROP TABLE IF EXISTS tbl1") - ... _ = spark.sql( - ... "CREATE TABLE tbl1 (col STRING) USING TEXT LOCATION '{}'".format(d)) - ... _ = spark.sql("INSERT INTO tbl1 SELECT 'abc'") - ... spark.catalog.cacheTable("tbl1") - ... spark.table("tbl1").show() - +---+ - |col| - +---+ - |abc| - +---+ - - Because the table is cached, it computes from the cached data as below. - - >>> spark.table("tbl1").count() - 1 - - After refreshing the table by path, it shows 0 because the data does not exist anymore. - - >>> spark.catalog.refreshByPath(d) - >>> spark.table("tbl1").count() - 0 - - >>> _ = spark.sql("DROP TABLE tbl1") - """ diff --git a/python/pyspark/sql/connect/catalog.py b/python/pyspark/sql/connect/catalog.py index c0029e54..6373289b 100644 --- a/python/pyspark/sql/connect/catalog.py +++ b/python/pyspark/sql/connect/catalog.py @@ -14,9 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""Catalog under the connect import path (mirrors pyspark.sql.connect.catalog).""" + +"""The Catalog of Spark Connect, as PySpark Connect's ``pyspark.sql.connect.catalog``: the native +Catalog of the Rust core, whose methods return the NamedTuples of PySpark's classic +``pyspark.sql.catalog`` (Database, Table, Column, Function, CatalogMetadata and TablePartition), +which PySpark Connect's module imports too.""" +from pyspark._pyspark import Catalog from pyspark.sql.catalog import ( - Catalog, CatalogMetadata, Column, Database, diff --git a/python/pyspark/sql/connect/client/__init__.py b/python/pyspark/sql/connect/client/__init__.py index c0f8fe72..d5d042f0 100644 --- a/python/pyspark/sql/connect/client/__init__.py +++ b/python/pyspark/sql/connect/client/__init__.py @@ -1446,7 +1446,7 @@ def register_udtf( valid arguments.""" import pyspark.sql.connect.proto as pb2 from pyspark.sql.connect.utils import get_python_ver - from pyspark.sql.udtf import _pickled_handler + from pyspark.sql.connect.udtf import _pickled_handler # As PySpark's PythonUDTF, which reads the name of the handler class when it is made. _ = function.__name__ diff --git a/python/pyspark/sql/connect/group.py b/python/pyspark/sql/connect/group.py index f866e1f3..f097a68e 100644 --- a/python/pyspark/sql/connect/group.py +++ b/python/pyspark/sql/connect/group.py @@ -14,8 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""GroupedData / PandasCogroupedOps under the connect import path.""" -from pyspark.sql.group import GroupedData -from pyspark.sql.pandas.group_ops import PandasCogroupedOps + +"""GroupedData / PandasCogroupedOps of Spark Connect, as PySpark Connect's +``pyspark.sql.connect.group``: the native classes of the Rust core.""" +from pyspark._pyspark import GroupedData, PandasCogroupedOps __all__ = ["GroupedData", "PandasCogroupedOps"] diff --git a/python/pyspark/sql/connect/readwriter.py b/python/pyspark/sql/connect/readwriter.py index cee5c691..03cd70f3 100644 --- a/python/pyspark/sql/connect/readwriter.py +++ b/python/pyspark/sql/connect/readwriter.py @@ -14,15 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""DataFrameReader / DataFrameWriter(V2) under the connect import path.""" + +"""DataFrameReader / DataFrameWriter(V2) of Spark Connect, as PySpark Connect's +``pyspark.sql.connect.readwriter``: the native readers and writers of the Rust core.""" +from pyspark._pyspark import DataFrameReader, DataFrameWriter, DataFrameWriterV2 + # PathOrPaths and TupleOrListOfString: PySpark Connect's aliases of the types of the paths and -# of the lists of column names, which are those of pyspark.sql.readwriter. -from pyspark.sql.readwriter import ( # noqa: F401 - DataFrameReader, - DataFrameWriter, - DataFrameWriterV2, - PathOrPaths, - TupleOrListOfString, -) +# of the lists of column names, which are those of PySpark's classic pyspark.sql.readwriter. +from pyspark.sql.readwriter import PathOrPaths, TupleOrListOfString # noqa: F401 __all__ = ["DataFrameReader", "DataFrameWriter", "DataFrameWriterV2"] diff --git a/python/pyspark/sql/connect/streaming/query.py b/python/pyspark/sql/connect/streaming/query.py index 0bda959d..afac9907 100644 --- a/python/pyspark/sql/connect/streaming/query.py +++ b/python/pyspark/sql/connect/streaming/query.py @@ -14,7 +14,93 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""StreamingQuery / StreamingQueryManager under the connect import path.""" -from pyspark.sql.streaming.query import StreamingQuery, StreamingQueryManager + +""" +The streaming queries of Spark Connect, as PySpark Connect's +``pyspark.sql.connect.streaming.query``: the native StreamingQuery and StreamingQueryManager of the +Rust core, and the Python side of the listener bus of the client, which decodes and dispatches the +events of the listeners. +""" + +import json +import warnings + +from pyspark._pyspark import ( + StreamingQuery as _StreamingQuery, +) +from pyspark._pyspark import ( + StreamingQueryManager as _StreamingQueryManager, +) + +# As PySpark's, the exception of a failed query is that of pyspark.errors. +from pyspark.errors import ( + PySparkValueError, + StreamingQueryException, # noqa: F401 +) +from pyspark.sql.streaming.listener import ( + QueryIdleEvent, + QueryProgressEvent, + QueryStartedEvent, + QueryTerminatedEvent, + StreamingQueryListener, # noqa: F401 (an attribute of PySpark Connect's module too) +) + +# StreamingQueryEventType values (spark.connect.StreamingQueryEventType); the Rust +# listener stream yields these as the event_type int alongside each event's JSON. The started +# event of a query comes with the result of its start, which the Rust core posts as +# spark_connect::streaming::QUERY_STARTED_EVENT, of no StreamingQueryEventType. +_QUERY_STARTED_EVENT = -1 +_QUERY_PROGRESS_EVENT = 1 +_QUERY_TERMINATED_EVENT = 2 +_QUERY_IDLE_EVENT = 3 + +# Re-export the core classes from the Rust extension +StreamingQuery = _StreamingQuery + + +def _deserialize_listener_event(event_type, event_json, *, started=False): + """Decode a protocol event before entering callback exception isolation. + + Started events belong to the synchronous start response, not the asynchronous + protocol enum. Malformed JSON and missing fields propagate to the caller, as in + reference PySpark's writer and listener-bus deserializers. + """ + if started and event_type == _QUERY_STARTED_EVENT: + return QueryStartedEvent.fromJson(json.loads(event_json)) + if event_type == _QUERY_PROGRESS_EVENT: + return QueryProgressEvent.fromJson(json.loads(event_json)) + if event_type == _QUERY_TERMINATED_EVENT: + return QueryTerminatedEvent.fromJson(json.loads(event_json)) + if event_type == _QUERY_IDLE_EVENT: + return QueryIdleEvent.fromJson(json.loads(event_json)) + raise PySparkValueError( + errorClass="UNKNOWN_VALUE_FOR", + messageParameters={"var": f"proto.StreamingQueryEventType: {event_type}"}, + ) + + +def _dispatch_listener_event(listener, event): + """Warn on callback Exception; BaseException and warning-as-error escape to the bus.""" + try: + if isinstance(event, QueryStartedEvent): + listener.onQueryStarted(event) + elif isinstance(event, QueryProgressEvent): + listener.onQueryProgress(event) + elif isinstance(event, QueryIdleEvent): + listener.onQueryIdle(event) + elif isinstance(event, QueryTerminatedEvent): + listener.onQueryTerminated(event) + else: + warnings.warn(f"Unknown StreamingQueryListener event: {event}") + except Exception as e: # noqa: BLE001 + warnings.warn(f"Listener {str(listener)} threw an exception\n{e}") + + +# ``spark.streams`` returns the native (Rust) StreamingQueryManager, which owns the +# client-side listener bus: addListener / removeListener / close are implemented in the +# Rust core (so Rust clients get the listener feature too). Re-export it as the public +# class. +StreamingQueryManager = _StreamingQueryManager + __all__ = ["StreamingQuery", "StreamingQueryManager"] diff --git a/python/pyspark/sql/connect/streaming/query.pyi b/python/pyspark/sql/connect/streaming/query.pyi index 6ed7707f..42b164e8 100644 --- a/python/pyspark/sql/connect/streaming/query.pyi +++ b/python/pyspark/sql/connect/streaming/query.pyi @@ -20,10 +20,10 @@ import warnings from typing import Any, cast, Dict, List, Optional, Union, Iterator from threading import Thread, Lock -from pyspark.errors import StreamingQueryException +from pyspark.errors import StreamingQueryException as StreamingQueryException, PySparkValueError as PySparkValueError import pyspark.sql.connect.proto as pb2 -from pyspark.sql.streaming import StreamingQueryListener -from pyspark.sql.streaming.listener import QueryStartedEvent, QueryProgressEvent, QueryIdleEvent, QueryTerminatedEvent, StreamingQueryProgress +from pyspark.sql.streaming import StreamingQueryListener as StreamingQueryListener +from pyspark.sql.streaming.listener import QueryStartedEvent as QueryStartedEvent, QueryProgressEvent as QueryProgressEvent, QueryIdleEvent as QueryIdleEvent, QueryTerminatedEvent as QueryTerminatedEvent, StreamingQueryProgress from pyspark.sql.connect.session import SparkSession class StreamingQuery: diff --git a/python/pyspark/sql/connect/streaming/readwriter.py b/python/pyspark/sql/connect/streaming/readwriter.py index 1df9a85c..557d5442 100644 --- a/python/pyspark/sql/connect/streaming/readwriter.py +++ b/python/pyspark/sql/connect/streaming/readwriter.py @@ -14,10 +14,124 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""DataStreamReader / DataStreamWriter under the connect import path.""" + +""" +The streaming readers and writers of Spark Connect, as PySpark Connect's +``pyspark.sql.connect.streaming.readwriter``: the native classes of the Rust core, and its native +``Trigger``. ``DataStreamWriter.foreachBatch`` and ``.foreach`` take a Python callable and +cloudpickle it inside the extension (with ``pyspark.serializers.CloudPickleSerializer``); +``.foreach`` sends the function of ``_construct_foreach_function``, as PySpark does. +""" + +from collections.abc import Iterator +from typing import Any, Callable, Union, cast + # As PySpark Connect's module, which imports pyspark.sql.connect.readwriter: the official doctests # of the stream reader and writer take that module's namespace as their globals. import pyspark.sql.connect.readwriter # noqa: F401 -from pyspark.sql.streaming.readwriter import DataStreamReader, DataStreamWriter +from pyspark._pyspark import ( + DataStreamReader as _DataStreamReader, +) +from pyspark._pyspark import ( + DataStreamWriter as _DataStreamWriter, +) +from pyspark._pyspark import ( + Trigger as _Trigger, +) +from pyspark.errors import PySparkAttributeError, PySparkRuntimeError + +DataStreamReader = _DataStreamReader +DataStreamWriter = _DataStreamWriter +Trigger = _Trigger + + +def _construct_foreach_function( + f: Union[Callable[[Any], None], Any], +) -> Callable[[Any, Iterator], Iterator]: + """ + The function of the partition and the rows of each partition that ``foreach(f)`` sends, + which the Python worker of the server runs, as PySpark's + ``DataStreamWriter._construct_foreach_function``: for a function ``f``, one that calls ``f`` + with each row, and for an object, one that calls its ``open(partition_id, epoch_id)``, its + ``process(row)`` with each row, and its ``close(error)``. + + Ported from PySpark, but for ``TaskContext``, which the function imports on the worker. + """ + if callable(f): + # The provided object is a callable function that is supposed to be called on each row. + # Construct a function that takes an iterator and calls the provided function on each + # row. + def func_without_process(_: Any, iterator: Iterator) -> Iterator: + for x in iterator: + f(x) + return iter([]) + + return func_without_process + + else: + # The provided object is not a callable function. Then it is expected to have a + # 'process(row)' method, and optional 'open(partition_id, epoch_id)' and + # 'close(error)' methods. + + if not hasattr(f, "process"): + raise PySparkAttributeError( + errorClass="ATTRIBUTE_NOT_CALLABLE", + messageParameters={"attr_name": "process", "obj_name": "f"}, + ) + + if not callable(f.process): + raise PySparkAttributeError( + errorClass="ATTRIBUTE_NOT_CALLABLE", + messageParameters={"attr_name": "process", "obj_name": "f"}, + ) + + def doesMethodExist(method_name: str) -> bool: + exists = hasattr(f, method_name) + if exists and not callable(getattr(f, method_name)): + raise PySparkAttributeError( + errorClass="ATTRIBUTE_NOT_CALLABLE", + messageParameters={"attr_name": method_name, "obj_name": "f"}, + ) + return exists + + open_exists = doesMethodExist("open") + close_exists = doesMethodExist("close") + + def func_with_open_process_close(partition_id: Any, iterator: Iterator) -> Iterator: + from pyspark.taskcontext import TaskContext + + epoch_id = cast(TaskContext, TaskContext.get()).getLocalProperty( + "streaming.sql.batchId" + ) + if epoch_id: + int_epoch_id = int(epoch_id) + else: + raise PySparkRuntimeError( + errorClass="CANNOT_GET_BATCH_ID", + messageParameters={"obj_name": "TaskContext"}, + ) + + # Check if the data should be processed + should_process = True + if open_exists: + should_process = f.open(partition_id, int_epoch_id) + + error = None + + try: + if should_process: + for x in iterator: + f.process(x) + except Exception as ex: + error = ex + finally: + if close_exists: + f.close(error) + if error: + raise error + + return iter([]) + + return func_with_open_process_close __all__ = ["DataStreamReader", "DataStreamWriter"] diff --git a/python/pyspark/sql/connect/streaming/readwriter.pyi b/python/pyspark/sql/connect/streaming/readwriter.pyi index d1575d95..ef3432a8 100644 --- a/python/pyspark/sql/connect/streaming/readwriter.pyi +++ b/python/pyspark/sql/connect/streaming/readwriter.pyi @@ -1682,3 +1682,16 @@ class DataStreamWriter: +...---------+-----+ ... """ + +# Names of the drop-in's module that PySpark's does not have. +from pyspark.errors import PySparkAttributeError as PySparkAttributeError +from pyspark.errors import PySparkRuntimeError as PySparkRuntimeError +class Trigger: + @staticmethod + def processingTime(interval: str) -> Trigger: ... + @staticmethod + def once() -> Trigger: ... + @staticmethod + def availableNow() -> Trigger: ... + @staticmethod + def continuous(interval: str) -> Trigger: ... diff --git a/python/pyspark/sql/connect/udf.py b/python/pyspark/sql/connect/udf.py index ea4c9bb5..9b1cf073 100644 --- a/python/pyspark/sql/connect/udf.py +++ b/python/pyspark/sql/connect/udf.py @@ -14,9 +14,441 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""UserDefinedFunction / UDFRegistration under the connect import path, and the -``_create_udf`` with which ``pyspark.sql.pandas.functions`` creates pandas and Arrow UDFs, as -in PySpark Connect.""" -from pyspark.sql.udf import UDFRegistration, UserDefinedFunction, _create_udf # noqa: F401 + +""" +User-defined functions of Spark Connect, as PySpark Connect's ``pyspark.sql.connect.udf``: the +UDF objects of ``pyspark.sql.functions.udf`` and ``pandas_udf``, and the registration of +``spark.udf``. PySpark's ``pandas_udf`` (``pyspark.sql.pandas.functions``, vendored as it is) +makes its UDFs with ``_create_udf`` of this module on Spark Connect, as PySpark's does. PySpark's +classic ``pyspark.sql.udf``, vendored as it is too, has the classes of Spark Classic. +""" + +import functools +import sys +import warnings +from typing import Callable, Optional, Any + +from pyspark.errors import PySparkRuntimeError, PySparkTypeError +from pyspark.sql.types import DataType, StringType +# As PySpark Connect's, which checks the return type of a UDF for its eval type with the +# static method of PySpark's classic class (its PySparkUserDefinedFunction). +from pyspark.sql.udf import UserDefinedFunction as _PySparkUserDefinedFunction +# As PySpark's module, of whose namespace it is (see the stubs). +from pyspark.serializers import CloudPickleSerializer # noqa: F401 +from pyspark.util import PythonEvalType __all__ = ["UDFRegistration", "UserDefinedFunction"] + + +def _as_concrete_datatype(returnType: Any) -> DataType: + """Return a concrete, picklable pyspark ``DataType``. + + A DDL string (e.g. ``"double"``, ``"a int, b string"``) is parsed into its + concrete type with ``DataType.fromDDL``. This matters because the cloudpickled UDF + command must carry a real ``DataType`` (not a DDL string): the pandas/Arrow + worker pairs each result column with its spark type to build the Arrow batch, + so a bare string breaks it ("not enough values to unpack"). Mirrors how the + reference Connect client pickles the *parsed* output type. + """ + if isinstance(returnType, str): + return DataType.fromDDL(returnType) + return returnType + + +class UserDefinedFunction: + """ + User defined function in Python + + Notes + ----- + The constructor of this class is not supposed to be directly called. + Use :meth:`pyspark.sql.functions.udf` or :meth:`pyspark.sql.functions.pandas_udf` + to create this instance. + """ + + # The class of PySpark Connect, of pyspark.sql.connect.udf: a UDF that a function captures is + # pickled by reference to that name, which the Python worker of the server has as the Connect + # class. PySpark's classic pyspark.sql.udf.UserDefinedFunction is another class, as in PySpark. + + def __init__( + self, + func: Callable[..., Any], + returnType: Any = StringType(), + name: Optional[str] = None, + evalType: int = PythonEvalType.SQL_BATCHED_UDF, + deterministic: bool = True, + ): + # cloudpickle pickles this class by reference only when its module is imported. + import pyspark.sql.connect.udf # noqa: F401 + + # The checks of PySpark Connect's UserDefinedFunction. + if not callable(func): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "callable", + "arg_name": "func", + "arg_type": type(func).__name__, + }, + ) + + if not isinstance(returnType, (DataType, str)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "DataType or str", + "arg_name": "returnType", + "arg_type": type(returnType).__name__, + }, + ) + + if not isinstance(evalType, int): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "int", + "arg_name": "evalType", + "arg_type": type(evalType).__name__, + }, + ) + + self.func = func + self._returnType = returnType + self._returnType_placeholder: Optional[DataType] = None + self._name = name or ( + func.__name__ if hasattr(func, "__name__") else func.__class__.__name__ + ) + self.evalType = evalType + self.deterministic = deterministic + + @property + def returnType(self) -> DataType: + # As PySpark's, the return type is parsed once, and checked for the eval type each + # time. It is a concrete DataType, which the pickled command carries for the pandas + # and Arrow workers; a DDL string is parsed on the client. + if self._returnType_placeholder is None: + self._returnType_placeholder = _as_concrete_datatype(self._returnType) + + _PySparkUserDefinedFunction._check_return_type(self._returnType_placeholder, self.evalType) + return self._returnType_placeholder + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + """ + Call the UDF with Column arguments. + Returns a Column representing the UDF call. + """ + from pyspark import _pyspark + + # A UDF argument is ColumnOrName: as PySpark's, a bare string is a column name, of + # which pyfunc_make_udf makes a ColumnReference, and each keyword argument is a + # NamedArgumentExpression, after the positional ones. + cols = list(args) + named = list(kwargs.items()) + + # As PySpark's PythonUDF, which keeps the function and the return type object: each + # request that builds the plan converts the output type of that object and pickles + # (func, returnType) for the server-side Python worker with CloudPickleSerializer, as + # PySpark's `PythonUDF.to_plan` does. So the function is pickled as it is then, with + # the globals that it reads, and a StructType changed in place after the call is the + # type that is sent, and pickled, as it is then; a function that does not pickle raises + # the PicklingError of `CloudPickleSerializer.dumps` from the request. + return _pyspark.functions.pyfunc_make_udf( + self._name, + self.returnType, + self.evalType, + self.func, + "%d.%d" % sys.version_info[:2], + *cols, + deterministic=self.deterministic, + named_args=named, + ) + + # Verbatim from PySpark Connect (pyspark.sql.connect.udf.UserDefinedFunction). + # This function is for improving the online help system in the interactive interpreter. + # For example, the built-in help / pydoc.help. It wraps the UDF with the docstring and + # argument annotation. (See: SPARK-19161) + def _wrapped(self) -> Any: + """ + Wrap this udf with a function and attach docstring from func + """ + + # It is possible for a callable instance without __name__ attribute or/and + # __module__ attribute to be wrapped here. For example, functools.partial. In this case, + # we should avoid wrapping the attributes from the wrapped function to the wrapper + # function. So, we take out these attribute names from the default names to set and + # then manually assign it after being wrapped. + assignments = tuple( + a for a in functools.WRAPPER_ASSIGNMENTS if a != "__name__" and a != "__module__" + ) + + @functools.wraps(self.func, assigned=assignments) + def wrapper(*args: Any, **kwargs: Any) -> Any: + return self(*args, **kwargs) + + wrapper.__name__ = self._name + wrapper.__module__ = ( + self.func.__module__ + if hasattr(self.func, "__module__") + else self.func.__class__.__module__ + ) + + wrapper.func = self.func # type: ignore[attr-defined] + wrapper.returnType = self.returnType # type: ignore[attr-defined] + wrapper.evalType = self.evalType # type: ignore[attr-defined] + wrapper.deterministic = self.deterministic # type: ignore[attr-defined] + wrapper.asNondeterministic = functools.wraps( # type: ignore[attr-defined] + self.asNondeterministic + )(lambda: self.asNondeterministic()._wrapped()) + wrapper._unwrapped = self # type: ignore[attr-defined] + return wrapper # type: ignore[return-value] + + def asNondeterministic(self) -> "UserDefinedFunction": + """ + Updates UserDefinedFunction to nondeterministic. + + .. versionadded:: 3.4.0 + """ + self.deterministic = False + return self + + +def udf( + f: Optional[Any] = None, + returnType: Any = StringType(), + *, + useArrow: Optional[bool] = None, +) -> Any: + """ + Create a user-defined function (UDF). + + Parameters + ---------- + f : callable, optional + The Python function to wrap as a UDF. + returnType : DataType, optional + The return type of the UDF. Defaults to StringType(). + useArrow : bool, optional + Whether to use Arrow optimization. Defaults to None (auto-detect). + + Returns + ------- + function or callable + If f is provided, returns the function wrapper of the UDF, as PySpark does. If f is + None or a return type, returns a decorator. + + Examples + -------- + >>> from pyspark.sql import functions as F + >>> from pyspark.sql.types import IntegerType + >>> u = F.udf(lambda x: x + 1, IntegerType()) + >>> result = spark.range(3).select(u(F.col('id')).alias('inc_id')) + """ + # Verbatim from PySpark Connect (pyspark.sql.connect.functions.builtin.udf). + if f is None or isinstance(f, (str, DataType)): + # If DataType has been passed as a positional argument + # for decorator use it as a returnType + return_type = f or returnType + return functools.partial( + _create_py_udf, + returnType=return_type, + useArrow=useArrow, + ) + else: + return _create_py_udf(f=f, returnType=returnType, useArrow=useArrow) + + +# Verbatim from PySpark Connect (pyspark.sql.connect.udf). +def _create_py_udf( + f: Callable[..., Any], + returnType: Any, + useArrow: Optional[bool] = None, +) -> Any: + from pyspark.sql.pandas.utils import ( + require_minimum_pandas_version, + require_minimum_pyarrow_version, + ) + + is_arrow_enabled = False + if useArrow is None: + is_arrow_enabled = False + try: + from pyspark.sql import SparkSession + + session = SparkSession.active() + is_arrow_enabled = ( + str(session.conf.get("spark.sql.execution.pythonUDF.arrow.enabled")).lower() + == "true" + ) + except PySparkRuntimeError as e: + if e.getCondition() == "NO_ACTIVE_OR_DEFAULT_SESSION": + pass # Just uses the default if no session found. + else: + raise e + else: + is_arrow_enabled = useArrow + + if is_arrow_enabled: + try: + require_minimum_pandas_version() + require_minimum_pyarrow_version() + except ImportError: + is_arrow_enabled = False + warnings.warn( + "Arrow optimization failed to enable because PyArrow or Pandas is not installed. " + "Falling back to a non-Arrow-optimized UDF.", + RuntimeWarning, + ) + + eval_type: Optional[int] = None + if useArrow is None: + # If the user doesn't explicitly set useArrow + from pyspark.sql.pandas.typehints import infer_eval_type_for_udf + + try: + # Try to infer the eval type from type hints + eval_type = infer_eval_type_for_udf(f) + except Exception: + warnings.warn("Cannot infer the eval type from type hints. ", UserWarning) + + if eval_type is None: + if is_arrow_enabled: + # Arrow optimized Python UDF + eval_type = PythonEvalType.SQL_ARROW_BATCHED_UDF + else: + # Fallback to Regular Python UDF + eval_type = PythonEvalType.SQL_BATCHED_UDF + + return _create_udf(f, returnType, eval_type) + + +# Verbatim from PySpark Connect (pyspark.sql.connect.udf); pyspark.sql.pandas.functions +# creates pandas and Arrow UDFs with it. +def _create_udf( + f: Callable[..., Any], + returnType: Any, + evalType: int, + name: Optional[str] = None, + deterministic: bool = True, +) -> Any: + # Set the name of the UserDefinedFunction object to be the name of function f + udf_obj = UserDefinedFunction( + f, returnType=returnType, name=name, evalType=evalType, deterministic=deterministic + ) + return udf_obj._wrapped() + + +class UDFRegistration: + """ + Wrapper for user-defined function registration. + """ + + # The class of `spark.udf`, as PySpark Connect's. + + def __init__(self, sparkSession: Any): + self.sparkSession = sparkSession + + def register( + self, + name: str, + f: Callable[..., Any], + returnType: Optional[DataType] = None, + ) -> Any: + """ + Register a Python UDF with the given name. + + Parameters + ---------- + name : str + Name to register the UDF with. + f : callable + The Python function to register. + returnType : DataType, optional + The return type of the UDF. Defaults to StringType(). + + Returns + ------- + function + The registered UDF that can be called with columns: ``f`` if it is a UDF, + or the function wrapper of the UDF of ``f``, as PySpark returns. + + Examples + -------- + >>> from pyspark.sql.types import IntegerType + >>> spark.udf.register("inc_id", lambda x: x + 1, IntegerType()) + >>> result = spark.sql("SELECT inc_id(id) AS incremented FROM range(3)") + """ + # As PySpark Connect's: a UDF (from udf() or pandas_udf()) is registered with its + # eval type, and a plain Python function as a SQL_BATCHED_UDF of `name`. The + # registration is a RegisterFunction command carrying the cloudpickled PythonUDF + # with no bound arguments, as the reference client.register_udf sends, so `name` + # resolves in SQL. + if hasattr(f, "asNondeterministic"): + if returnType is not None: + raise PySparkTypeError( + errorClass="CANNOT_SPECIFY_RETURN_TYPE_FOR_UDF", + messageParameters={"arg_name": "f", "return_type": str(returnType)}, + ) + if f.evalType not in [ + PythonEvalType.SQL_BATCHED_UDF, + PythonEvalType.SQL_ARROW_BATCHED_UDF, + PythonEvalType.SQL_SCALAR_PANDAS_UDF, + PythonEvalType.SQL_SCALAR_ARROW_UDF, + PythonEvalType.SQL_SCALAR_PANDAS_ITER_UDF, + PythonEvalType.SQL_SCALAR_ARROW_ITER_UDF, + PythonEvalType.SQL_GROUPED_AGG_PANDAS_UDF, + PythonEvalType.SQL_GROUPED_AGG_ARROW_UDF, + PythonEvalType.SQL_GROUPED_AGG_PANDAS_ITER_UDF, + PythonEvalType.SQL_GROUPED_AGG_ARROW_ITER_UDF, + ]: + raise PySparkTypeError( + errorClass="INVALID_UDF_EVAL_TYPE", + messageParameters={ + "eval_type": "SQL_BATCHED_UDF, SQL_ARROW_BATCHED_UDF, " + "SQL_SCALAR_PANDAS_UDF, SQL_SCALAR_ARROW_UDF, " + "SQL_SCALAR_PANDAS_ITER_UDF, SQL_SCALAR_ARROW_ITER_UDF, " + "SQL_GROUPED_AGG_PANDAS_UDF, SQL_GROUPED_AGG_ARROW_UDF, " + "SQL_GROUPED_AGG_PANDAS_ITER_UDF or SQL_GROUPED_AGG_ARROW_ITER_UDF" + }, + ) + self.sparkSession._client.register_udf( + f.func, f.returnType, name, f.evalType, f.deterministic + ) + return f + else: + if returnType is None: + returnType = StringType() + py_udf = _create_udf( + f, returnType=returnType, evalType=PythonEvalType.SQL_BATCHED_UDF, name=name + ) + + # As PySpark's: the registration takes the return type as given, not the one that + # the wrapper parsed, so a DDL string is parsed again for the command. + self.sparkSession._client.register_udf(py_udf.func, returnType, name) + return py_udf + + def registerJavaFunction( + self, + name: str, + javaClassName: str, + returnType: Optional[DataType] = None, + ) -> None: + """Register a Java UDF by fully-qualified class name so it is callable by + ``name`` in SQL. Mirrors ``UDFRegistration.registerJavaFunction``. + + Parameters + ---------- + name : str + Name to register the function with. + javaClassName : str + Fully-qualified name of the Java class implementing the UDF. + returnType : DataType or str, optional + Return type: a DataType, or a DDL string, which the server parses, as PySpark + sends it. Defaults to the class's own type. + """ + self.sparkSession._client.register_java(name, javaClassName, returnType) + + def registerJavaUDAF(self, name: str, javaClassName: str) -> None: + """Register a Java user-defined aggregate function (UDAF) by class name. + Mirrors ``UDFRegistration.registerJavaUDAF``. + """ + self.sparkSession._client.register_java(name, javaClassName, aggregate=True) diff --git a/python/pyspark/sql/connect/udf.pyi b/python/pyspark/sql/connect/udf.pyi index 42497afc..28e9e968 100644 --- a/python/pyspark/sql/connect/udf.pyi +++ b/python/pyspark/sql/connect/udf.pyi @@ -20,15 +20,16 @@ """User-defined function related classes and functions""" from typing import Callable, Any, Optional, Union -from pyspark.util import PythonEvalType +from pyspark.util import PythonEvalType as PythonEvalType from pyspark.sql.connect.expressions import CommonInlineUserDefinedFunction from pyspark.sql.connect.column import Column -from pyspark.sql.types import DataType, StringType, _parse_datatype_string +from pyspark.sql.types import DataType as DataType, StringType as StringType, _parse_datatype_string from pyspark.sql.udf import UserDefinedFunction as PySparkUserDefinedFunction -from pyspark.errors import PySparkTypeError +from pyspark.errors import PySparkTypeError as PySparkTypeError, PySparkRuntimeError as PySparkRuntimeError from pyspark.sql.connect._typing import ColumnOrName, DataTypeOrString, UserDefinedFunctionLike from pyspark.sql.connect.session import SparkSession +def _create_py_udf(f: Callable[..., Any], returnType: 'DataTypeOrString', useArrow: Optional[bool]=None) -> 'UserDefinedFunctionLike': ... def _create_udf(f: Callable[..., Any], returnType: 'DataTypeOrString', evalType: int, name: Optional[str]=None, deterministic: bool=True) -> 'UserDefinedFunctionLike': ... class UserDefinedFunction: @@ -269,3 +270,7 @@ class UDFRegistration: >>> spark.sql(q).collect() # doctest: +SKIP [Row(name='b', avg=102.0), Row(name='a', avg=102.0)] """ + +# Names of the drop-in's module that PySpark's does not have. +from pyspark.serializers import CloudPickleSerializer as CloudPickleSerializer +from pyspark.sql.functions.builtin import udf as udf diff --git a/python/pyspark/sql/connect/udtf.py b/python/pyspark/sql/connect/udtf.py index a7034ca7..6fed1b0c 100644 --- a/python/pyspark/sql/connect/udtf.py +++ b/python/pyspark/sql/connect/udtf.py @@ -14,12 +14,303 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""User-defined table function classes under the connect import path.""" -from pyspark.sql.udtf import ( - AnalyzeArgument, - AnalyzeResult, - UDTFRegistration, - UserDefinedTableFunction, -) + +""" +User-defined table functions of Spark Connect, as PySpark Connect's ``pyspark.sql.connect.udtf``: +the UDTF objects of ``pyspark.sql.functions.udtf`` and ``arrow_udtf``, and the registration of +``spark.udtf``. PySpark's classic ``pyspark.sql.udtf``, vendored as it is, holds the dataclasses of +``analyze`` that this module takes. +""" + +import functools +import pickle +import sys +from typing import Any, Optional, Type, Union + +from pyspark.errors import PySparkAttributeError, PySparkPicklingError, PySparkTypeError +from pyspark.serializers import CloudPickleSerializer +from pyspark.sql.types import StructType + +# PySpark's classic dataclasses of the analyze method of a UDTF and its validation of a handler, +# as PySpark Connect's module imports them: a handler that the client pickles may capture them, +# and the Python worker of the server unpickles them as PySpark's. +from pyspark.sql.udtf import AnalyzeArgument, AnalyzeResult, _validate_udtf_handler __all__ = ["AnalyzeArgument", "AnalyzeResult", "UDTFRegistration", "UserDefinedTableFunction"] + +# Eval types for Python UDTFs (mirrors pyspark.util.PythonEvalType). +SQL_TABLE_UDF = 300 +SQL_ARROW_TABLE_UDF = 301 +SQL_ARROW_UDTF = 302 + + +# Verbatim from PySpark Connect (pyspark.sql.connect.udtf): validates the handler class of a +# PyArrow UDTF. +def _validate_arrow_udtf_handler(cls: Any, returnType: Optional[Any]) -> None: + """Validate the handler class of a PyArrow UDTF.""" + # First run standard UDTF validation + _validate_udtf_handler(cls, returnType) + + # Block analyze method usage in arrow UDTFs + # TODO(SPARK-53286): Support analyze method for Arrow UDTFs to enable dynamic return types + has_analyze = hasattr(cls, "analyze") + if has_analyze: + raise PySparkAttributeError( + errorClass="INVALID_ARROW_UDTF_WITH_ANALYZE", + messageParameters={"name": cls.__name__}, + ) + + +def _python_ver() -> str: + """The Python version of the client, ``major.minor``, as PySpark's ``get_python_ver``, which + the plan of a UDTF carries.""" + return f"{sys.version_info.major}.{sys.version_info.minor}" + + +def _pickled_handler(func: Type, name: Optional[str] = None) -> bytes: + """The command of a UDTF of the handler class ``func``, as PySpark Connect's + ``PythonUDTF.to_plan`` pickles it each time it builds a plan that holds the UDTF, and its + registration when it registers it: ``UDTF_SERIALIZATION_ERROR`` of the name of the class, of + a handler that does not pickle, which is ``name``, the name that the ``PythonUDTF`` of a call + read at the call, or else the name of the class now. Connect sends the schema separately, in + ``PythonUDTF.return_type``, so the command is only the handler class, unlike that of a scalar + UDF.""" + try: + return CloudPickleSerializer().dumps(func) + except pickle.PicklingError: + raise PySparkPicklingError( + errorClass="UDTF_SERIALIZATION_ERROR", + messageParameters={ + "name": func.__name__ if name is None else name, + "message": "Please check the stack trace and " + "make sure the function is serializable.", + }, + ) + + +def _udtf_command(func: Type, name: str, returnType: Optional[Any]) -> bytes: + """The ``PythonUDTF`` message of the return type and the command of a call of a UDTF, as + PySpark Connect's ``PythonUDTF.to_plan`` builds them each time it builds a plan that holds the + call, of what the ``PythonUDTF`` of the call keeps: the handler class ``func``, its ``name`` + then and the return type object ``returnType``. It converts the return type first, of the + object as it is now, with ``pyspark_types_to_proto_types``: so a ``StructType`` changed in + place after the call is the type that the request sends, and the error of a type that does not + convert, such as ``UNSUPPORTED_OPERATION`` of a value that is not a DataType, is raised before + the handler is pickled. Then it pickles the handler (see ``_pickled_handler``). The request + sends this type and this command, with the eval type and the Python version of the call.""" + import pyspark.sql.connect.proto as pb2 + from pyspark.sql.connect.types import pyspark_types_to_proto_types + + udtf = pb2.PythonUDTF() + if returnType is not None: + udtf.return_type.CopyFrom(pyspark_types_to_proto_types(returnType)) + udtf.command = _pickled_handler(func, name) + return udtf.SerializeToString() + + +class UserDefinedTableFunction: + """ + User defined function in Python + + Notes + ----- + The constructor of this class is not supposed to be directly called. + Use :meth:`pyspark.sql.functions.udtf` to create this instance. + """ + + # A Python handler class that yields rows: calling the UDTF with Column arguments returns a + # DataFrame of the CommonInlineUserDefinedTableFunction relation, as PySpark Connect's. + + def __init__( + self, + func: Type, + returnType: Optional[Union[StructType, str]], + name: Optional[str] = None, + evalType: int = SQL_TABLE_UDF, + deterministic: bool = False, + ) -> None: + # cloudpickle pickles this class by reference only when its module is imported. + import pyspark.sql.connect.udtf # noqa: F401 + + # As PySpark's UserDefinedTableFunction, for udtf(), arrow_udtf() and register(), whose + # returnType has no default, and which sets its attributes in this order, as vars() shows + # them. A returnType that is neither a DataType nor a str is kept as it is, as PySpark + # converts it when it builds a plan of a call. + _validate_udtf_handler(func, returnType) + self.func = func + # As PySpark Connect's: a DDL string is a type that the server parses. + if isinstance(returnType, str): + from pyspark.sql.connect.types import UnparsedDataType + + returnType = UnparsedDataType(returnType) + self.returnType = returnType + self._name = name or func.__name__ + self.evalType = evalType + self.deterministic = deterministic + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + from pyspark import _pyspark + from pyspark.sql import SparkSession + from pyspark.sql.connect.types import UnparsedDataType + + # As PySpark's: an argument is ColumnOrName, so a bare string is a column name, of + # which pyfunc_make_udtf makes a ColumnReference, and each keyword argument is a + # NamedArgumentExpression, after the positional ones. As its `PythonUDTF`, made at the + # call, the plan keeps the handler class, its name and the return type object as they + # are now, a str as an UnparsedDataType, so a returnType or a func set after the call is + # not that of the call. As its `PythonUDTF.to_plan`, each request converts that type, as + # the object is then, and pickles the handler class when it builds the plan of the + # DataFrame, after the arguments: `_udtf_command` gives both, so the call passes no type. + cols = list(args) + named = list(kwargs.items()) + returnType = self.returnType + if isinstance(returnType, str): + returnType = UnparsedDataType(returnType) + return _pyspark.functions.pyfunc_make_udtf( + SparkSession.active(), + self._name, + None, + self.evalType, + functools.partial(_udtf_command, self.func, self.func.__name__, returnType), + _python_ver(), + *cols, + deterministic=self.deterministic, + named_args=named, + ) + + def asDeterministic(self) -> "UserDefinedTableFunction": + """Mark this UDTF as deterministic, and return it. Mirrors + ``UserDefinedTableFunction.asDeterministic``.""" + self.deterministic = True + return self + + +def udtf( + cls: Optional[Type] = None, + *, + returnType: Optional[Any] = None, + useArrow: Optional[bool] = None, +) -> Any: + """Create a user-defined table function (UDTF). + + Use as ``@udtf(returnType=...)`` on a handler class, or call + ``udtf(MyClass, returnType=...)`` directly. As PySpark's ``_create_py_udtf``, the UDTF is + Arrow-optimized when ``useArrow`` is True, or is not given and + ``spark.sql.execution.pythonUDTF.arrow.enabled`` of the active session is true. + """ + + def _udtf_decorator(handler: Type) -> UserDefinedTableFunction: + if useArrow is not None: + arrow_enabled = useArrow + else: + from pyspark.errors import PySparkRuntimeError + from pyspark.sql import SparkSession + + arrow_enabled = False + try: + session = SparkSession.active() + arrow_enabled = ( + str(session.conf.get("spark.sql.execution.pythonUDTF.arrow.enabled")).lower() + == "true" + ) + except PySparkRuntimeError as e: + if e.getCondition() == "NO_ACTIVE_OR_DEFAULT_SESSION": + pass # Just uses the default if no session found. + else: + raise e + eval_type = SQL_TABLE_UDF + if arrow_enabled: + try: + from pyspark.sql.pandas.utils import ( + require_minimum_pandas_version, + require_minimum_pyarrow_version, + ) + + require_minimum_pandas_version() + require_minimum_pyarrow_version() + eval_type = SQL_ARROW_TABLE_UDF + except ImportError as e: + import warnings + + warnings.warn( + f"Arrow optimization for Python UDTFs cannot be enabled: {str(e)}. " + f"Falling back to using regular Python UDTFs.", + UserWarning, + ) + return UserDefinedTableFunction(handler, returnType=returnType, evalType=eval_type) + + if cls is not None: + return _udtf_decorator(cls) + return _udtf_decorator + + +def arrow_udtf( + cls: Optional[Type] = None, + *, + returnType: Optional[Any] = None, +) -> Any: + """Create an Arrow-based user-defined table function (UDTF). + + Mirrors ``pyspark.sql.functions.arrow_udtf``: a PyArrow-native UDTF (``SQL_ARROW_UDTF``), + whose ``eval`` takes ``pyarrow`` arrays and returns ``pyarrow`` tables. + """ + def _arrow_udtf_decorator(handler: Type) -> UserDefinedTableFunction: + from pyspark.sql.pandas.utils import require_minimum_pyarrow_version + + require_minimum_pyarrow_version() + _validate_arrow_udtf_handler(handler, returnType) + return UserDefinedTableFunction(handler, returnType=returnType, evalType=SQL_ARROW_UDTF) + + if cls is not None: + return _arrow_udtf_decorator(cls) + return _arrow_udtf_decorator + + +class UDTFRegistration: + """ + Wrapper for user-defined table function registration. + + .. versionadded:: 3.5.0 + """ + + # The class of `spark.udtf`, as PySpark Connect's. + + def __init__(self, sparkSession: Any): + self.sparkSession = sparkSession + + def register( + self, + name: str, + f: "UserDefinedTableFunction", + ) -> "UserDefinedTableFunction": + """Register the UDTF ``f`` as ``name`` on the server, so that SQL calls it, and return + ``f``. As PySpark's, ``f`` is a UDTF of ``udtf()`` or ``arrow_udtf()``.""" + if not isinstance(f, UserDefinedTableFunction): + raise PySparkTypeError( + errorClass="CANNOT_REGISTER_UDTF", + messageParameters={ + "name": name, + }, + ) + + if f.evalType not in [SQL_TABLE_UDF, SQL_ARROW_TABLE_UDF, SQL_ARROW_UDTF]: + raise PySparkTypeError( + errorClass="INVALID_UDTF_EVAL_TYPE", + messageParameters={ + "name": name, + "eval_type": "SQL_TABLE_UDF, SQL_ARROW_TABLE_UDF, SQL_ARROW_UDTF", + }, + ) + + # As PySpark Connect's `register_udtf`, which takes the Python version of the client, + # which a UDTF does not keep, and sets the name and converts the return type before it + # pickles the handler class: `_registerPythonUdtf` calls the pickling after them. + self.sparkSession._registerPythonUdtf( + name, + f.returnType, + f.evalType, + functools.partial(_pickled_handler, f.func), + _python_ver(), + f.deterministic, + ) + return f diff --git a/python/pyspark/sql/connect/udtf.pyi b/python/pyspark/sql/connect/udtf.pyi index addb040e..fdb199f3 100644 --- a/python/pyspark/sql/connect/udtf.pyi +++ b/python/pyspark/sql/connect/udtf.pyi @@ -19,17 +19,21 @@ """User-defined table function related classes and functions""" -from typing import Type, Optional, Union +from typing import Type, Optional, Union, Any from pyspark.util import PythonEvalType from pyspark.sql.connect.plan import CommonInlineUserDefinedTableFunction from pyspark.sql.connect.types import UnparsedDataType from pyspark.sql.udtf import AnalyzeArgument as AnalyzeArgument, AnalyzeResult as AnalyzeResult from pyspark.sql.udtf import _validate_udtf_handler -from pyspark.sql.types import DataType, StructType +from pyspark.sql.types import DataType, StructType as StructType +from pyspark.errors import PySparkTypeError as PySparkTypeError, PySparkAttributeError as PySparkAttributeError from pyspark.sql.connect._typing import ColumnOrName from pyspark.sql.connect.dataframe import DataFrame from pyspark.sql.connect.session import SparkSession +def _validate_arrow_udtf_handler(cls: Any, returnType: Optional[Union[StructType, str]]) -> None: + """Validate the handler class of a PyArrow UDTF.""" + class UserDefinedTableFunction: """User defined function in Python @@ -107,3 +111,12 @@ class UDTFRegistration: >>> spark.sql("SELECT * FROM VALUES (0, 1), (1, 2) t(x, y), LATERAL plus_one(x)").collect() [Row(x=0, y=1, c1=0, c2=1), Row(x=1, y=2, c1=1, c2=2)] """ + +# Names of the drop-in's module that PySpark's does not have. +from pyspark.errors import PySparkPicklingError as PySparkPicklingError +from pyspark.serializers import CloudPickleSerializer as CloudPickleSerializer +from pyspark.sql.functions.builtin import arrow_udtf as arrow_udtf +from pyspark.sql.functions.builtin import udtf as udtf +SQL_TABLE_UDF: int +SQL_ARROW_TABLE_UDF: int +SQL_ARROW_UDTF: int diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py index 87540c59..ba15f48b 100644 --- a/python/pyspark/sql/functions.py +++ b/python/pyspark/sql/functions.py @@ -315,14 +315,13 @@ def _dict_to_options_column(options_dict): return _call_function("create_map", *[_pyfunc_lit(arg) for arg in args]) # Import UDF / UDTF functions +from pyspark.sql.connect.udf import udf +from pyspark.sql.connect.udtf import arrow_udtf, udtf from pyspark.sql.pandas.functions import arrow_udf, pandas_udf -from pyspark.sql.udf import udf from pyspark.sql.udtf import ( OrderingColumn, PartitioningColumn, SelectedColumn, - arrow_udtf, - udtf, ) @@ -1332,7 +1331,7 @@ def bucket(numBuckets, col): not _bname.startswith("_") and _inspect.isfunction(_bobj) and _bobj.__module__ - in ("functions_generated", __name__, "pyspark.sql.udf", "pyspark.sql.udtf") + in ("functions_generated", __name__, "pyspark.sql.connect.udf", "pyspark.sql.connect.udtf") and getattr(builtin, _bobj.__qualname__, None) is _bobj ): _bobj.__module__ = builtin.__name__ diff --git a/python/pyspark/sql/group.py b/python/pyspark/sql/group.py index 5d38e596..221105d9 100644 --- a/python/pyspark/sql/group.py +++ b/python/pyspark/sql/group.py @@ -14,7 +14,547 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""Re-export of the Rust-backed class(es) under the official pyspark.sql import path.""" -from pyspark._pyspark import GroupedData + +# mypy: disable-error-code="empty-body" + +import sys + +from typing import Callable, List, Optional, TYPE_CHECKING, overload, Dict, Union, cast, Tuple + +from pyspark.sql.column import Column +from pyspark.sql.session import SparkSession +from pyspark.sql.dataframe import DataFrame +from pyspark.sql.pandas.group_ops import PandasGroupedOpsMixin + +if TYPE_CHECKING: + from py4j.java_gateway import JavaObject + from pyspark.sql._typing import LiteralType __all__ = ["GroupedData"] + + +def dfapi(f: Callable[..., DataFrame]) -> Callable[..., DataFrame]: + def _api(self: "GroupedData") -> "DataFrame": + name = f.__name__ + jdf = getattr(self._jgd, name)() + return DataFrame(jdf, self.session) + + _api.__name__ = f.__name__ + _api.__doc__ = f.__doc__ + return _api + + +def df_varargs_api(f: Callable[..., DataFrame]) -> Callable[..., DataFrame]: + def _api(self: "GroupedData", *cols: str) -> "DataFrame": + from pyspark.sql.classic.column import _to_seq + + name = f.__name__ + jdf = getattr(self._jgd, name)(_to_seq(self.session._sc, cols)) + return DataFrame(jdf, self.session) + + _api.__name__ = f.__name__ + _api.__doc__ = f.__doc__ + return _api + + +class GroupedData(PandasGroupedOpsMixin): + """ + A set of methods for aggregations on a :class:`DataFrame`, + created by :func:`DataFrame.groupBy`. + + .. versionadded:: 1.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + """ + + def __init__(self, jgd: "JavaObject", df: DataFrame): + self._jgd = jgd + self._df = df + self.session: SparkSession = df.sparkSession + + def __repr__(self) -> str: + index = 26 # index to truncate string from the JVM side + jvm_string = self._jgd.toString() + if jvm_string is not None and len(jvm_string) > index and jvm_string[index] == "[": + return f"GroupedData{jvm_string[index:]}" + else: + return super().__repr__() + + @overload + def agg(self, *exprs: Column) -> "DataFrame": ... + + @overload + def agg(self, __exprs: Dict[str, str]) -> "DataFrame": ... + + def agg(self, *exprs: Union[Column, Dict[str, str]]) -> "DataFrame": + """Compute aggregates and returns the result as a :class:`DataFrame`. + + The available aggregate functions can be: + + 1. built-in aggregation functions, such as `avg`, `max`, `min`, `sum`, `count` + + 2. group aggregate pandas UDFs, created with :func:`pyspark.sql.functions.pandas_udf` + + .. note:: There is no partial aggregation with group aggregate UDFs, i.e., + a full shuffle is required. Also, all the data of a group will be loaded into + memory, so the user should be aware of the potential OOM risk if data is skewed + and certain groups are too large to fit in memory. + + .. seealso:: :func:`pyspark.sql.functions.pandas_udf` + + If ``exprs`` is a single :class:`dict` mapping from string to string, then the key + is the column to perform aggregation on, and the value is the aggregate function. + + Alternatively, ``exprs`` can also be a list of aggregate :class:`Column` expressions. + + .. versionadded:: 1.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + exprs : dict + a dict mapping from column name (string) to aggregate functions (string), + or a list of :class:`Column`. + + Notes + ----- + Built-in aggregation functions and group aggregate pandas UDFs cannot be mixed + in a single call to this function. + + Examples + -------- + >>> import pandas as pd + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [(2, "Alice"), (3, "Alice"), (5, "Bob"), (10, "Bob")], ["age", "name"]) + >>> df.show() + +---+-----+ + |age| name| + +---+-----+ + | 2|Alice| + | 3|Alice| + | 5| Bob| + | 10| Bob| + +---+-----+ + + Group-by name, and count each group. + + >>> df.groupBy(df.name) + GroupedData[grouping...: [name...], value: [age: bigint, name: string], type: GroupBy] + + >>> df.groupBy(df.name).agg({"*": "count"}).sort("name").show() + +-----+--------+ + | name|count(1)| + +-----+--------+ + |Alice| 2| + | Bob| 2| + +-----+--------+ + + Group-by name, and calculate the minimum age. + + >>> df.groupBy(df.name).agg(sf.min(df.age)).sort("name").show() + +-----+--------+ + | name|min(age)| + +-----+--------+ + |Alice| 2| + | Bob| 5| + +-----+--------+ + + Same as above but uses pandas UDF. + + >>> from pyspark.sql.functions import pandas_udf + >>> @pandas_udf('int') + ... def min_udf(v: pd.Series) -> int: + ... return v.min() + ... + >>> df.groupBy(df.name).agg(min_udf(df.age)).sort("name").show() + +-----+------------+ + | name|min_udf(age)| + +-----+------------+ + |Alice| 2| + | Bob| 5| + +-----+------------+ + """ + from pyspark.sql.classic.column import _to_seq + + assert exprs, "exprs should not be empty" + if len(exprs) == 1 and isinstance(exprs[0], dict): + jdf = self._jgd.agg(exprs[0]) + else: + # Columns + assert all(isinstance(c, Column) for c in exprs), "all exprs should be Column" + exprs = cast(Tuple[Column, ...], exprs) + jdf = self._jgd.agg(exprs[0]._jc, _to_seq(self.session._sc, [c._jc for c in exprs[1:]])) + return DataFrame(jdf, self.session) + + @dfapi + def count(self) -> "DataFrame": + """Counts the number of records for each group. + + .. versionadded:: 1.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Examples + -------- + >>> df = spark.createDataFrame( + ... [(2, "Alice"), (3, "Alice"), (5, "Bob"), (10, "Bob")], ["age", "name"]) + >>> df.show() + +---+-----+ + |age| name| + +---+-----+ + | 2|Alice| + | 3|Alice| + | 5| Bob| + | 10| Bob| + +---+-----+ + + Group-by name, and count each group. + + >>> df.groupBy(df.name).count().sort("name").show() + +-----+-----+ + | name|count| + +-----+-----+ + |Alice| 2| + | Bob| 2| + +-----+-----+ + """ + + @df_varargs_api + def mean(self, *cols: str) -> DataFrame: + """Computes average values for each numeric columns for each group. + + :func:`mean` is an alias for :func:`avg`. + + .. versionadded:: 1.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + cols : str + column names. Non-numeric columns are ignored. + """ + + @df_varargs_api + def avg(self, *cols: str) -> "DataFrame": + """Computes average values for each numeric columns for each group. + + :func:`mean` is an alias for :func:`avg`. + + .. versionadded:: 1.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + cols : str + column names. Non-numeric columns are ignored. + + Examples + -------- + >>> df = spark.createDataFrame([ + ... (2, "Alice", 80), (3, "Alice", 100), + ... (5, "Bob", 120), (10, "Bob", 140)], ["age", "name", "height"]) + >>> df.show() + +---+-----+------+ + |age| name|height| + +---+-----+------+ + | 2|Alice| 80| + | 3|Alice| 100| + | 5| Bob| 120| + | 10| Bob| 140| + +---+-----+------+ + + Group-by name, and calculate the mean of the age in each group. + + >>> df.groupBy("name").avg('age').sort("name").show() + +-----+--------+ + | name|avg(age)| + +-----+--------+ + |Alice| 2.5| + | Bob| 7.5| + +-----+--------+ + + Calculate the mean of the age and height in all data. + + >>> df.groupBy().avg('age', 'height').show() + +--------+-----------+ + |avg(age)|avg(height)| + +--------+-----------+ + | 5.0| 110.0| + +--------+-----------+ + """ + + @df_varargs_api + def max(self, *cols: str) -> "DataFrame": + """Computes the max value for each numeric columns for each group. + + .. versionadded:: 1.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Examples + -------- + >>> df = spark.createDataFrame([ + ... (2, "Alice", 80), (3, "Alice", 100), + ... (5, "Bob", 120), (10, "Bob", 140)], ["age", "name", "height"]) + >>> df.show() + +---+-----+------+ + |age| name|height| + +---+-----+------+ + | 2|Alice| 80| + | 3|Alice| 100| + | 5| Bob| 120| + | 10| Bob| 140| + +---+-----+------+ + + Group-by name, and calculate the max of the age in each group. + + >>> df.groupBy("name").max("age").sort("name").show() + +-----+--------+ + | name|max(age)| + +-----+--------+ + |Alice| 3| + | Bob| 10| + +-----+--------+ + + Calculate the max of the age and height in all data. + + >>> df.groupBy().max("age", "height").show() + +--------+-----------+ + |max(age)|max(height)| + +--------+-----------+ + | 10| 140| + +--------+-----------+ + """ + + @df_varargs_api + def min(self, *cols: str) -> "DataFrame": + """Computes the min value for each numeric column for each group. + + .. versionadded:: 1.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + cols : str + column names. Non-numeric columns are ignored. + + Examples + -------- + >>> df = spark.createDataFrame([ + ... (2, "Alice", 80), (3, "Alice", 100), + ... (5, "Bob", 120), (10, "Bob", 140)], ["age", "name", "height"]) + >>> df.show() + +---+-----+------+ + |age| name|height| + +---+-----+------+ + | 2|Alice| 80| + | 3|Alice| 100| + | 5| Bob| 120| + | 10| Bob| 140| + +---+-----+------+ + + Group-by name, and calculate the min of the age in each group. + + >>> df.groupBy("name").min("age").sort("name").show() + +-----+--------+ + | name|min(age)| + +-----+--------+ + |Alice| 2| + | Bob| 5| + +-----+--------+ + + Calculate the min of the age and height in all data. + + >>> df.groupBy().min("age", "height").show() + +--------+-----------+ + |min(age)|min(height)| + +--------+-----------+ + | 2| 80| + +--------+-----------+ + """ + + @df_varargs_api + def sum(self, *cols: str) -> DataFrame: + """Computes the sum for each numeric columns for each group. + + .. versionadded:: 1.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + cols : str + column names. Non-numeric columns are ignored. + + Examples + -------- + >>> df = spark.createDataFrame([ + ... (2, "Alice", 80), (3, "Alice", 100), + ... (5, "Bob", 120), (10, "Bob", 140)], ["age", "name", "height"]) + >>> df.show() + +---+-----+------+ + |age| name|height| + +---+-----+------+ + | 2|Alice| 80| + | 3|Alice| 100| + | 5| Bob| 120| + | 10| Bob| 140| + +---+-----+------+ + + Group-by name, and calculate the sum of the age in each group. + + >>> df.groupBy("name").sum("age").sort("name").show() + +-----+--------+ + | name|sum(age)| + +-----+--------+ + |Alice| 5| + | Bob| 15| + +-----+--------+ + + Calculate the sum of the age and height in all data. + + >>> df.groupBy().sum("age", "height").show() + +--------+-----------+ + |sum(age)|sum(height)| + +--------+-----------+ + | 20| 440| + +--------+-----------+ + """ + + def pivot(self, pivot_col: str, values: Optional[List["LiteralType"]] = None) -> "GroupedData": + """ + Pivots a column of the current :class:`DataFrame` and performs the specified aggregation. + + .. versionadded:: 1.6.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + pivot_col : str + Name of the column to pivot. + values : list, optional + List of values that will be translated to columns in the output DataFrame. + + If ``values`` is not provided, Spark will eagerly compute the distinct values in + ``pivot_col`` so it can determine the resulting schema of the transformation. To avoid + any eager computations, provide an explicit list of values. + + Examples + -------- + >>> from pyspark.sql import Row, functions as sf + >>> df1 = spark.createDataFrame([ + ... Row(course="dotNET", year=2012, earnings=10000), + ... Row(course="Java", year=2012, earnings=20000), + ... Row(course="dotNET", year=2012, earnings=5000), + ... Row(course="dotNET", year=2013, earnings=48000), + ... Row(course="Java", year=2013, earnings=30000), + ... ]) + >>> df1.show() + +------+----+--------+ + |course|year|earnings| + +------+----+--------+ + |dotNET|2012| 10000| + | Java|2012| 20000| + |dotNET|2012| 5000| + |dotNET|2013| 48000| + | Java|2013| 30000| + +------+----+--------+ + + >>> df2 = spark.createDataFrame([ + ... Row(training="expert", sales=Row(course="dotNET", year=2012, earnings=10000)), + ... Row(training="junior", sales=Row(course="Java", year=2012, earnings=20000)), + ... Row(training="expert", sales=Row(course="dotNET", year=2012, earnings=5000)), + ... Row(training="junior", sales=Row(course="dotNET", year=2013, earnings=48000)), + ... Row(training="expert", sales=Row(course="Java", year=2013, earnings=30000)), + ... ]) + >>> df2.show(truncate=False) + +--------+---------------------+ + |training|sales | + +--------+---------------------+ + |expert |{dotNET, 2012, 10000}| + |junior |{Java, 2012, 20000} | + |expert |{dotNET, 2012, 5000} | + |junior |{dotNET, 2013, 48000}| + |expert |{Java, 2013, 30000} | + +--------+---------------------+ + + Compute the sum of earnings for each year by course with each course as a separate column + + >>> df1.groupBy("year").pivot( + ... "course", ["dotNET", "Java"] + ... ).sum("earnings").sort("year").show() + +----+------+-----+ + |year|dotNET| Java| + +----+------+-----+ + |2012| 15000|20000| + |2013| 48000|30000| + +----+------+-----+ + + Or without specifying column values (less efficient) + + >>> df1.groupBy("year").pivot("course").sum("earnings").sort("year").show() + +----+-----+------+ + |year| Java|dotNET| + +----+-----+------+ + |2012|20000| 15000| + |2013|30000| 48000| + +----+-----+------+ + + >>> df2.groupBy("sales.year").pivot( + ... "sales.course" + ... ).agg(sf.sum("sales.earnings")).sort("year").show() + +----+-----+------+ + |year| Java|dotNET| + +----+-----+------+ + |2012|20000| 15000| + |2013|30000| 48000| + +----+-----+------+ + """ + if values is None: + jgd = self._jgd.pivot(pivot_col) + else: + jgd = self._jgd.pivot(pivot_col, values) + return GroupedData(jgd, self._df) + + +def _test() -> None: + import doctest + from pyspark.sql import SparkSession + import pyspark.sql.group + from pyspark.testing.utils import have_pandas, have_pyarrow + + globs = pyspark.sql.group.__dict__.copy() + + if not have_pandas or not have_pyarrow: + del pyspark.sql.group.GroupedData.agg.__doc__ + + spark = SparkSession.builder.master("local[4]").appName("sql.group tests").getOrCreate() + globs["spark"] = spark + + failure_count, test_count = doctest.testmod( + pyspark.sql.group, + globs=globs, + optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF, + ) + spark.stop() + if failure_count: + sys.exit(-1) + + +if __name__ == "__main__": + _test() diff --git a/python/pyspark/sql/group.pyi b/python/pyspark/sql/group.pyi deleted file mode 100644 index e78c06a9..00000000 --- a/python/pyspark/sql/group.pyi +++ /dev/null @@ -1,558 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Generated by scripts/gen_type_stubs.py from python/pyspark/sql/group.py of Apache Spark v4.3.0: -# do not edit. Type checkers read this stub instead of the module of the drop-in. - -from typing import Callable, List, Optional, overload, Dict -from pyspark.sql.column import Column -from pyspark.sql.session import SparkSession -from pyspark.sql.dataframe import DataFrame -from pyspark.sql.pandas.group_ops import PandasGroupedOpsMixin -from py4j.java_gateway import JavaObject -from pyspark.sql._typing import LiteralType - -__all__ = ['GroupedData'] - -def dfapi(f: Callable[..., DataFrame]) -> Callable[..., DataFrame]: ... -def df_varargs_api(f: Callable[..., DataFrame]) -> Callable[..., DataFrame]: ... - -class GroupedData(PandasGroupedOpsMixin): - """A set of methods for aggregations on a :class:`DataFrame`, - created by :func:`DataFrame.groupBy`. - - .. versionadded:: 1.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - """ - def __init__(self, jgd: 'JavaObject', df: DataFrame): - self._jgd = jgd - self._df = df - self.session: SparkSession = df.sparkSession - def __repr__(self) -> str: ... - @overload - def agg(self, *exprs: Column) -> 'DataFrame': - """Compute aggregates and returns the result as a :class:`DataFrame`. - - The available aggregate functions can be: - - 1. built-in aggregation functions, such as `avg`, `max`, `min`, `sum`, `count` - - 2. group aggregate pandas UDFs, created with :func:`pyspark.sql.functions.pandas_udf` - - .. note:: There is no partial aggregation with group aggregate UDFs, i.e., - a full shuffle is required. Also, all the data of a group will be loaded into - memory, so the user should be aware of the potential OOM risk if data is skewed - and certain groups are too large to fit in memory. - - .. seealso:: :func:`pyspark.sql.functions.pandas_udf` - - If ``exprs`` is a single :class:`dict` mapping from string to string, then the key - is the column to perform aggregation on, and the value is the aggregate function. - - Alternatively, ``exprs`` can also be a list of aggregate :class:`Column` expressions. - - .. versionadded:: 1.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - exprs : dict - a dict mapping from column name (string) to aggregate functions (string), - or a list of :class:`Column`. - - Notes - ----- - Built-in aggregation functions and group aggregate pandas UDFs cannot be mixed - in a single call to this function. - - Examples - -------- - >>> import pandas as pd - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(2, "Alice"), (3, "Alice"), (5, "Bob"), (10, "Bob")], ["age", "name"]) - >>> df.show() - +---+-----+ - |age| name| - +---+-----+ - | 2|Alice| - | 3|Alice| - | 5| Bob| - | 10| Bob| - +---+-----+ - - Group-by name, and count each group. - - >>> df.groupBy(df.name) - GroupedData[grouping...: [name...], value: [age: bigint, name: string], type: GroupBy] - - >>> df.groupBy(df.name).agg({"*": "count"}).sort("name").show() - +-----+--------+ - | name|count(1)| - +-----+--------+ - |Alice| 2| - | Bob| 2| - +-----+--------+ - - Group-by name, and calculate the minimum age. - - >>> df.groupBy(df.name).agg(sf.min(df.age)).sort("name").show() - +-----+--------+ - | name|min(age)| - +-----+--------+ - |Alice| 2| - | Bob| 5| - +-----+--------+ - - Same as above but uses pandas UDF. - - >>> from pyspark.sql.functions import pandas_udf - >>> @pandas_udf('int') - ... def min_udf(v: pd.Series) -> int: - ... return v.min() - ... - >>> df.groupBy(df.name).agg(min_udf(df.age)).sort("name").show() - +-----+------------+ - | name|min_udf(age)| - +-----+------------+ - |Alice| 2| - | Bob| 5| - +-----+------------+ - """ - @overload - def agg(self, __exprs: Dict[str, str]) -> 'DataFrame': - """Compute aggregates and returns the result as a :class:`DataFrame`. - - The available aggregate functions can be: - - 1. built-in aggregation functions, such as `avg`, `max`, `min`, `sum`, `count` - - 2. group aggregate pandas UDFs, created with :func:`pyspark.sql.functions.pandas_udf` - - .. note:: There is no partial aggregation with group aggregate UDFs, i.e., - a full shuffle is required. Also, all the data of a group will be loaded into - memory, so the user should be aware of the potential OOM risk if data is skewed - and certain groups are too large to fit in memory. - - .. seealso:: :func:`pyspark.sql.functions.pandas_udf` - - If ``exprs`` is a single :class:`dict` mapping from string to string, then the key - is the column to perform aggregation on, and the value is the aggregate function. - - Alternatively, ``exprs`` can also be a list of aggregate :class:`Column` expressions. - - .. versionadded:: 1.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - exprs : dict - a dict mapping from column name (string) to aggregate functions (string), - or a list of :class:`Column`. - - Notes - ----- - Built-in aggregation functions and group aggregate pandas UDFs cannot be mixed - in a single call to this function. - - Examples - -------- - >>> import pandas as pd - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(2, "Alice"), (3, "Alice"), (5, "Bob"), (10, "Bob")], ["age", "name"]) - >>> df.show() - +---+-----+ - |age| name| - +---+-----+ - | 2|Alice| - | 3|Alice| - | 5| Bob| - | 10| Bob| - +---+-----+ - - Group-by name, and count each group. - - >>> df.groupBy(df.name) - GroupedData[grouping...: [name...], value: [age: bigint, name: string], type: GroupBy] - - >>> df.groupBy(df.name).agg({"*": "count"}).sort("name").show() - +-----+--------+ - | name|count(1)| - +-----+--------+ - |Alice| 2| - | Bob| 2| - +-----+--------+ - - Group-by name, and calculate the minimum age. - - >>> df.groupBy(df.name).agg(sf.min(df.age)).sort("name").show() - +-----+--------+ - | name|min(age)| - +-----+--------+ - |Alice| 2| - | Bob| 5| - +-----+--------+ - - Same as above but uses pandas UDF. - - >>> from pyspark.sql.functions import pandas_udf - >>> @pandas_udf('int') - ... def min_udf(v: pd.Series) -> int: - ... return v.min() - ... - >>> df.groupBy(df.name).agg(min_udf(df.age)).sort("name").show() - +-----+------------+ - | name|min_udf(age)| - +-----+------------+ - |Alice| 2| - | Bob| 5| - +-----+------------+ - """ - @dfapi - def count(self) -> 'DataFrame': - """Counts the number of records for each group. - - .. versionadded:: 1.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Examples - -------- - >>> df = spark.createDataFrame( - ... [(2, "Alice"), (3, "Alice"), (5, "Bob"), (10, "Bob")], ["age", "name"]) - >>> df.show() - +---+-----+ - |age| name| - +---+-----+ - | 2|Alice| - | 3|Alice| - | 5| Bob| - | 10| Bob| - +---+-----+ - - Group-by name, and count each group. - - >>> df.groupBy(df.name).count().sort("name").show() - +-----+-----+ - | name|count| - +-----+-----+ - |Alice| 2| - | Bob| 2| - +-----+-----+ - """ - @df_varargs_api - def mean(self, *cols: str) -> DataFrame: - """Computes average values for each numeric columns for each group. - - :func:`mean` is an alias for :func:`avg`. - - .. versionadded:: 1.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - cols : str - column names. Non-numeric columns are ignored. - """ - @df_varargs_api - def avg(self, *cols: str) -> 'DataFrame': - """Computes average values for each numeric columns for each group. - - :func:`mean` is an alias for :func:`avg`. - - .. versionadded:: 1.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - cols : str - column names. Non-numeric columns are ignored. - - Examples - -------- - >>> df = spark.createDataFrame([ - ... (2, "Alice", 80), (3, "Alice", 100), - ... (5, "Bob", 120), (10, "Bob", 140)], ["age", "name", "height"]) - >>> df.show() - +---+-----+------+ - |age| name|height| - +---+-----+------+ - | 2|Alice| 80| - | 3|Alice| 100| - | 5| Bob| 120| - | 10| Bob| 140| - +---+-----+------+ - - Group-by name, and calculate the mean of the age in each group. - - >>> df.groupBy("name").avg('age').sort("name").show() - +-----+--------+ - | name|avg(age)| - +-----+--------+ - |Alice| 2.5| - | Bob| 7.5| - +-----+--------+ - - Calculate the mean of the age and height in all data. - - >>> df.groupBy().avg('age', 'height').show() - +--------+-----------+ - |avg(age)|avg(height)| - +--------+-----------+ - | 5.0| 110.0| - +--------+-----------+ - """ - @df_varargs_api - def max(self, *cols: str) -> 'DataFrame': - """Computes the max value for each numeric columns for each group. - - .. versionadded:: 1.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Examples - -------- - >>> df = spark.createDataFrame([ - ... (2, "Alice", 80), (3, "Alice", 100), - ... (5, "Bob", 120), (10, "Bob", 140)], ["age", "name", "height"]) - >>> df.show() - +---+-----+------+ - |age| name|height| - +---+-----+------+ - | 2|Alice| 80| - | 3|Alice| 100| - | 5| Bob| 120| - | 10| Bob| 140| - +---+-----+------+ - - Group-by name, and calculate the max of the age in each group. - - >>> df.groupBy("name").max("age").sort("name").show() - +-----+--------+ - | name|max(age)| - +-----+--------+ - |Alice| 3| - | Bob| 10| - +-----+--------+ - - Calculate the max of the age and height in all data. - - >>> df.groupBy().max("age", "height").show() - +--------+-----------+ - |max(age)|max(height)| - +--------+-----------+ - | 10| 140| - +--------+-----------+ - """ - @df_varargs_api - def min(self, *cols: str) -> 'DataFrame': - """Computes the min value for each numeric column for each group. - - .. versionadded:: 1.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - cols : str - column names. Non-numeric columns are ignored. - - Examples - -------- - >>> df = spark.createDataFrame([ - ... (2, "Alice", 80), (3, "Alice", 100), - ... (5, "Bob", 120), (10, "Bob", 140)], ["age", "name", "height"]) - >>> df.show() - +---+-----+------+ - |age| name|height| - +---+-----+------+ - | 2|Alice| 80| - | 3|Alice| 100| - | 5| Bob| 120| - | 10| Bob| 140| - +---+-----+------+ - - Group-by name, and calculate the min of the age in each group. - - >>> df.groupBy("name").min("age").sort("name").show() - +-----+--------+ - | name|min(age)| - +-----+--------+ - |Alice| 2| - | Bob| 5| - +-----+--------+ - - Calculate the min of the age and height in all data. - - >>> df.groupBy().min("age", "height").show() - +--------+-----------+ - |min(age)|min(height)| - +--------+-----------+ - | 2| 80| - +--------+-----------+ - """ - @df_varargs_api - def sum(self, *cols: str) -> DataFrame: - """Computes the sum for each numeric columns for each group. - - .. versionadded:: 1.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - cols : str - column names. Non-numeric columns are ignored. - - Examples - -------- - >>> df = spark.createDataFrame([ - ... (2, "Alice", 80), (3, "Alice", 100), - ... (5, "Bob", 120), (10, "Bob", 140)], ["age", "name", "height"]) - >>> df.show() - +---+-----+------+ - |age| name|height| - +---+-----+------+ - | 2|Alice| 80| - | 3|Alice| 100| - | 5| Bob| 120| - | 10| Bob| 140| - +---+-----+------+ - - Group-by name, and calculate the sum of the age in each group. - - >>> df.groupBy("name").sum("age").sort("name").show() - +-----+--------+ - | name|sum(age)| - +-----+--------+ - |Alice| 5| - | Bob| 15| - +-----+--------+ - - Calculate the sum of the age and height in all data. - - >>> df.groupBy().sum("age", "height").show() - +--------+-----------+ - |sum(age)|sum(height)| - +--------+-----------+ - | 20| 440| - +--------+-----------+ - """ - def pivot(self, pivot_col: str, values: Optional[List['LiteralType']]=None) -> 'GroupedData': - """Pivots a column of the current :class:`DataFrame` and performs the specified aggregation. - - .. versionadded:: 1.6.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - pivot_col : str - Name of the column to pivot. - values : list, optional - List of values that will be translated to columns in the output DataFrame. - - If ``values`` is not provided, Spark will eagerly compute the distinct values in - ``pivot_col`` so it can determine the resulting schema of the transformation. To avoid - any eager computations, provide an explicit list of values. - - Examples - -------- - >>> from pyspark.sql import Row, functions as sf - >>> df1 = spark.createDataFrame([ - ... Row(course="dotNET", year=2012, earnings=10000), - ... Row(course="Java", year=2012, earnings=20000), - ... Row(course="dotNET", year=2012, earnings=5000), - ... Row(course="dotNET", year=2013, earnings=48000), - ... Row(course="Java", year=2013, earnings=30000), - ... ]) - >>> df1.show() - +------+----+--------+ - |course|year|earnings| - +------+----+--------+ - |dotNET|2012| 10000| - | Java|2012| 20000| - |dotNET|2012| 5000| - |dotNET|2013| 48000| - | Java|2013| 30000| - +------+----+--------+ - - >>> df2 = spark.createDataFrame([ - ... Row(training="expert", sales=Row(course="dotNET", year=2012, earnings=10000)), - ... Row(training="junior", sales=Row(course="Java", year=2012, earnings=20000)), - ... Row(training="expert", sales=Row(course="dotNET", year=2012, earnings=5000)), - ... Row(training="junior", sales=Row(course="dotNET", year=2013, earnings=48000)), - ... Row(training="expert", sales=Row(course="Java", year=2013, earnings=30000)), - ... ]) - >>> df2.show(truncate=False) - +--------+---------------------+ - |training|sales | - +--------+---------------------+ - |expert |{dotNET, 2012, 10000}| - |junior |{Java, 2012, 20000} | - |expert |{dotNET, 2012, 5000} | - |junior |{dotNET, 2013, 48000}| - |expert |{Java, 2013, 30000} | - +--------+---------------------+ - - Compute the sum of earnings for each year by course with each course as a separate column - - >>> df1.groupBy("year").pivot( - ... "course", ["dotNET", "Java"] - ... ).sum("earnings").sort("year").show() - +----+------+-----+ - |year|dotNET| Java| - +----+------+-----+ - |2012| 15000|20000| - |2013| 48000|30000| - +----+------+-----+ - - Or without specifying column values (less efficient) - - >>> df1.groupBy("year").pivot("course").sum("earnings").sort("year").show() - +----+-----+------+ - |year| Java|dotNET| - +----+-----+------+ - |2012|20000| 15000| - |2013|30000| 48000| - +----+-----+------+ - - >>> df2.groupBy("sales.year").pivot( - ... "sales.course" - ... ).agg(sf.sum("sales.earnings")).sort("year").show() - +----+-----+------+ - |year| Java|dotNET| - +----+-----+------+ - |2012|20000| 15000| - |2013|30000| 48000| - +----+-----+------+ - """ diff --git a/python/pyspark/sql/pandas/group_ops.py b/python/pyspark/sql/pandas/group_ops.py index d31c3675..bbbe55ef 100644 --- a/python/pyspark/sql/pandas/group_ops.py +++ b/python/pyspark/sql/pandas/group_ops.py @@ -14,12 +14,1214 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""Rust-backed PandasCogroupedOps under the official import path.""" -from pyspark._pyspark import PandasCogroupedOps +import sys +from typing import List, Optional, Union, TYPE_CHECKING, cast, Any +import warnings -# As PySpark's module imports them, so that `import pyspark.sql` loads -# pyspark.sql.streaming.state too, a name of the package pyspark.sql.streaming there. -from pyspark.sql.streaming.state import GroupStateTimeout # noqa: F401 -from pyspark.sql.streaming.stateful_processor import StatefulProcessor # noqa: F401 +from pyspark.errors import PySparkTypeError +from pyspark.util import PythonEvalType +from pyspark.sql.column import Column +from pyspark.sql.dataframe import DataFrame +from pyspark.sql.streaming.state import GroupStateTimeout +from pyspark.sql.streaming.stateful_processor import StatefulProcessor +from pyspark.sql.types import StructType -__all__ = ["PandasCogroupedOps"] +if TYPE_CHECKING: + from pyspark.sql.pandas._typing import ( + GroupedMapPandasUserDefinedFunction, + PandasGroupedMapFunction, + PandasGroupedMapFunctionWithState, + PandasCogroupedMapFunction, + ArrowGroupedMapFunction, + ArrowCogroupedMapFunction, + ) + from pyspark.sql.group import GroupedData + + +class PandasGroupedOpsMixin: + """ + Min-in for pandas grouped operations. Currently, only :class:`GroupedData` + can use this class. + """ + + def apply(self, udf: "GroupedMapPandasUserDefinedFunction") -> "DataFrame": + """ + It is an alias of :meth:`pyspark.sql.GroupedData.applyInPandas`; however, it takes a + :meth:`pyspark.sql.functions.pandas_udf` whereas + :meth:`pyspark.sql.GroupedData.applyInPandas` takes a Python native function. + + .. versionadded:: 2.3.0 + + .. versionchanged:: 3.4.0 + Support Spark Connect. + + Parameters + ---------- + udf : :func:`pyspark.sql.functions.pandas_udf` + a grouped map user-defined function returned by + :func:`pyspark.sql.functions.pandas_udf`. + + Notes + ----- + It is preferred to use :meth:`pyspark.sql.GroupedData.applyInPandas` over this + API. This API will be deprecated in the future releases. + + Examples + -------- + >>> from pyspark.sql.functions import pandas_udf, PandasUDFType + >>> df = spark.createDataFrame( + ... [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)], + ... ("id", "v")) + >>> @pandas_udf("id long, v double", PandasUDFType.GROUPED_MAP) + ... def normalize(pdf): + ... v = pdf.v + ... return pdf.assign(v=(v - v.mean()) / v.std()) + ... + >>> df.groupby("id").apply(normalize).sort("id", "v").show() + +---+-------------------+ + | id| v| + +---+-------------------+ + | 1|-0.7071067811865...| + | 1| 0.7071067811865...| + | 2|-0.8320502943378...| + | 2|-0.2773500981126...| + | 2| 1.1094003924504...| + +---+-------------------+ + + See Also + -------- + pyspark.sql.functions.pandas_udf + """ + # Columns are special because hasattr always return True + if ( + isinstance(udf, Column) + or not hasattr(udf, "func") + or ( + udf.evalType # type: ignore[attr-defined] + != PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF + ) + ): + raise PySparkTypeError( + errorClass="INVALID_UDF_EVAL_TYPE", + messageParameters={"eval_type": "SQL_GROUPED_MAP_PANDAS_UDF"}, + ) + + warnings.warn( + "It is preferred to use 'applyInPandas' over this " + "API. This API will be deprecated in the future releases. See SPARK-28264 for " + "more details.", + UserWarning, + ) + + return self.applyInPandas(udf.func, schema=udf.returnType) # type: ignore[attr-defined] + + def applyInPandas( + self, func: "PandasGroupedMapFunction", schema: Union["StructType", str] + ) -> "DataFrame": + """ + Maps each group of the current :class:`DataFrame` using a pandas udf and returns the result + as a `DataFrame`. + + The function can take one of two forms: It can take a `pandas.DataFrame` and return a + `pandas.DataFrame`, or it can take an iterator of `pandas.DataFrame` and yield + `pandas.DataFrame`. Alternatively each form can take a tuple of grouping keys + as the first argument in addition to the input type above. + For each group, all columns are passed together as a `pandas.DataFrame` or iterator of + `pandas.DataFrame`, and the returned `pandas.DataFrame` or iterator of `pandas.DataFrame` + are combined as a :class:`DataFrame`. + + The `schema` should be a :class:`StructType` describing the schema of the returned + `pandas.DataFrame`. The column labels of the returned `pandas.DataFrame` must either match + the field names in the defined schema if specified as strings, or match the + field data types by position if not strings, e.g. integer indices. + The length of the returned `pandas.DataFrame` can be arbitrary. + + .. versionadded:: 3.0.0 + + .. versionchanged:: 3.4.0 + Support Spark Connect. + + .. versionchanged:: 4.1.0 + Added support for an iterator of `pandas.DataFrame` API. + + Parameters + ---------- + func : function + a Python native function that either takes a `pandas.DataFrame` and outputs a + `pandas.DataFrame` or takes an iterator of `pandas.DataFrame` and yields + `pandas.DataFrame`. Additionally, each form can take a tuple of grouping keys + as the first argument, with the `pandas.DataFrame` or iterator of `pandas.DataFrame` + as the second argument. + schema : :class:`pyspark.sql.types.DataType` or str + the return type of the `func` in PySpark. The value can be either a + :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. + + Examples + -------- + >>> import pandas as pd + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)], + ... ("id", "v")) + >>> def normalize(pdf): + ... v = pdf.v + ... return pdf.assign(v=(v - v.mean()) / v.std()) + ... + >>> df.groupby("id").applyInPandas( + ... normalize, schema="id long, v double" + ... ).sort("id", "v").show() + +---+-------------------+ + | id| v| + +---+-------------------+ + | 1|-0.7071067811865...| + | 1| 0.7071067811865...| + | 2|-0.8320502943378...| + | 2|-0.2773500981126...| + | 2| 1.1094003924504...| + +---+-------------------+ + + Alternatively, the user can pass a function that takes two arguments. + In this case, the grouping key(s) will be passed as the first argument and the data will + be passed as the second argument. The grouping key(s) will be passed as a tuple of numpy + data types, e.g., `numpy.int32` and `numpy.float64`. The data will still be passed in + as a `pandas.DataFrame` containing all columns from the original Spark DataFrame. + This is useful when the user does not want to hardcode grouping key(s) in the function. + + >>> df = spark.createDataFrame( + ... [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)], + ... ("id", "v")) + >>> def mean_func(key, pdf): + ... # key is a tuple of one numpy.int64, which is the value + ... # of 'id' for the current group + ... return pd.DataFrame([key + (pdf.v.mean(),)]) + ... + >>> df.groupby("id").applyInPandas( + ... mean_func, schema="id long, v double" + ... ).sort("id").show() + +---+---+ + | id| v| + +---+---+ + | 1|1.5| + | 2|6.0| + +---+---+ + + >>> def sum_func(key, pdf): + ... # key is a tuple of two numpy.int64s, which is the values + ... # of 'id' and 'ceil(df.v / 2)' for the current group + ... return pd.DataFrame([key + (pdf.v.sum(),)]) + ... + >>> df.groupby(df.id, sf.ceil(df.v / 2)).applyInPandas( + ... sum_func, schema="id long, `ceil(v / 2)` long, v double" + ... ).sort("id", "v").show() + +---+-----------+----+ + | id|ceil(v / 2)| v| + +---+-----------+----+ + | 1| 1| 3.0| + | 2| 2| 3.0| + | 2| 3| 5.0| + | 2| 5|10.0| + +---+-----------+----+ + + The function can also take and return an iterator of `pandas.DataFrame` using type + hints. + + >>> from typing import Iterator + >>> df = spark.createDataFrame( + ... [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)], + ... ("id", "v")) + >>> def filter_func( + ... batches: Iterator[pd.DataFrame] + ... ) -> Iterator[pd.DataFrame]: + ... for batch in batches: + ... # Process and yield each batch independently + ... filtered = batch[batch['v'] > 2.0] + ... if not filtered.empty: + ... yield filtered[['v']] + >>> df.groupby("id").applyInPandas( + ... filter_func, schema="v double" + ... ).sort("v").show() + +----+ + | v| + +----+ + | 3.0| + | 5.0| + |10.0| + +----+ + + Alternatively, the user can pass a function that takes two arguments. + In this case, the grouping key(s) will be passed as the first argument and the data will + be passed as the second argument. The grouping key(s) will be passed as a tuple of numpy + data types. The data will still be passed in as an iterator of `pandas.DataFrame`. + + >>> from typing import Iterator, Tuple, Any + >>> def transform_func( + ... key: Tuple[Any, ...], batches: Iterator[pd.DataFrame] + ... ) -> Iterator[pd.DataFrame]: + ... for batch in batches: + ... # Yield transformed results for each batch + ... result = batch.assign(id=key[0], v_doubled=batch['v'] * 2) + ... yield result[['id', 'v_doubled']] + >>> df.groupby("id").applyInPandas( + ... transform_func, schema="id long, v_doubled double" + ... ).sort("id", "v_doubled").show() + +---+---------+ + | id|v_doubled| + +---+---------+ + | 1| 2.0| + | 1| 4.0| + | 2| 6.0| + | 2| 10.0| + | 2| 20.0| + +---+---------+ + + Notes + ----- + This function requires a full shuffle. If using the `pandas.DataFrame` API, all data of a + group will be loaded into memory, so the user should be aware of the potential OOM risk if + data is skewed and certain groups are too large to fit in memory, and can use the + iterator of `pandas.DataFrame` API to mitigate this. + + See Also + -------- + pyspark.sql.functions.pandas_udf + """ + from pyspark.sql import GroupedData + from pyspark.sql.functions import pandas_udf + from pyspark.sql.pandas.typehints import infer_group_pandas_eval_type_from_func + + assert isinstance(self, GroupedData) + + # Try to infer the eval type from type hints + eval_type = None + try: + eval_type = infer_group_pandas_eval_type_from_func(func) + except Exception as e: + warnings.warn(f"Cannot infer the eval type from type hints: {e}", UserWarning) + + if eval_type is None: + eval_type = PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF + + udf = pandas_udf(func, returnType=schema, functionType=eval_type) + df = self._df + udf_column = udf(*[df[col] for col in df.columns]) + jdf = self._jgd.flatMapGroupsInPandas(udf_column._jc) + return DataFrame(jdf, self.session) + + def applyInPandasWithState( + self, + func: "PandasGroupedMapFunctionWithState", + outputStructType: Union[StructType, str], + stateStructType: Union[StructType, str], + outputMode: str, + timeoutConf: str, + ) -> "DataFrame": + """ + Applies the given function to each group of data, while maintaining a user-defined + per-group state. The result Dataset will represent the flattened record returned by the + function. + + For a streaming :class:`DataFrame`, the function will be invoked first for all input groups + and then for all timed out states where the input data is set to be empty. Updates to each + group's state will be saved across invocations. + + The function should take parameters (key, Iterator[`pandas.DataFrame`], state) and + return another Iterator[`pandas.DataFrame`]. The grouping key(s) will be passed as a tuple + of numpy data types, e.g., `numpy.int32` and `numpy.float64`. The state will be passed as + :class:`pyspark.sql.streaming.state.GroupState`. + + For each group, all columns are passed together as `pandas.DataFrame` to the user-function, + and the returned `pandas.DataFrame` across all invocations are combined as a + :class:`DataFrame`. Note that the user function should not make a guess of the number of + elements in the iterator. To process all data, the user function needs to iterate all + elements and process them. On the other hand, the user function is not strictly required to + iterate through all elements in the iterator if it intends to read a part of data. + + The `outputStructType` should be a :class:`StructType` describing the schema of all + elements in the returned value, `pandas.DataFrame`. The column labels of all elements in + returned `pandas.DataFrame` must either match the field names in the defined schema if + specified as strings, or match the field data types by position if not strings, + e.g. integer indices. + + The `stateStructType` should be :class:`StructType` describing the schema of the + user-defined state. The value of the state will be presented as a tuple, as well as the + update should be performed with the tuple. The corresponding Python types for + :class:DataType are supported. Please refer to the page + https://spark.apache.org/docs/latest/sql-ref-datatypes.html (Python tab). + + The size of each `pandas.DataFrame` in both the input and output can be arbitrary. The + number of `pandas.DataFrame` in both the input and output can also be arbitrary. + + .. versionadded:: 3.4.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + func : function + a Python native function to be called on every group. It should take parameters + (key, Iterator[`pandas.DataFrame`], state) and return Iterator[`pandas.DataFrame`]. + Note that the type of the key is tuple and the type of the state is + :class:`pyspark.sql.streaming.state.GroupState`. + outputStructType : :class:`pyspark.sql.types.DataType` or str + the type of the output records. The value can be either a + :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. + stateStructType : :class:`pyspark.sql.types.DataType` or str + the type of the user-defined state. The value can be either a + :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. + outputMode : str + the output mode of the function. + timeoutConf : str + timeout configuration for groups that do not receive data for a while. valid values + are defined in :class:`pyspark.sql.streaming.state.GroupStateTimeout`. + + Examples + -------- + >>> import pandas as pd # doctest: +SKIP + >>> from pyspark.sql.streaming.state import GroupStateTimeout + >>> def count_fn(key, pdf_iter, state): + ... assert isinstance(state, GroupStateImpl) + ... total_len = 0 + ... for pdf in pdf_iter: + ... total_len += len(pdf) + ... state.update((total_len,)) + ... yield pd.DataFrame({"id": [key[0]], "countAsString": [str(total_len)]}) + ... + >>> df.groupby("id").applyInPandasWithState( + ... count_fn, outputStructType="id long, countAsString string", + ... stateStructType="len long", outputMode="Update", + ... timeoutConf=GroupStateTimeout.NoTimeout) # doctest: +SKIP + + Notes + ----- + This function requires a full shuffle. + """ + + from pyspark.sql import GroupedData + from pyspark.sql.functions import pandas_udf + + assert isinstance(self, GroupedData) + assert timeoutConf in [ + GroupStateTimeout.NoTimeout, + GroupStateTimeout.ProcessingTimeTimeout, + GroupStateTimeout.EventTimeTimeout, + ] + + if isinstance(outputStructType, str): + outputStructType = cast(StructType, self._df._session._parse_ddl(outputStructType)) + if isinstance(stateStructType, str): + stateStructType = cast(StructType, self._df._session._parse_ddl(stateStructType)) + + udf = pandas_udf( + func, # type: ignore[call-overload] + returnType=outputStructType, + functionType=PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF_WITH_STATE, + ) + df = self._df + udf_column = udf(*[df[col] for col in df.columns]) + jdf = self._jgd.applyInPandasWithState( + udf_column._jc, + self.session._jsparkSession.parseDataType(outputStructType.json()), + self.session._jsparkSession.parseDataType(stateStructType.json()), + outputMode, + timeoutConf, + ) + return DataFrame(jdf, self.session) + + def transformWithStateInPandas( + self, + statefulProcessor: StatefulProcessor, + outputStructType: Union[StructType, str], + outputMode: str, + timeMode: str, + initialState: Optional["GroupedData"] = None, + eventTimeColumnName: str = "", + ) -> "DataFrame": + """ + Invokes methods defined in the stateful processor used in arbitrary state API v2. It + requires protobuf, pandas and pyarrow as dependencies to process input/state data. We + allow the user to act on per-group set of input rows along with keyed state and the user + can choose to output/return 0 or more rows. + + For a streaming dataframe, we will repeatedly invoke the interface methods for new rows + in each trigger and the user's state/state variables will be stored persistently across + invocations. + + The `statefulProcessor` should be a Python class that implements the interface defined in + :class:`StatefulProcessor`. + + The `outputStructType` should be a :class:`StructType` describing the schema of all + elements in the returned value, `pandas.DataFrame`. The column labels of all elements in + returned `pandas.DataFrame` must either match the field names in the defined schema if + specified as strings, or match the field data types by position if not strings, + e.g. integer indices. + + The size of each `pandas.DataFrame` in both the input and output can be arbitrary. The + number of `pandas.DataFrame` in both the input and output can also be arbitrary. + + .. versionadded:: 4.0.0 + + Parameters + ---------- + statefulProcessor : :class:`pyspark.sql.streaming.stateful_processor.StatefulProcessor` + Instance of StatefulProcessor whose functions will be invoked by the operator. + outputStructType : :class:`pyspark.sql.types.DataType` or str + The type of the output records. The value can be either a + :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. + outputMode : str + The output mode of the stateful processor. + timeMode : str + The time mode semantics of the stateful processor for timers and TTL. + initialState : :class:`pyspark.sql.GroupedData` + Optional. The grouped dataframe as initial states used for initialization + of state variables in the first batch. + + Examples + -------- + >>> from typing import Iterator + ... + >>> import pandas as pd # doctest: +SKIP + ... + >>> from pyspark.sql import Row + >>> from pyspark.sql.functions import col, split + >>> from pyspark.sql.streaming import StatefulProcessor, StatefulProcessorHandle + >>> from pyspark.sql.types import IntegerType, LongType, StringType, StructField, StructType + ... + >>> spark.conf.set("spark.sql.streaming.stateStore.providerClass", + ... "org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider") + ... # Below is a simple example to find erroneous sensors from temperature sensor data. The + ... # processor returns a count of total readings, while keeping erroneous reading counts + ... # in streaming state. A violation is defined when the temperature is above 100. + ... # The input data is a DataFrame with the following schema: + ... # `id: string, temperature: long`. + ... # The output schema and state schema are defined as below. + >>> output_schema = StructType([ + ... StructField("id", StringType(), True), + ... StructField("count", IntegerType(), True) + ... ]) + >>> state_schema = StructType([ + ... StructField("value", IntegerType(), True) + ... ]) + >>> class SimpleStatefulProcessor(StatefulProcessor): + ... def init(self, handle: StatefulProcessorHandle): + ... self.num_violations_state = handle.getValueState("numViolations", state_schema) + ... + ... def handleInputRows(self, key, rows): + ... new_violations = 0 + ... count = 0 + ... exists = self.num_violations_state.exists() + ... if exists: + ... existing_violations_row = self.num_violations_state.get() + ... existing_violations = existing_violations_row[0] + ... else: + ... existing_violations = 0 + ... for pdf in rows: + ... pdf_count = pdf.count() + ... count += pdf_count.get('temperature') + ... violations_pdf = pdf.loc[pdf['temperature'] > 100] + ... new_violations += violations_pdf.count().get('temperature') + ... updated_violations = new_violations + existing_violations + ... self.num_violations_state.update((updated_violations,)) + ... yield pd.DataFrame({'id': key, 'count': count}) + ... + ... def close(self) -> None: + ... pass + + Input DataFrame: + +---+-----------+ + | id|temperature| + +---+-----------+ + | 0| 123| + | 0| 23| + | 1| 33| + | 1| 188| + | 1| 88| + +---+-----------+ + + >>> df.groupBy("value").transformWithStateInPandas(statefulProcessor = + ... SimpleStatefulProcessor(), outputStructType=output_schema, outputMode="Update", + ... timeMode="None") # doctest: +SKIP + + Output DataFrame: + +---+-----+ + | id|count| + +---+-----+ + | 0| 2| + | 1| 3| + +---+-----+ + + Notes + ----- + This function requires a full shuffle. + """ + return self.__transformWithState( + statefulProcessor, + outputStructType, + outputMode, + timeMode, + True, + initialState, + eventTimeColumnName, + ) + + def transformWithState( + self, + statefulProcessor: StatefulProcessor, + outputStructType: Union[StructType, str], + outputMode: str, + timeMode: str, + initialState: Optional["GroupedData"] = None, + eventTimeColumnName: str = "", + ) -> "DataFrame": + """ + Invokes methods defined in the stateful processor used in arbitrary state API v2. It + requires protobuf and pyarrow as dependencies to process input/state data. We allow + the user to act on per-group set of input rows along with keyed state and the user + can choose to output/return 0 or more rows. + + For a streaming dataframe, we will repeatedly invoke the interface methods for new rows + in each trigger and the user's state/state variables will be stored persistently across + invocations. + + The `statefulProcessor` should be a Python class that implements the interface defined in + :class:`StatefulProcessor`. + + The `outputStructType` should be a :class:`StructType` describing the schema of all + elements in the returned value, `Row`. The column labels of all elements in + returned `Row` must either match the field names in the defined schema. + + The number of `Row`s in the iterator in both the input and output can be arbitrary. + + .. versionadded:: 4.1.0 + + Parameters + ---------- + statefulProcessor : :class:`pyspark.sql.streaming.stateful_processor.StatefulProcessor` + Instance of StatefulProcessor whose functions will be invoked by the operator. + outputStructType : :class:`pyspark.sql.types.DataType` or str + The type of the output records. The value can be either a + :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. + outputMode : str + The output mode of the stateful processor. + timeMode : str + The time mode semantics of the stateful processor for timers and TTL. + initialState : :class:`pyspark.sql.GroupedData` + Optional. The grouped dataframe as initial states used for initialization + of state variables in the first batch. + + Examples + -------- + >>> from typing import Iterator + ... + >>> from pyspark.sql import Row + >>> from pyspark.sql.functions import col, split + >>> from pyspark.sql.streaming import StatefulProcessor, StatefulProcessorHandle + >>> from pyspark.sql.types import IntegerType, LongType, StringType, StructField, StructType + ... + >>> spark.conf.set("spark.sql.streaming.stateStore.providerClass", + ... "org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider") + ... # Below is a simple example to find erroneous sensors from temperature sensor data. The + ... # processor returns a count of total readings, while keeping erroneous reading counts + ... # in streaming state. A violation is defined when the temperature is above 100. + ... # The input data is a DataFrame with the following schema: + ... # `id: string, temperature: long`. + ... # The output schema and state schema are defined as below. + >>> output_schema = StructType([ + ... StructField("id", StringType(), True), + ... StructField("count", IntegerType(), True) + ... ]) + >>> state_schema = StructType([ + ... StructField("value", IntegerType(), True) + ... ]) + >>> class SimpleStatefulProcessor(StatefulProcessor): + ... def init(self, handle: StatefulProcessorHandle): + ... self.num_violations_state = handle.getValueState("numViolations", state_schema) + ... + ... def handleInputRows(self, key, rows): + ... new_violations = 0 + ... count = 0 + ... exists = self.num_violations_state.exists() + ... if exists: + ... existing_violations_row = self.num_violations_state.get() + ... existing_violations = existing_violations_row[0] + ... else: + ... existing_violations = 0 + ... for row in rows: + ... if row.temperature is not None: + ... count += 1 + ... if row.temperature > 100: + ... new_violations += 1 + ... updated_violations = new_violations + existing_violations + ... self.num_violations_state.update((updated_violations,)) + ... yield Row(id=key, count=count) + ... + ... def close(self) -> None: + ... pass + + Input DataFrame: + +---+-----------+ + | id|temperature| + +---+-----------+ + | 0| 123| + | 0| 23| + | 1| 33| + | 1| 188| + | 1| 88| + +---+-----------+ + + >>> df.groupBy("value").transformWithState(statefulProcessor = + ... SimpleStatefulProcessor(), outputStructType=output_schema, outputMode="Update", + ... timeMode="None") # doctest: +SKIP + + Output DataFrame: + +---+-----+ + | id|count| + +---+-----+ + | 0| 2| + | 1| 3| + +---+-----+ + + Notes + ----- + This function requires a full shuffle. + """ + return self.__transformWithState( + statefulProcessor, + outputStructType, + outputMode, + timeMode, + False, + initialState, + eventTimeColumnName, + ) + + def __transformWithState( + self, + statefulProcessor: StatefulProcessor, + outputStructType: Union[StructType, str], + outputMode: str, + timeMode: str, + usePandas: bool, + initialState: Optional["GroupedData"], + eventTimeColumnName: str, + ) -> "DataFrame": + from pyspark.sql import GroupedData + from pyspark.sql.functions import pandas_udf + from pyspark.sql.streaming.stateful_processor_util import ( + TransformWithStateInPandasUdfUtils, + ) + + assert isinstance(self, GroupedData) + if initialState is not None: + assert isinstance(initialState, GroupedData) + if isinstance(outputStructType, str): + outputStructType = cast(StructType, self._df._session._parse_ddl(outputStructType)) + + df = self._df + udf_util = TransformWithStateInPandasUdfUtils(statefulProcessor, timeMode) + + # explicitly set the type to Any since it could match to various types (literals) + functionType: Any = None + if usePandas and initialState is None: + functionType = PythonEvalType.SQL_TRANSFORM_WITH_STATE_PANDAS_UDF + elif usePandas and initialState is not None: + functionType = PythonEvalType.SQL_TRANSFORM_WITH_STATE_PANDAS_INIT_STATE_UDF + elif not usePandas and initialState is None: + functionType = PythonEvalType.SQL_TRANSFORM_WITH_STATE_PYTHON_ROW_UDF + else: + # not usePandas and initialState is not None + functionType = PythonEvalType.SQL_TRANSFORM_WITH_STATE_PYTHON_ROW_INIT_STATE_UDF + + if initialState is None: + initial_state_java_obj = None + udf = pandas_udf( + udf_util.transformWithStateUDF, # type: ignore + returnType=outputStructType, + functionType=functionType, + ) + else: + initial_state_java_obj = initialState._jgd + udf = pandas_udf( + udf_util.transformWithStateWithInitStateUDF, # type: ignore + returnType=outputStructType, + functionType=functionType, + ) + + udf_column = udf(*[df[col] for col in df.columns]) + + if usePandas: + jdf = self._jgd.transformWithStateInPandas( + udf_column._jc, + self.session._jsparkSession.parseDataType(outputStructType.json()), + outputMode, + timeMode, + initial_state_java_obj, + eventTimeColumnName, + ) + else: + jdf = self._jgd.transformWithStateInPySpark( + udf_column._jc, + self.session._jsparkSession.parseDataType(outputStructType.json()), + outputMode, + timeMode, + initial_state_java_obj, + eventTimeColumnName, + ) + return DataFrame(jdf, self.session) + + def applyInArrow( + self, func: "ArrowGroupedMapFunction", schema: Union[StructType, str] + ) -> "DataFrame": + """ + Maps each group of the current :class:`DataFrame` using an Arrow udf and returns the result + as a `DataFrame`. + + The function can take one of two forms: It can take a `pyarrow.Table` and return a + `pyarrow.Table`, or it can take an iterator of `pyarrow.RecordBatch` and yield + `pyarrow.RecordBatch`. Alternatively each form can take a tuple of `pyarrow.Scalar` + as the first argument in addition to the input type above. For each group, all columns + are passed together in the `pyarrow.Table` or `pyarrow.RecordBatch`, and the returned + `pyarrow.Table` or iterator of `pyarrow.RecordBatch` are combined as a :class:`DataFrame`. + + The `schema` should be a :class:`StructType` describing the schema of the returned + `pyarrow.Table` or `pyarrow.RecordBatch`. The column labels of the returned `pyarrow.Table` + or `pyarrow.RecordBatch` must either match the field names in the defined schema if + specified as strings, or match the field data types by position if not strings, e.g. + integer indices. The length of the returned `pyarrow.Table` or iterator of + `pyarrow.RecordBatch` can be arbitrary. + + .. versionadded:: 4.0.0 + + .. versionchanged:: 4.1.0 + Added support for an iterator of `pyarrow.RecordBatch` API. + + Parameters + ---------- + func : function + a Python native function that either takes a `pyarrow.Table` and outputs a + `pyarrow.Table` or takes an iterator of `pyarrow.RecordBatch` and yields + `pyarrow.RecordBatch`. Additionally, each form can take a tuple of grouping keys + as the first argument, with the `pyarrow.Table` or iterator of `pyarrow.RecordBatch` + as the second argument. + schema : :class:`pyspark.sql.types.DataType` or str + the return type of the `func` in PySpark. The value can be either a + :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> import pyarrow as pa + >>> import pyarrow.compute as pc + >>> df = spark.createDataFrame( + ... [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)], + ... ("id", "v")) + >>> def normalize(table): + ... v = table.column("v") + ... norm = pc.divide(pc.subtract(v, pc.mean(v)), pc.stddev(v, ddof=1)) + ... return table.set_column(1, "v", norm) + >>> df.groupby("id").applyInArrow( + ... normalize, schema="id long, v double" + ... ).sort("id", "v").show() + +---+-------------------+ + | id| v| + +---+-------------------+ + | 1|-0.7071067811865...| + | 1| 0.7071067811865...| + | 2|-0.8320502943378...| + | 2|-0.2773500981126...| + | 2| 1.1094003924504...| + +---+-------------------+ + + The function can also take and return an iterator of `pyarrow.RecordBatch` using type + hints. + + >>> from typing import Iterator, Tuple + >>> df = spark.createDataFrame( + ... [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)], + ... ("id", "v")) + >>> def sum_func( + ... batches: Iterator[pa.RecordBatch] + ... ) -> Iterator[pa.RecordBatch]: + ... total = 0 + ... for batch in batches: + ... total += pc.sum(batch.column("v")).as_py() + ... yield pa.RecordBatch.from_pydict({"v": [total]}) + >>> df.groupby("id").applyInArrow( + ... sum_func, schema="v double" + ... ).sort("v").show() + +----+ + | v| + +----+ + | 3.0| + |18.0| + +----+ + + Alternatively, the user can pass a function that takes two arguments. + In this case, the grouping key(s) will be passed as the first argument and the data will + be passed as the second argument. The grouping key(s) will be passed as a tuple of Arrow + scalars types, e.g., `pyarrow.Int32Scalar` and `pyarrow.FloatScalar`. The data will still + be passed in as a `pyarrow.Table` containing all columns from the original Spark DataFrame. + This is useful when the user does not want to hardcode grouping key(s) in the function. + + >>> df = spark.createDataFrame( + ... [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)], + ... ("id", "v")) + >>> def mean_func(key, table): + ... # key is a tuple of one pyarrow.Int64Scalar, which is the value + ... # of 'id' for the current group + ... mean = pc.mean(table.column("v")) + ... return pa.Table.from_pydict({"id": [key[0].as_py()], "v": [mean.as_py()]}) + >>> df.groupby('id').applyInArrow( + ... mean_func, schema="id long, v double" + ... ).sort("id").show() + +---+---+ + | id| v| + +---+---+ + | 1|1.5| + | 2|6.0| + +---+---+ + + >>> def sum_func(key, table): + ... # key is a tuple of two pyarrow.Int64Scalars, which is the values + ... # of 'id' and 'ceil(df.v / 2)' for the current group + ... sum = pc.sum(table.column("v")) + ... return pa.Table.from_pydict({ + ... "id": [key[0].as_py()], + ... "ceil(v / 2)": [key[1].as_py()], + ... "v": [sum.as_py()] + ... }) + >>> df.groupby(df.id, sf.ceil(df.v / 2)).applyInArrow( + ... sum_func, schema="id long, `ceil(v / 2)` long, v double" + ... ).sort("id", "v").show() + +---+-----------+----+ + | id|ceil(v / 2)| v| + +---+-----------+----+ + | 1| 1| 3.0| + | 2| 2| 3.0| + | 2| 3| 5.0| + | 2| 5|10.0| + +---+-----------+----+ + + >>> def sum_func( + ... key: Tuple[pa.Scalar, ...], batches: Iterator[pa.RecordBatch] + ... ) -> Iterator[pa.RecordBatch]: + ... total = 0 + ... for batch in batches: + ... total += pc.sum(batch.column("v")).as_py() + ... yield pa.RecordBatch.from_pydict({"id": [key[0].as_py()], "v": [total]}) + >>> df.groupby("id").applyInArrow( + ... sum_func, schema="id long, v double" + ... ).sort("id").show() + +---+----+ + | id| v| + +---+----+ + | 1| 3.0| + | 2|18.0| + +---+----+ + + Notes + ----- + This function requires a full shuffle. If using the `pyarrow.Table` API, all data of a + group will be loaded into memory, so the user should be aware of the potential OOM risk + if data is skewed and certain groups are too large to fit in memory, and can use the + iterator of `pyarrow.RecordBatch` API to mitigate this. + + This API is unstable, and for developers. + + See Also + -------- + pyspark.sql.functions.pandas_udf + """ + from pyspark.sql import GroupedData + from pyspark.sql.functions import pandas_udf + from pyspark.sql.pandas.typehints import infer_group_arrow_eval_type_from_func + + assert isinstance(self, GroupedData) + + try: + # Try to infer the eval type from type hints + eval_type = infer_group_arrow_eval_type_from_func(func) + except Exception: + warnings.warn("Cannot infer the eval type from type hints. ", UserWarning) + + if eval_type is None: + eval_type = PythonEvalType.SQL_GROUPED_MAP_ARROW_UDF + + # The usage of the pandas_udf is internal so type checking is disabled. + udf = pandas_udf(func, returnType=schema, functionType=eval_type) # type: ignore[call-overload] + df = self._df + udf_column = udf(*[df[col] for col in df.columns]) + jdf = self._jgd.flatMapGroupsInArrow(udf_column._jc) + return DataFrame(jdf, self.session) + + def cogroup(self, other: "GroupedData") -> "PandasCogroupedOps": + """ + Cogroups this group with another group so that we can run cogrouped operations. + + .. versionadded:: 3.0.0 + + .. versionchanged:: 3.4.0 + Support Spark Connect. + + See :class:`PandasCogroupedOps` for the operations that can be run. + """ + from pyspark.sql import GroupedData + + assert isinstance(self, GroupedData) + + return PandasCogroupedOps(self, other) + + +class PandasCogroupedOps: + """ + A logical grouping of two :class:`GroupedData`, + created by :func:`GroupedData.cogroup`. + + .. versionadded:: 3.0.0 + + .. versionchanged:: 3.4.0 + Support Spark Connect. + """ + + def __init__(self, gd1: "GroupedData", gd2: "GroupedData"): + self._gd1 = gd1 + self._gd2 = gd2 + + def applyInPandas( + self, func: "PandasCogroupedMapFunction", schema: Union["StructType", str] + ) -> "DataFrame": + """ + Applies a function to each cogroup using pandas and returns the result + as a `DataFrame`. + + The function should take two `pandas.DataFrame`\\s and return another + `pandas.DataFrame`. Alternatively, the user can pass a function that takes + a tuple of the grouping key(s) and the two `pandas.DataFrame`\\s. + For each side of the cogroup, all columns are passed together as a + `pandas.DataFrame` to the user-function and the returned `pandas.DataFrame` are combined as + a :class:`DataFrame`. + + The `schema` should be a :class:`StructType` describing the schema of the returned + `pandas.DataFrame`. The column labels of the returned `pandas.DataFrame` must either match + the field names in the defined schema if specified as strings, or match the + field data types by position if not strings, e.g. integer indices. + The length of the returned `pandas.DataFrame` can be arbitrary. + + .. versionadded:: 3.0.0 + + .. versionchanged:: 3.4.0 + Support Spark Connect. + + Parameters + ---------- + func : function + a Python native function that takes two `pandas.DataFrame`\\s, and + outputs a `pandas.DataFrame`, or that takes one tuple (grouping keys) and two + ``pandas.DataFrame``\\s, and outputs a ``pandas.DataFrame``. + schema : :class:`pyspark.sql.types.DataType` or str + the return type of the `func` in PySpark. The value can be either a + :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. + + Examples + -------- + >>> import pandas as pd + >>> df1 = spark.createDataFrame( + ... [(20000101, 1, 1.0), (20000101, 2, 2.0), (20000102, 1, 3.0), (20000102, 2, 4.0)], + ... ("time", "id", "v1")) + >>> df2 = spark.createDataFrame( + ... [(20000101, 1, "x"), (20000101, 2, "y")], + ... ("time", "id", "v2")) + >>> def asof_join(l, r): + ... return pd.merge_asof(l, r, on="time", by="id") + ... + >>> df1.groupby("id").cogroup(df2.groupby("id")).applyInPandas( + ... asof_join, schema="time int, id int, v1 double, v2 string" + ... ).sort("id", "time").show() + +--------+---+---+---+ + | time| id| v1| v2| + +--------+---+---+---+ + |20000101| 1|1.0| x| + |20000102| 1|3.0| x| + |20000101| 2|2.0| y| + |20000102| 2|4.0| y| + +--------+---+---+---+ + + Alternatively, the user can define a function that takes three arguments. In this case, + the grouping key(s) will be passed as the first argument and the data will be passed as the + second and third arguments. The grouping key(s) will be passed as a tuple of numpy data + types, e.g., `numpy.int32` and `numpy.float64`. The data will still be passed in as two + `pandas.DataFrame` containing all columns from the original Spark DataFrames. + + >>> def asof_join(k, l, r): + ... if k == (1,): + ... return pd.merge_asof(l, r, on="time", by="id") + ... else: + ... return pd.DataFrame(columns=['time', 'id', 'v1', 'v2']) + ... + >>> df1.groupby("id").cogroup(df2.groupby("id")).applyInPandas( + ... asof_join, "time int, id int, v1 double, v2 string" + ... ).sort("time").show() + +--------+---+---+---+ + | time| id| v1| v2| + +--------+---+---+---+ + |20000101| 1|1.0| x| + |20000102| 1|3.0| x| + +--------+---+---+---+ + + Notes + ----- + This function requires a full shuffle. All the data of a cogroup will be loaded + into memory, so the user should be aware of the potential OOM risk if data is skewed + and certain groups are too large to fit in memory. + + See Also + -------- + pyspark.sql.functions.pandas_udf + """ + from pyspark.sql.pandas.functions import pandas_udf + + # The usage of the pandas_udf is internal so type checking is disabled. + udf = pandas_udf( + func, returnType=schema, functionType=PythonEvalType.SQL_COGROUPED_MAP_PANDAS_UDF + ) # type: ignore[call-overload] + + all_cols = self._extract_cols(self._gd1) + self._extract_cols(self._gd2) + udf_column = udf(*all_cols) + jdf = self._gd1._jgd.flatMapCoGroupsInPandas(self._gd2._jgd, udf_column._jc) + return DataFrame(jdf, self._gd1.session) + + def applyInArrow( + self, func: "ArrowCogroupedMapFunction", schema: Union[StructType, str] + ) -> "DataFrame": + """ + Applies a function to each cogroup using Arrow and returns the result + as a `DataFrame`. + + The function should take two `pyarrow.Table`\\s and return another + `pyarrow.Table`. Alternatively, the user can pass a function that takes + a tuple of `pyarrow.Scalar` grouping key(s) and the two `pyarrow.Table`\\s. + For each side of the cogroup, all columns are passed together as a + `pyarrow.Table` to the user-function and the returned `pyarrow.Table` are combined as + a :class:`DataFrame`. + + The `schema` should be a :class:`StructType` describing the schema of the returned + `pyarrow.Table`. The column labels of the returned `pyarrow.Table` must either match + the field names in the defined schema if specified as strings, or match the + field data types by position if not strings, e.g. integer indices. + The length of the returned `pyarrow.Table` can be arbitrary. + + .. versionadded:: 4.0.0 + + Parameters + ---------- + func : function + a Python native function that takes two `pyarrow.Table`\\s, and + outputs a `pyarrow.Table`, or that takes one tuple (grouping keys) and two + ``pyarrow.Table``\\s, and outputs a ``pyarrow.Table``. + schema : :class:`pyspark.sql.types.DataType` or str + the return type of the `func` in PySpark. The value can be either a + :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. + + Examples + -------- + >>> import pyarrow as pa + >>> df1 = spark.createDataFrame([(1, 1.0), (2, 2.0), (1, 3.0), (2, 4.0)], ("id", "v1")) + >>> df2 = spark.createDataFrame([(1, "x"), (2, "y")], ("id", "v2")) + >>> def summarize(l, r): + ... return pa.Table.from_pydict({ + ... "left": [l.num_rows], + ... "right": [r.num_rows] + ... }) + >>> df1.groupby("id").cogroup(df2.groupby("id")).applyInArrow( + ... summarize, schema="left long, right long" + ... ).show() + +----+-----+ + |left|right| + +----+-----+ + | 2| 1| + | 2| 1| + +----+-----+ + + Alternatively, the user can define a function that takes three arguments. In this case, + the grouping key(s) will be passed as the first argument and the data will be passed as the + second and third arguments. The grouping key(s) will be passed as a tuple of Arrow scalars + types, e.g., `pyarrow.Int32Scalar` and `pyarrow.FloatScalar`. The data will still be passed + in as two `pyarrow.Table`\\s containing all columns from the original Spark DataFrames. + + >>> def summarize(key, l, r): + ... return pa.Table.from_pydict({ + ... "key": [key[0].as_py()], + ... "left": [l.num_rows], + ... "right": [r.num_rows] + ... }) + >>> df1.groupby("id").cogroup(df2.groupby("id")).applyInArrow( + ... summarize, schema="key long, left long, right long" + ... ).sort("key").show() + +---+----+-----+ + |key|left|right| + +---+----+-----+ + | 1| 2| 1| + | 2| 2| 1| + +---+----+-----+ + + Notes + ----- + This function requires a full shuffle. All the data of a cogroup will be loaded + into memory, so the user should be aware of the potential OOM risk if data is skewed + and certain groups are too large to fit in memory. + + This API is unstable, and for developers. + + See Also + -------- + pyspark.sql.functions.pandas_udf + """ + from pyspark.sql.pandas.functions import pandas_udf + + # The usage of the pandas_udf is internal so type checking is disabled. + udf = pandas_udf( + func, returnType=schema, functionType=PythonEvalType.SQL_COGROUPED_MAP_ARROW_UDF + ) # type: ignore[call-overload] + + all_cols = self._extract_cols(self._gd1) + self._extract_cols(self._gd2) + udf_column = udf(*all_cols) + jdf = self._gd1._jgd.flatMapCoGroupsInArrow(self._gd2._jgd, udf_column._jc) + return DataFrame(jdf, self._gd1.session) + + @staticmethod + def _extract_cols(gd: "GroupedData") -> List[Column]: + df = gd._df + return [df[col] for col in df.columns] + + +def _test() -> None: + import doctest + from pyspark.sql import SparkSession + import pyspark.sql.pandas.group_ops + from pyspark.testing.utils import have_pandas, have_pyarrow + + globs = pyspark.sql.pandas.group_ops.__dict__.copy() + + if not have_pandas or not have_pyarrow: + del pyspark.sql.pandas.group_ops.PandasGroupedOpsMixin.apply.__doc__ + del pyspark.sql.pandas.group_ops.PandasGroupedOpsMixin.applyInPandas.__doc__ + del pyspark.sql.pandas.group_ops.PandasCogroupedOps.applyInPandas.__doc__ + + if not have_pyarrow: + del pyspark.sql.pandas.group_ops.PandasGroupedOpsMixin.applyInArrow.__doc__ + del pyspark.sql.pandas.group_ops.PandasCogroupedOps.applyInArrow.__doc__ + + spark = SparkSession.builder.master("local[4]").appName("sql.pandas.group tests").getOrCreate() + globs["spark"] = spark + failure_count, test_count = doctest.testmod( + pyspark.sql.pandas.group_ops, + globs=globs, + optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF, + ) + spark.stop() + if failure_count: + sys.exit(-1) + + +if __name__ == "__main__": + _test() diff --git a/python/pyspark/sql/pandas/group_ops.pyi b/python/pyspark/sql/pandas/group_ops.pyi deleted file mode 100644 index 6ceb30f3..00000000 --- a/python/pyspark/sql/pandas/group_ops.pyi +++ /dev/null @@ -1,913 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Generated by scripts/gen_type_stubs.py from python/pyspark/sql/pandas/group_ops.py of Apache Spark v4.3.0: -# do not edit. Type checkers read this stub instead of the module of the drop-in. - -from typing import List, Optional, Union -from pyspark.sql.column import Column -from pyspark.sql.dataframe import DataFrame -from pyspark.sql.streaming.state import GroupStateTimeout as GroupStateTimeout -from pyspark.sql.streaming.stateful_processor import StatefulProcessor as StatefulProcessor -from pyspark.sql.types import StructType -from pyspark.sql.pandas._typing import GroupedMapPandasUserDefinedFunction, PandasGroupedMapFunction, PandasGroupedMapFunctionWithState, PandasCogroupedMapFunction, ArrowGroupedMapFunction, ArrowCogroupedMapFunction -from pyspark.sql.group import GroupedData - -class PandasGroupedOpsMixin: - """Min-in for pandas grouped operations. Currently, only :class:`GroupedData` - can use this class. - """ - def apply(self, udf: 'GroupedMapPandasUserDefinedFunction') -> 'DataFrame': - """It is an alias of :meth:`pyspark.sql.GroupedData.applyInPandas`; however, it takes a - :meth:`pyspark.sql.functions.pandas_udf` whereas - :meth:`pyspark.sql.GroupedData.applyInPandas` takes a Python native function. - - .. versionadded:: 2.3.0 - - .. versionchanged:: 3.4.0 - Support Spark Connect. - - Parameters - ---------- - udf : :func:`pyspark.sql.functions.pandas_udf` - a grouped map user-defined function returned by - :func:`pyspark.sql.functions.pandas_udf`. - - Notes - ----- - It is preferred to use :meth:`pyspark.sql.GroupedData.applyInPandas` over this - API. This API will be deprecated in the future releases. - - Examples - -------- - >>> from pyspark.sql.functions import pandas_udf, PandasUDFType - >>> df = spark.createDataFrame( - ... [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)], - ... ("id", "v")) - >>> @pandas_udf("id long, v double", PandasUDFType.GROUPED_MAP) - ... def normalize(pdf): - ... v = pdf.v - ... return pdf.assign(v=(v - v.mean()) / v.std()) - ... - >>> df.groupby("id").apply(normalize).sort("id", "v").show() - +---+-------------------+ - | id| v| - +---+-------------------+ - | 1|-0.7071067811865...| - | 1| 0.7071067811865...| - | 2|-0.8320502943378...| - | 2|-0.2773500981126...| - | 2| 1.1094003924504...| - +---+-------------------+ - - See Also - -------- - pyspark.sql.functions.pandas_udf - """ - def applyInPandas(self, func: 'PandasGroupedMapFunction', schema: Union['StructType', str]) -> 'DataFrame': - """Maps each group of the current :class:`DataFrame` using a pandas udf and returns the result - as a `DataFrame`. - - The function can take one of two forms: It can take a `pandas.DataFrame` and return a - `pandas.DataFrame`, or it can take an iterator of `pandas.DataFrame` and yield - `pandas.DataFrame`. Alternatively each form can take a tuple of grouping keys - as the first argument in addition to the input type above. - For each group, all columns are passed together as a `pandas.DataFrame` or iterator of - `pandas.DataFrame`, and the returned `pandas.DataFrame` or iterator of `pandas.DataFrame` - are combined as a :class:`DataFrame`. - - The `schema` should be a :class:`StructType` describing the schema of the returned - `pandas.DataFrame`. The column labels of the returned `pandas.DataFrame` must either match - the field names in the defined schema if specified as strings, or match the - field data types by position if not strings, e.g. integer indices. - The length of the returned `pandas.DataFrame` can be arbitrary. - - .. versionadded:: 3.0.0 - - .. versionchanged:: 3.4.0 - Support Spark Connect. - - .. versionchanged:: 4.1.0 - Added support for an iterator of `pandas.DataFrame` API. - - Parameters - ---------- - func : function - a Python native function that either takes a `pandas.DataFrame` and outputs a - `pandas.DataFrame` or takes an iterator of `pandas.DataFrame` and yields - `pandas.DataFrame`. Additionally, each form can take a tuple of grouping keys - as the first argument, with the `pandas.DataFrame` or iterator of `pandas.DataFrame` - as the second argument. - schema : :class:`pyspark.sql.types.DataType` or str - the return type of the `func` in PySpark. The value can be either a - :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. - - Examples - -------- - >>> import pandas as pd - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)], - ... ("id", "v")) - >>> def normalize(pdf): - ... v = pdf.v - ... return pdf.assign(v=(v - v.mean()) / v.std()) - ... - >>> df.groupby("id").applyInPandas( - ... normalize, schema="id long, v double" - ... ).sort("id", "v").show() - +---+-------------------+ - | id| v| - +---+-------------------+ - | 1|-0.7071067811865...| - | 1| 0.7071067811865...| - | 2|-0.8320502943378...| - | 2|-0.2773500981126...| - | 2| 1.1094003924504...| - +---+-------------------+ - - Alternatively, the user can pass a function that takes two arguments. - In this case, the grouping key(s) will be passed as the first argument and the data will - be passed as the second argument. The grouping key(s) will be passed as a tuple of numpy - data types, e.g., `numpy.int32` and `numpy.float64`. The data will still be passed in - as a `pandas.DataFrame` containing all columns from the original Spark DataFrame. - This is useful when the user does not want to hardcode grouping key(s) in the function. - - >>> df = spark.createDataFrame( - ... [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)], - ... ("id", "v")) - >>> def mean_func(key, pdf): - ... # key is a tuple of one numpy.int64, which is the value - ... # of 'id' for the current group - ... return pd.DataFrame([key + (pdf.v.mean(),)]) - ... - >>> df.groupby("id").applyInPandas( - ... mean_func, schema="id long, v double" - ... ).sort("id").show() - +---+---+ - | id| v| - +---+---+ - | 1|1.5| - | 2|6.0| - +---+---+ - - >>> def sum_func(key, pdf): - ... # key is a tuple of two numpy.int64s, which is the values - ... # of 'id' and 'ceil(df.v / 2)' for the current group - ... return pd.DataFrame([key + (pdf.v.sum(),)]) - ... - >>> df.groupby(df.id, sf.ceil(df.v / 2)).applyInPandas( - ... sum_func, schema="id long, `ceil(v / 2)` long, v double" - ... ).sort("id", "v").show() - +---+-----------+----+ - | id|ceil(v / 2)| v| - +---+-----------+----+ - | 1| 1| 3.0| - | 2| 2| 3.0| - | 2| 3| 5.0| - | 2| 5|10.0| - +---+-----------+----+ - - The function can also take and return an iterator of `pandas.DataFrame` using type - hints. - - >>> from typing import Iterator - >>> df = spark.createDataFrame( - ... [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)], - ... ("id", "v")) - >>> def filter_func( - ... batches: Iterator[pd.DataFrame] - ... ) -> Iterator[pd.DataFrame]: - ... for batch in batches: - ... # Process and yield each batch independently - ... filtered = batch[batch['v'] > 2.0] - ... if not filtered.empty: - ... yield filtered[['v']] - >>> df.groupby("id").applyInPandas( - ... filter_func, schema="v double" - ... ).sort("v").show() - +----+ - | v| - +----+ - | 3.0| - | 5.0| - |10.0| - +----+ - - Alternatively, the user can pass a function that takes two arguments. - In this case, the grouping key(s) will be passed as the first argument and the data will - be passed as the second argument. The grouping key(s) will be passed as a tuple of numpy - data types. The data will still be passed in as an iterator of `pandas.DataFrame`. - - >>> from typing import Iterator, Tuple, Any - >>> def transform_func( - ... key: Tuple[Any, ...], batches: Iterator[pd.DataFrame] - ... ) -> Iterator[pd.DataFrame]: - ... for batch in batches: - ... # Yield transformed results for each batch - ... result = batch.assign(id=key[0], v_doubled=batch['v'] * 2) - ... yield result[['id', 'v_doubled']] - >>> df.groupby("id").applyInPandas( - ... transform_func, schema="id long, v_doubled double" - ... ).sort("id", "v_doubled").show() - +---+---------+ - | id|v_doubled| - +---+---------+ - | 1| 2.0| - | 1| 4.0| - | 2| 6.0| - | 2| 10.0| - | 2| 20.0| - +---+---------+ - - Notes - ----- - This function requires a full shuffle. If using the `pandas.DataFrame` API, all data of a - group will be loaded into memory, so the user should be aware of the potential OOM risk if - data is skewed and certain groups are too large to fit in memory, and can use the - iterator of `pandas.DataFrame` API to mitigate this. - - See Also - -------- - pyspark.sql.functions.pandas_udf - """ - def applyInPandasWithState(self, func: 'PandasGroupedMapFunctionWithState', outputStructType: Union[StructType, str], stateStructType: Union[StructType, str], outputMode: str, timeoutConf: str) -> 'DataFrame': - """Applies the given function to each group of data, while maintaining a user-defined - per-group state. The result Dataset will represent the flattened record returned by the - function. - - For a streaming :class:`DataFrame`, the function will be invoked first for all input groups - and then for all timed out states where the input data is set to be empty. Updates to each - group's state will be saved across invocations. - - The function should take parameters (key, Iterator[`pandas.DataFrame`], state) and - return another Iterator[`pandas.DataFrame`]. The grouping key(s) will be passed as a tuple - of numpy data types, e.g., `numpy.int32` and `numpy.float64`. The state will be passed as - :class:`pyspark.sql.streaming.state.GroupState`. - - For each group, all columns are passed together as `pandas.DataFrame` to the user-function, - and the returned `pandas.DataFrame` across all invocations are combined as a - :class:`DataFrame`. Note that the user function should not make a guess of the number of - elements in the iterator. To process all data, the user function needs to iterate all - elements and process them. On the other hand, the user function is not strictly required to - iterate through all elements in the iterator if it intends to read a part of data. - - The `outputStructType` should be a :class:`StructType` describing the schema of all - elements in the returned value, `pandas.DataFrame`. The column labels of all elements in - returned `pandas.DataFrame` must either match the field names in the defined schema if - specified as strings, or match the field data types by position if not strings, - e.g. integer indices. - - The `stateStructType` should be :class:`StructType` describing the schema of the - user-defined state. The value of the state will be presented as a tuple, as well as the - update should be performed with the tuple. The corresponding Python types for - :class:DataType are supported. Please refer to the page - https://spark.apache.org/docs/latest/sql-ref-datatypes.html (Python tab). - - The size of each `pandas.DataFrame` in both the input and output can be arbitrary. The - number of `pandas.DataFrame` in both the input and output can also be arbitrary. - - .. versionadded:: 3.4.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - func : function - a Python native function to be called on every group. It should take parameters - (key, Iterator[`pandas.DataFrame`], state) and return Iterator[`pandas.DataFrame`]. - Note that the type of the key is tuple and the type of the state is - :class:`pyspark.sql.streaming.state.GroupState`. - outputStructType : :class:`pyspark.sql.types.DataType` or str - the type of the output records. The value can be either a - :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. - stateStructType : :class:`pyspark.sql.types.DataType` or str - the type of the user-defined state. The value can be either a - :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. - outputMode : str - the output mode of the function. - timeoutConf : str - timeout configuration for groups that do not receive data for a while. valid values - are defined in :class:`pyspark.sql.streaming.state.GroupStateTimeout`. - - Examples - -------- - >>> import pandas as pd # doctest: +SKIP - >>> from pyspark.sql.streaming.state import GroupStateTimeout - >>> def count_fn(key, pdf_iter, state): - ... assert isinstance(state, GroupStateImpl) - ... total_len = 0 - ... for pdf in pdf_iter: - ... total_len += len(pdf) - ... state.update((total_len,)) - ... yield pd.DataFrame({"id": [key[0]], "countAsString": [str(total_len)]}) - ... - >>> df.groupby("id").applyInPandasWithState( - ... count_fn, outputStructType="id long, countAsString string", - ... stateStructType="len long", outputMode="Update", - ... timeoutConf=GroupStateTimeout.NoTimeout) # doctest: +SKIP - - Notes - ----- - This function requires a full shuffle. - """ - def transformWithStateInPandas(self, statefulProcessor: StatefulProcessor, outputStructType: Union[StructType, str], outputMode: str, timeMode: str, initialState: Optional['GroupedData']=None, eventTimeColumnName: str="") -> 'DataFrame': - """Invokes methods defined in the stateful processor used in arbitrary state API v2. It - requires protobuf, pandas and pyarrow as dependencies to process input/state data. We - allow the user to act on per-group set of input rows along with keyed state and the user - can choose to output/return 0 or more rows. - - For a streaming dataframe, we will repeatedly invoke the interface methods for new rows - in each trigger and the user's state/state variables will be stored persistently across - invocations. - - The `statefulProcessor` should be a Python class that implements the interface defined in - :class:`StatefulProcessor`. - - The `outputStructType` should be a :class:`StructType` describing the schema of all - elements in the returned value, `pandas.DataFrame`. The column labels of all elements in - returned `pandas.DataFrame` must either match the field names in the defined schema if - specified as strings, or match the field data types by position if not strings, - e.g. integer indices. - - The size of each `pandas.DataFrame` in both the input and output can be arbitrary. The - number of `pandas.DataFrame` in both the input and output can also be arbitrary. - - .. versionadded:: 4.0.0 - - Parameters - ---------- - statefulProcessor : :class:`pyspark.sql.streaming.stateful_processor.StatefulProcessor` - Instance of StatefulProcessor whose functions will be invoked by the operator. - outputStructType : :class:`pyspark.sql.types.DataType` or str - The type of the output records. The value can be either a - :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. - outputMode : str - The output mode of the stateful processor. - timeMode : str - The time mode semantics of the stateful processor for timers and TTL. - initialState : :class:`pyspark.sql.GroupedData` - Optional. The grouped dataframe as initial states used for initialization - of state variables in the first batch. - - Examples - -------- - >>> from typing import Iterator - ... - >>> import pandas as pd # doctest: +SKIP - ... - >>> from pyspark.sql import Row - >>> from pyspark.sql.functions import col, split - >>> from pyspark.sql.streaming import StatefulProcessor, StatefulProcessorHandle - >>> from pyspark.sql.types import IntegerType, LongType, StringType, StructField, StructType - ... - >>> spark.conf.set("spark.sql.streaming.stateStore.providerClass", - ... "org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider") - ... # Below is a simple example to find erroneous sensors from temperature sensor data. The - ... # processor returns a count of total readings, while keeping erroneous reading counts - ... # in streaming state. A violation is defined when the temperature is above 100. - ... # The input data is a DataFrame with the following schema: - ... # `id: string, temperature: long`. - ... # The output schema and state schema are defined as below. - >>> output_schema = StructType([ - ... StructField("id", StringType(), True), - ... StructField("count", IntegerType(), True) - ... ]) - >>> state_schema = StructType([ - ... StructField("value", IntegerType(), True) - ... ]) - >>> class SimpleStatefulProcessor(StatefulProcessor): - ... def init(self, handle: StatefulProcessorHandle): - ... self.num_violations_state = handle.getValueState("numViolations", state_schema) - ... - ... def handleInputRows(self, key, rows): - ... new_violations = 0 - ... count = 0 - ... exists = self.num_violations_state.exists() - ... if exists: - ... existing_violations_row = self.num_violations_state.get() - ... existing_violations = existing_violations_row[0] - ... else: - ... existing_violations = 0 - ... for pdf in rows: - ... pdf_count = pdf.count() - ... count += pdf_count.get('temperature') - ... violations_pdf = pdf.loc[pdf['temperature'] > 100] - ... new_violations += violations_pdf.count().get('temperature') - ... updated_violations = new_violations + existing_violations - ... self.num_violations_state.update((updated_violations,)) - ... yield pd.DataFrame({'id': key, 'count': count}) - ... - ... def close(self) -> None: - ... pass - - Input DataFrame: - +---+-----------+ - | id|temperature| - +---+-----------+ - | 0| 123| - | 0| 23| - | 1| 33| - | 1| 188| - | 1| 88| - +---+-----------+ - - >>> df.groupBy("value").transformWithStateInPandas(statefulProcessor = - ... SimpleStatefulProcessor(), outputStructType=output_schema, outputMode="Update", - ... timeMode="None") # doctest: +SKIP - - Output DataFrame: - +---+-----+ - | id|count| - +---+-----+ - | 0| 2| - | 1| 3| - +---+-----+ - - Notes - ----- - This function requires a full shuffle. - """ - def transformWithState(self, statefulProcessor: StatefulProcessor, outputStructType: Union[StructType, str], outputMode: str, timeMode: str, initialState: Optional['GroupedData']=None, eventTimeColumnName: str="") -> 'DataFrame': - """Invokes methods defined in the stateful processor used in arbitrary state API v2. It - requires protobuf and pyarrow as dependencies to process input/state data. We allow - the user to act on per-group set of input rows along with keyed state and the user - can choose to output/return 0 or more rows. - - For a streaming dataframe, we will repeatedly invoke the interface methods for new rows - in each trigger and the user's state/state variables will be stored persistently across - invocations. - - The `statefulProcessor` should be a Python class that implements the interface defined in - :class:`StatefulProcessor`. - - The `outputStructType` should be a :class:`StructType` describing the schema of all - elements in the returned value, `Row`. The column labels of all elements in - returned `Row` must either match the field names in the defined schema. - - The number of `Row`s in the iterator in both the input and output can be arbitrary. - - .. versionadded:: 4.1.0 - - Parameters - ---------- - statefulProcessor : :class:`pyspark.sql.streaming.stateful_processor.StatefulProcessor` - Instance of StatefulProcessor whose functions will be invoked by the operator. - outputStructType : :class:`pyspark.sql.types.DataType` or str - The type of the output records. The value can be either a - :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. - outputMode : str - The output mode of the stateful processor. - timeMode : str - The time mode semantics of the stateful processor for timers and TTL. - initialState : :class:`pyspark.sql.GroupedData` - Optional. The grouped dataframe as initial states used for initialization - of state variables in the first batch. - - Examples - -------- - >>> from typing import Iterator - ... - >>> from pyspark.sql import Row - >>> from pyspark.sql.functions import col, split - >>> from pyspark.sql.streaming import StatefulProcessor, StatefulProcessorHandle - >>> from pyspark.sql.types import IntegerType, LongType, StringType, StructField, StructType - ... - >>> spark.conf.set("spark.sql.streaming.stateStore.providerClass", - ... "org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider") - ... # Below is a simple example to find erroneous sensors from temperature sensor data. The - ... # processor returns a count of total readings, while keeping erroneous reading counts - ... # in streaming state. A violation is defined when the temperature is above 100. - ... # The input data is a DataFrame with the following schema: - ... # `id: string, temperature: long`. - ... # The output schema and state schema are defined as below. - >>> output_schema = StructType([ - ... StructField("id", StringType(), True), - ... StructField("count", IntegerType(), True) - ... ]) - >>> state_schema = StructType([ - ... StructField("value", IntegerType(), True) - ... ]) - >>> class SimpleStatefulProcessor(StatefulProcessor): - ... def init(self, handle: StatefulProcessorHandle): - ... self.num_violations_state = handle.getValueState("numViolations", state_schema) - ... - ... def handleInputRows(self, key, rows): - ... new_violations = 0 - ... count = 0 - ... exists = self.num_violations_state.exists() - ... if exists: - ... existing_violations_row = self.num_violations_state.get() - ... existing_violations = existing_violations_row[0] - ... else: - ... existing_violations = 0 - ... for row in rows: - ... if row.temperature is not None: - ... count += 1 - ... if row.temperature > 100: - ... new_violations += 1 - ... updated_violations = new_violations + existing_violations - ... self.num_violations_state.update((updated_violations,)) - ... yield Row(id=key, count=count) - ... - ... def close(self) -> None: - ... pass - - Input DataFrame: - +---+-----------+ - | id|temperature| - +---+-----------+ - | 0| 123| - | 0| 23| - | 1| 33| - | 1| 188| - | 1| 88| - +---+-----------+ - - >>> df.groupBy("value").transformWithState(statefulProcessor = - ... SimpleStatefulProcessor(), outputStructType=output_schema, outputMode="Update", - ... timeMode="None") # doctest: +SKIP - - Output DataFrame: - +---+-----+ - | id|count| - +---+-----+ - | 0| 2| - | 1| 3| - +---+-----+ - - Notes - ----- - This function requires a full shuffle. - """ - def __transformWithState(self, statefulProcessor: StatefulProcessor, outputStructType: Union[StructType, str], outputMode: str, timeMode: str, usePandas: bool, initialState: Optional['GroupedData'], eventTimeColumnName: str) -> 'DataFrame': ... - def applyInArrow(self, func: 'ArrowGroupedMapFunction', schema: Union[StructType, str]) -> 'DataFrame': - """Maps each group of the current :class:`DataFrame` using an Arrow udf and returns the result - as a `DataFrame`. - - The function can take one of two forms: It can take a `pyarrow.Table` and return a - `pyarrow.Table`, or it can take an iterator of `pyarrow.RecordBatch` and yield - `pyarrow.RecordBatch`. Alternatively each form can take a tuple of `pyarrow.Scalar` - as the first argument in addition to the input type above. For each group, all columns - are passed together in the `pyarrow.Table` or `pyarrow.RecordBatch`, and the returned - `pyarrow.Table` or iterator of `pyarrow.RecordBatch` are combined as a :class:`DataFrame`. - - The `schema` should be a :class:`StructType` describing the schema of the returned - `pyarrow.Table` or `pyarrow.RecordBatch`. The column labels of the returned `pyarrow.Table` - or `pyarrow.RecordBatch` must either match the field names in the defined schema if - specified as strings, or match the field data types by position if not strings, e.g. - integer indices. The length of the returned `pyarrow.Table` or iterator of - `pyarrow.RecordBatch` can be arbitrary. - - .. versionadded:: 4.0.0 - - .. versionchanged:: 4.1.0 - Added support for an iterator of `pyarrow.RecordBatch` API. - - Parameters - ---------- - func : function - a Python native function that either takes a `pyarrow.Table` and outputs a - `pyarrow.Table` or takes an iterator of `pyarrow.RecordBatch` and yields - `pyarrow.RecordBatch`. Additionally, each form can take a tuple of grouping keys - as the first argument, with the `pyarrow.Table` or iterator of `pyarrow.RecordBatch` - as the second argument. - schema : :class:`pyspark.sql.types.DataType` or str - the return type of the `func` in PySpark. The value can be either a - :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> import pyarrow as pa - >>> import pyarrow.compute as pc - >>> df = spark.createDataFrame( - ... [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)], - ... ("id", "v")) - >>> def normalize(table): - ... v = table.column("v") - ... norm = pc.divide(pc.subtract(v, pc.mean(v)), pc.stddev(v, ddof=1)) - ... return table.set_column(1, "v", norm) - >>> df.groupby("id").applyInArrow( - ... normalize, schema="id long, v double" - ... ).sort("id", "v").show() - +---+-------------------+ - | id| v| - +---+-------------------+ - | 1|-0.7071067811865...| - | 1| 0.7071067811865...| - | 2|-0.8320502943378...| - | 2|-0.2773500981126...| - | 2| 1.1094003924504...| - +---+-------------------+ - - The function can also take and return an iterator of `pyarrow.RecordBatch` using type - hints. - - >>> from typing import Iterator, Tuple - >>> df = spark.createDataFrame( - ... [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)], - ... ("id", "v")) - >>> def sum_func( - ... batches: Iterator[pa.RecordBatch] - ... ) -> Iterator[pa.RecordBatch]: - ... total = 0 - ... for batch in batches: - ... total += pc.sum(batch.column("v")).as_py() - ... yield pa.RecordBatch.from_pydict({"v": [total]}) - >>> df.groupby("id").applyInArrow( - ... sum_func, schema="v double" - ... ).sort("v").show() - +----+ - | v| - +----+ - | 3.0| - |18.0| - +----+ - - Alternatively, the user can pass a function that takes two arguments. - In this case, the grouping key(s) will be passed as the first argument and the data will - be passed as the second argument. The grouping key(s) will be passed as a tuple of Arrow - scalars types, e.g., `pyarrow.Int32Scalar` and `pyarrow.FloatScalar`. The data will still - be passed in as a `pyarrow.Table` containing all columns from the original Spark DataFrame. - This is useful when the user does not want to hardcode grouping key(s) in the function. - - >>> df = spark.createDataFrame( - ... [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)], - ... ("id", "v")) - >>> def mean_func(key, table): - ... # key is a tuple of one pyarrow.Int64Scalar, which is the value - ... # of 'id' for the current group - ... mean = pc.mean(table.column("v")) - ... return pa.Table.from_pydict({"id": [key[0].as_py()], "v": [mean.as_py()]}) - >>> df.groupby('id').applyInArrow( - ... mean_func, schema="id long, v double" - ... ).sort("id").show() - +---+---+ - | id| v| - +---+---+ - | 1|1.5| - | 2|6.0| - +---+---+ - - >>> def sum_func(key, table): - ... # key is a tuple of two pyarrow.Int64Scalars, which is the values - ... # of 'id' and 'ceil(df.v / 2)' for the current group - ... sum = pc.sum(table.column("v")) - ... return pa.Table.from_pydict({ - ... "id": [key[0].as_py()], - ... "ceil(v / 2)": [key[1].as_py()], - ... "v": [sum.as_py()] - ... }) - >>> df.groupby(df.id, sf.ceil(df.v / 2)).applyInArrow( - ... sum_func, schema="id long, `ceil(v / 2)` long, v double" - ... ).sort("id", "v").show() - +---+-----------+----+ - | id|ceil(v / 2)| v| - +---+-----------+----+ - | 1| 1| 3.0| - | 2| 2| 3.0| - | 2| 3| 5.0| - | 2| 5|10.0| - +---+-----------+----+ - - >>> def sum_func( - ... key: Tuple[pa.Scalar, ...], batches: Iterator[pa.RecordBatch] - ... ) -> Iterator[pa.RecordBatch]: - ... total = 0 - ... for batch in batches: - ... total += pc.sum(batch.column("v")).as_py() - ... yield pa.RecordBatch.from_pydict({"id": [key[0].as_py()], "v": [total]}) - >>> df.groupby("id").applyInArrow( - ... sum_func, schema="id long, v double" - ... ).sort("id").show() - +---+----+ - | id| v| - +---+----+ - | 1| 3.0| - | 2|18.0| - +---+----+ - - Notes - ----- - This function requires a full shuffle. If using the `pyarrow.Table` API, all data of a - group will be loaded into memory, so the user should be aware of the potential OOM risk - if data is skewed and certain groups are too large to fit in memory, and can use the - iterator of `pyarrow.RecordBatch` API to mitigate this. - - This API is unstable, and for developers. - - See Also - -------- - pyspark.sql.functions.pandas_udf - """ - def cogroup(self, other: 'GroupedData') -> 'PandasCogroupedOps': - """Cogroups this group with another group so that we can run cogrouped operations. - - .. versionadded:: 3.0.0 - - .. versionchanged:: 3.4.0 - Support Spark Connect. - - See :class:`PandasCogroupedOps` for the operations that can be run. - """ - -class PandasCogroupedOps: - """A logical grouping of two :class:`GroupedData`, - created by :func:`GroupedData.cogroup`. - - .. versionadded:: 3.0.0 - - .. versionchanged:: 3.4.0 - Support Spark Connect. - """ - def __init__(self, gd1: 'GroupedData', gd2: 'GroupedData'): - self._gd1 = gd1 - self._gd2 = gd2 - def applyInPandas(self, func: 'PandasCogroupedMapFunction', schema: Union['StructType', str]) -> 'DataFrame': - r"""Applies a function to each cogroup using pandas and returns the result - as a `DataFrame`. - - The function should take two `pandas.DataFrame`\s and return another - `pandas.DataFrame`. Alternatively, the user can pass a function that takes - a tuple of the grouping key(s) and the two `pandas.DataFrame`\s. - For each side of the cogroup, all columns are passed together as a - `pandas.DataFrame` to the user-function and the returned `pandas.DataFrame` are combined as - a :class:`DataFrame`. - - The `schema` should be a :class:`StructType` describing the schema of the returned - `pandas.DataFrame`. The column labels of the returned `pandas.DataFrame` must either match - the field names in the defined schema if specified as strings, or match the - field data types by position if not strings, e.g. integer indices. - The length of the returned `pandas.DataFrame` can be arbitrary. - - .. versionadded:: 3.0.0 - - .. versionchanged:: 3.4.0 - Support Spark Connect. - - Parameters - ---------- - func : function - a Python native function that takes two `pandas.DataFrame`\s, and - outputs a `pandas.DataFrame`, or that takes one tuple (grouping keys) and two - ``pandas.DataFrame``\s, and outputs a ``pandas.DataFrame``. - schema : :class:`pyspark.sql.types.DataType` or str - the return type of the `func` in PySpark. The value can be either a - :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. - - Examples - -------- - >>> import pandas as pd - >>> df1 = spark.createDataFrame( - ... [(20000101, 1, 1.0), (20000101, 2, 2.0), (20000102, 1, 3.0), (20000102, 2, 4.0)], - ... ("time", "id", "v1")) - >>> df2 = spark.createDataFrame( - ... [(20000101, 1, "x"), (20000101, 2, "y")], - ... ("time", "id", "v2")) - >>> def asof_join(l, r): - ... return pd.merge_asof(l, r, on="time", by="id") - ... - >>> df1.groupby("id").cogroup(df2.groupby("id")).applyInPandas( - ... asof_join, schema="time int, id int, v1 double, v2 string" - ... ).sort("id", "time").show() - +--------+---+---+---+ - | time| id| v1| v2| - +--------+---+---+---+ - |20000101| 1|1.0| x| - |20000102| 1|3.0| x| - |20000101| 2|2.0| y| - |20000102| 2|4.0| y| - +--------+---+---+---+ - - Alternatively, the user can define a function that takes three arguments. In this case, - the grouping key(s) will be passed as the first argument and the data will be passed as the - second and third arguments. The grouping key(s) will be passed as a tuple of numpy data - types, e.g., `numpy.int32` and `numpy.float64`. The data will still be passed in as two - `pandas.DataFrame` containing all columns from the original Spark DataFrames. - - >>> def asof_join(k, l, r): - ... if k == (1,): - ... return pd.merge_asof(l, r, on="time", by="id") - ... else: - ... return pd.DataFrame(columns=['time', 'id', 'v1', 'v2']) - ... - >>> df1.groupby("id").cogroup(df2.groupby("id")).applyInPandas( - ... asof_join, "time int, id int, v1 double, v2 string" - ... ).sort("time").show() - +--------+---+---+---+ - | time| id| v1| v2| - +--------+---+---+---+ - |20000101| 1|1.0| x| - |20000102| 1|3.0| x| - +--------+---+---+---+ - - Notes - ----- - This function requires a full shuffle. All the data of a cogroup will be loaded - into memory, so the user should be aware of the potential OOM risk if data is skewed - and certain groups are too large to fit in memory. - - See Also - -------- - pyspark.sql.functions.pandas_udf - """ - def applyInArrow(self, func: 'ArrowCogroupedMapFunction', schema: Union[StructType, str]) -> 'DataFrame': - r"""Applies a function to each cogroup using Arrow and returns the result - as a `DataFrame`. - - The function should take two `pyarrow.Table`\s and return another - `pyarrow.Table`. Alternatively, the user can pass a function that takes - a tuple of `pyarrow.Scalar` grouping key(s) and the two `pyarrow.Table`\s. - For each side of the cogroup, all columns are passed together as a - `pyarrow.Table` to the user-function and the returned `pyarrow.Table` are combined as - a :class:`DataFrame`. - - The `schema` should be a :class:`StructType` describing the schema of the returned - `pyarrow.Table`. The column labels of the returned `pyarrow.Table` must either match - the field names in the defined schema if specified as strings, or match the - field data types by position if not strings, e.g. integer indices. - The length of the returned `pyarrow.Table` can be arbitrary. - - .. versionadded:: 4.0.0 - - Parameters - ---------- - func : function - a Python native function that takes two `pyarrow.Table`\s, and - outputs a `pyarrow.Table`, or that takes one tuple (grouping keys) and two - ``pyarrow.Table``\s, and outputs a ``pyarrow.Table``. - schema : :class:`pyspark.sql.types.DataType` or str - the return type of the `func` in PySpark. The value can be either a - :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. - - Examples - -------- - >>> import pyarrow as pa - >>> df1 = spark.createDataFrame([(1, 1.0), (2, 2.0), (1, 3.0), (2, 4.0)], ("id", "v1")) - >>> df2 = spark.createDataFrame([(1, "x"), (2, "y")], ("id", "v2")) - >>> def summarize(l, r): - ... return pa.Table.from_pydict({ - ... "left": [l.num_rows], - ... "right": [r.num_rows] - ... }) - >>> df1.groupby("id").cogroup(df2.groupby("id")).applyInArrow( - ... summarize, schema="left long, right long" - ... ).show() - +----+-----+ - |left|right| - +----+-----+ - | 2| 1| - | 2| 1| - +----+-----+ - - Alternatively, the user can define a function that takes three arguments. In this case, - the grouping key(s) will be passed as the first argument and the data will be passed as the - second and third arguments. The grouping key(s) will be passed as a tuple of Arrow scalars - types, e.g., `pyarrow.Int32Scalar` and `pyarrow.FloatScalar`. The data will still be passed - in as two `pyarrow.Table`\s containing all columns from the original Spark DataFrames. - - >>> def summarize(key, l, r): - ... return pa.Table.from_pydict({ - ... "key": [key[0].as_py()], - ... "left": [l.num_rows], - ... "right": [r.num_rows] - ... }) - >>> df1.groupby("id").cogroup(df2.groupby("id")).applyInArrow( - ... summarize, schema="key long, left long, right long" - ... ).sort("key").show() - +---+----+-----+ - |key|left|right| - +---+----+-----+ - | 1| 2| 1| - | 2| 2| 1| - +---+----+-----+ - - Notes - ----- - This function requires a full shuffle. All the data of a cogroup will be loaded - into memory, so the user should be aware of the potential OOM risk if data is skewed - and certain groups are too large to fit in memory. - - This API is unstable, and for developers. - - See Also - -------- - pyspark.sql.functions.pandas_udf - """ - @staticmethod - def _extract_cols(gd: 'GroupedData') -> List[Column]: ... diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py index 9fac7369..afe8000b 100644 --- a/python/pyspark/sql/readwriter.py +++ b/python/pyspark/sql/readwriter.py @@ -14,15 +14,2630 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""Re-export of the Rust-backed class(es) under the official pyspark.sql import path.""" -import typing as _typing +import sys +from typing import cast, overload, Dict, Iterable, List, Optional, Tuple, TYPE_CHECKING, Union -from pyspark._pyspark import DataFrameReader, DataFrameWriter, DataFrameWriterV2 +from pyspark.util import is_remote_only +from pyspark.sql.types import StructType +from pyspark.sql import utils +from pyspark.sql.utils import to_str +from pyspark.errors import PySparkTypeError, PySparkValueError + +if TYPE_CHECKING: + from py4j.java_gateway import JavaObject + from pyspark.core.rdd import RDD + from pyspark.sql._typing import OptionalPrimitiveType, ColumnOrName + from pyspark.sql.session import SparkSession + from pyspark.sql.dataframe import DataFrame + from pyspark.sql.streaming import StreamingQuery __all__ = ["DataFrameReader", "DataFrameWriter", "DataFrameWriterV2"] -# PySpark's aliases of the types of the paths and of the lists of column names, which its -# readers and writers are annotated with, and an application's own code can be: of typing's -# List and Tuple, as PySpark's. -PathOrPaths = _typing.Union[str, _typing.List[str]] # noqa: UP006 -TupleOrListOfString = _typing.Union[_typing.List[str], _typing.Tuple[str, ...]] # noqa: UP006 +PathOrPaths = Union[str, List[str]] +TupleOrListOfString = Union[List[str], Tuple[str, ...]] + + +class OptionUtils: + def _set_opts( + self, + schema: Optional[Union[StructType, str]] = None, + **options: "OptionalPrimitiveType", + ) -> None: + """ + Set named options (filter out those the value is None) + """ + if schema is not None: + self.schema(schema) # type: ignore[attr-defined] + for k, v in options.items(): + if v is not None: + self.option(k, v) # type: ignore[attr-defined] + + +class DataFrameReader(OptionUtils): + """ + Interface used to load a :class:`DataFrame` from external storage systems + (e.g. file systems, key-value stores, etc). Use :attr:`SparkSession.read` + to access this. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + """ + + def __init__(self, spark: "SparkSession"): + self._jreader = spark._jsparkSession.read() + self._spark = spark + + def _df(self, jdf: "JavaObject") -> "DataFrame": + from pyspark.sql.dataframe import DataFrame + + return DataFrame(jdf, self._spark) + + def format(self, source: str) -> "DataFrameReader": + """Specifies the input data source format. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + source : str + string, name of the data source, e.g. 'json', 'parquet'. + + Examples + -------- + >>> spark.read.format('json') + <...readwriter.DataFrameReader object ...> + + Write a DataFrame into a JSON file and read it back. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="format") as d: + ... # Write a DataFrame into a JSON file + ... spark.createDataFrame( + ... [{"age": 100, "name": "Hyukjin Kwon"}] + ... ).write.mode("overwrite").format("json").save(d) + ... + ... # Read the JSON file as a DataFrame. + ... spark.read.format('json').load(d).show() + +---+------------+ + |age| name| + +---+------------+ + |100|Hyukjin Kwon| + +---+------------+ + """ + self._jreader = self._jreader.format(source) + return self + + def schema(self, schema: Union[StructType, str]) -> "DataFrameReader": + """Specifies the input schema. + + Some data sources (e.g. JSON) can infer the input schema automatically from data. + By specifying the schema here, the underlying data source can skip the schema + inference step, and thus speed up data loading. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + schema : :class:`pyspark.sql.types.StructType` or str + a :class:`pyspark.sql.types.StructType` object or a DDL-formatted string + (For example ``col0 INT, col1 DOUBLE``). + + Examples + -------- + >>> spark.read.schema("col0 INT, col1 DOUBLE") + <...readwriter.DataFrameReader object ...> + + Specify the schema with reading a CSV file. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="schema") as d: + ... spark.read.schema("col0 INT, col1 DOUBLE").format("csv").load(d).printSchema() + root + |-- col0: integer (nullable = true) + |-- col1: double (nullable = true) + """ + from pyspark.sql import SparkSession + + spark = SparkSession._getActiveSessionOrCreate() + if isinstance(schema, StructType): + jschema = spark._jsparkSession.parseDataType(schema.json()) + self._jreader = self._jreader.schema(jschema) + elif isinstance(schema, str): + self._jreader = self._jreader.schema(schema) + else: + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "schema", + "expected_type": "str or struct type", + "arg_type": type(schema).__name__, + }, + ) + return self + + def option(self, key: str, value: "OptionalPrimitiveType") -> "DataFrameReader": + """ + Adds an input option for the underlying data source. + + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + key : str + The key for the option to set. + value + The value for the option to set. + + Examples + -------- + >>> spark.read.option("key", "value") + <...readwriter.DataFrameReader object ...> + + Specify the option 'nullValue' with reading a CSV file. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="option") as d: + ... # Write a DataFrame into a CSV file + ... df = spark.createDataFrame([{"age": 100, "name": "Hyukjin Kwon"}]) + ... df.write.mode("overwrite").format("csv").save(d) + ... + ... # Read the CSV file as a DataFrame with 'nullValue' option set to 'Hyukjin Kwon'. + ... spark.read.schema(df.schema).option( + ... "nullValue", "Hyukjin Kwon").format('csv').load(d).show() + +---+----+ + |age|name| + +---+----+ + |100|NULL| + +---+----+ + """ + if value is None: + return self + self._jreader = self._jreader.option(key, to_str(value)) + return self + + def options(self, **options: "OptionalPrimitiveType") -> "DataFrameReader": + """ + Adds input options for the underlying data source. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + **options : dict + The dictionary of string keys and primitive-type values. + + Examples + -------- + >>> spark.read.options(key="value") + <...readwriter.DataFrameReader object ...> + + Specify options in a dictionary. + + >>> spark.read.options(**{"k1": "v1", "k2": "v2"}) + <...readwriter.DataFrameReader object ...> + + Specify the option 'nullValue' and 'header' with reading a CSV file. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="options") as d: + ... # Write a DataFrame into a CSV file with a header. + ... df = spark.createDataFrame([{"age": 100, "name": "Hyukjin Kwon"}]) + ... df.write.option("header", True).mode("overwrite").format("csv").save(d) + ... + ... # Read the CSV file as a DataFrame with 'nullValue' option set to 'Hyukjin Kwon', + ... # and 'header' option set to `True`. + ... spark.read.options( + ... nullValue="Hyukjin Kwon", + ... header=True + ... ).format('csv').load(d).show() + +---+----+ + |age|name| + +---+----+ + |100|NULL| + +---+----+ + """ + for k, v in options.items(): + if v is not None: + self._jreader = self._jreader.option(k, to_str(v)) + return self + + def load( + self, + path: Optional[PathOrPaths] = None, + format: Optional[str] = None, + schema: Optional[Union[StructType, str]] = None, + **options: "OptionalPrimitiveType", + ) -> "DataFrame": + """Loads data from a data source and returns it as a :class:`DataFrame`. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + path : str or list, optional + optional string or a list of string for file-system backed data sources. + format : str, optional + optional string for format of the data source. Default to 'parquet'. + schema : :class:`pyspark.sql.types.StructType` or str, optional + optional :class:`pyspark.sql.types.StructType` for the input schema + or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). + **options : dict + all other string options + + Examples + -------- + Load a CSV file with format, schema and options specified. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="load") as d: + ... # Write a DataFrame into a CSV file with a header + ... df = spark.createDataFrame([{"age": 100, "name": "Hyukjin Kwon"}]) + ... df.write.option("header", True).mode("overwrite").format("csv").save(d) + ... + ... # Read the CSV file as a DataFrame with 'nullValue' option set to 'Hyukjin Kwon', + ... # and 'header' option set to `True`. + ... df = spark.read.load( + ... d, schema=df.schema, format="csv", nullValue="Hyukjin Kwon", header=True) + ... df.printSchema() + ... df.show() + root + |-- age: long (nullable = true) + |-- name: string (nullable = true) + +---+----+ + |age|name| + +---+----+ + |100|NULL| + +---+----+ + """ + if format is not None: + self.format(format) + if schema is not None: + self.schema(schema) + self.options(**options) + if isinstance(path, str): + return self._df(self._jreader.load(path)) + elif path is not None: + if not isinstance(path, list): + path = [path] + assert self._spark._sc._jvm is not None + return self._df(self._jreader.load(self._spark._sc._jvm.PythonUtils.toSeq(path))) + else: + return self._df(self._jreader.load()) + + def json( + self, + path: Union[str, List[str], "RDD[str]", "DataFrame"], + schema: Optional[Union[StructType, str]] = None, + primitivesAsString: Optional[Union[bool, str]] = None, + prefersDecimal: Optional[Union[bool, str]] = None, + allowComments: Optional[Union[bool, str]] = None, + allowUnquotedFieldNames: Optional[Union[bool, str]] = None, + allowSingleQuotes: Optional[Union[bool, str]] = None, + allowNumericLeadingZero: Optional[Union[bool, str]] = None, + allowBackslashEscapingAnyCharacter: Optional[Union[bool, str]] = None, + mode: Optional[str] = None, + columnNameOfCorruptRecord: Optional[str] = None, + dateFormat: Optional[str] = None, + timestampFormat: Optional[str] = None, + multiLine: Optional[Union[bool, str]] = None, + allowUnquotedControlChars: Optional[Union[bool, str]] = None, + lineSep: Optional[str] = None, + samplingRatio: Optional[Union[float, str]] = None, + dropFieldIfAllNull: Optional[Union[bool, str]] = None, + encoding: Optional[str] = None, + locale: Optional[str] = None, + pathGlobFilter: Optional[Union[bool, str]] = None, + recursiveFileLookup: Optional[Union[bool, str]] = None, + modifiedBefore: Optional[Union[bool, str]] = None, + modifiedAfter: Optional[Union[bool, str]] = None, + allowNonNumericNumbers: Optional[Union[bool, str]] = None, + useUnsafeRow: Optional[Union[bool, str]] = None, + ) -> "DataFrame": + """ + Loads JSON files and returns the results as a :class:`DataFrame`. + + `JSON Lines `_ (newline-delimited JSON) is supported by default. + For JSON (one record per file), set the ``multiLine`` parameter to ``true``. + + If the ``schema`` parameter is not specified, this function goes + through the input once to determine the input schema. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + .. versionchanged:: 4.2.0 + Supports DataFrame input. + + Parameters + ---------- + path : str, list, :class:`RDD`, or :class:`DataFrame` + string represents path to the JSON dataset, or a list of paths, + or RDD of Strings storing JSON objects, + or a DataFrame with a single string column containing JSON strings. + schema : :class:`pyspark.sql.types.StructType` or str, optional + an optional :class:`pyspark.sql.types.StructType` for the input schema or + a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ + for the version you use. + + .. # noqa + + Examples + -------- + Example 1: Write a DataFrame into a JSON file and read it back. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="json1") as d: + ... # Write a DataFrame into a JSON file + ... spark.createDataFrame( + ... [{"age": 100, "name": "Hyukjin"}] + ... ).write.mode("overwrite").format("json").save(d) + ... + ... # Read the JSON file as a DataFrame. + ... spark.read.json(d).show() + +---+-------+ + |age| name| + +---+-------+ + |100|Hyukjin| + +---+-------+ + + Example 2: Read JSON from multiple files in a directory + + >>> from tempfile import TemporaryDirectory + >>> with TemporaryDirectory(prefix="json2") as d1, TemporaryDirectory(prefix="json3") as d2: + ... # Write a DataFrame into a JSON file + ... spark.createDataFrame( + ... [{"age": 30, "name": "Bob"}] + ... ).write.mode("overwrite").format("json").save(d1) + ... + ... # Read the JSON files as a DataFrame. + ... spark.createDataFrame( + ... [{"age": 25, "name": "Alice"}] + ... ).write.mode("overwrite").format("json").save(d2) + ... spark.read.json([d1, d2]).show() + +---+-----+ + |age| name| + +---+-----+ + | 25|Alice| + | 30| Bob| + +---+-----+ + + Example 3: Read JSON with a custom schema + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="json4") as d: + ... # Write a DataFrame into a JSON file + ... spark.createDataFrame( + ... [{"age": 30, "name": "Bob"}] + ... ).write.mode("overwrite").format("json").save(d) + ... custom_schema = "name STRING, age INT" + ... spark.read.json(d, schema=custom_schema).show() + +----+---+ + |name|age| + +----+---+ + | Bob| 30| + +----+---+ + + Example 4: Parse JSON from a DataFrame with a single string column. + + >>> json_df = spark.createDataFrame( + ... [('{"name": "Alice", "age": 25}',), ('{"name": "Bob", "age": 30}',)], + ... schema="value STRING", + ... ) + >>> spark.read.json(json_df).sort("name").show() + +---+-----+ + |age| name| + +---+-----+ + | 25|Alice| + | 30| Bob| + +---+-----+ + """ + self._set_opts( + schema=schema, + primitivesAsString=primitivesAsString, + prefersDecimal=prefersDecimal, + allowComments=allowComments, + allowUnquotedFieldNames=allowUnquotedFieldNames, + allowSingleQuotes=allowSingleQuotes, + allowNumericLeadingZero=allowNumericLeadingZero, + allowBackslashEscapingAnyCharacter=allowBackslashEscapingAnyCharacter, + mode=mode, + columnNameOfCorruptRecord=columnNameOfCorruptRecord, + dateFormat=dateFormat, + timestampFormat=timestampFormat, + multiLine=multiLine, + allowUnquotedControlChars=allowUnquotedControlChars, + lineSep=lineSep, + samplingRatio=samplingRatio, + dropFieldIfAllNull=dropFieldIfAllNull, + encoding=encoding, + locale=locale, + pathGlobFilter=pathGlobFilter, + recursiveFileLookup=recursiveFileLookup, + modifiedBefore=modifiedBefore, + modifiedAfter=modifiedAfter, + allowNonNumericNumbers=allowNonNumericNumbers, + useUnsafeRow=useUnsafeRow, + ) + if isinstance(path, str): + path = [path] + if isinstance(path, list): + assert self._spark._sc._jvm is not None + return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path))) + + if not is_remote_only(): + from pyspark.core.rdd import RDD + + if not is_remote_only() and isinstance(path, RDD): + + def func(iterator: Iterable) -> Iterable: + for x in iterator: + if not isinstance(x, str): + x = str(x) + if isinstance(x, str): + x = x.encode("utf-8") + yield x + + keyed = path.mapPartitions(func) + keyed._bypass_serializer = True # type: ignore[attr-defined] + assert self._spark._jvm is not None + jrdd = keyed._jrdd.map(self._spark._jvm.BytesToString()) + return self._df(self._jreader.json(jrdd)) + + from pyspark.sql.dataframe import DataFrame + + if isinstance(path, DataFrame): + assert self._spark._jvm is not None + return self._df( + self._spark._jvm.PythonSQLUtils.jsonFromDataFrame(self._jreader, path._jdf) + ) + else: + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "path", + "expected_type": "str, list, RDD, or DataFrame", + "arg_type": type(path).__name__, + }, + ) + + def table(self, tableName: str) -> "DataFrame": + """Returns the specified table as a :class:`DataFrame`. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + tableName : str + string, name of the table. + + Examples + -------- + >>> df = spark.range(10) + >>> df.createOrReplaceTempView('tblA') + >>> spark.read.table('tblA').show() + +---+ + | id| + +---+ + | 0| + | 1| + | 2| + | 3| + | 4| + | 5| + | 6| + | 7| + | 8| + | 9| + +---+ + >>> _ = spark.sql("DROP TABLE tblA") + """ + return self._df(self._jreader.table(tableName)) + + def changes(self, tableName: str) -> "DataFrame": + """Returns the row-level changes (Change Data Capture) from the specified table + as a :class:`DataFrame`. Currently this API is only supported for Data Source V2 + tables whose catalog implements ``TableCatalog.loadChangelog()``. + + Use :meth:`option` to specify the version/timestamp range and processing options. + + .. versionadded:: 4.2.0 + + Parameters + ---------- + tableName : str + string, name of the table. + + Returns + ------- + :class:`DataFrame` + + Examples + -------- + >>> spark.read.option("startingVersion", "10").option( + ... "endingVersion", "20" + ... ).changes("my_table") # doctest: +SKIP + """ + return self._df(self._jreader.changes(tableName)) + + def parquet(self, *paths: str, **options: "OptionalPrimitiveType") -> "DataFrame": + """ + Loads Parquet files, returning the result as a :class:`DataFrame`. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + paths : str + One or more file paths to read the Parquet files from. + + Other Parameters + ---------------- + **options + For the extra options, refer to + `Data Source Option `_ + for the version you use. + + .. # noqa + + Returns + ------- + :class:`DataFrame` + A DataFrame containing the data from the Parquet files. + + Examples + -------- + Create sample dataframes. + + >>> df = spark.createDataFrame( + ... [(10, "Alice"), (15, "Bob"), (20, "Tom")], schema=["age", "name"]) + >>> df2 = spark.createDataFrame([(70, "Alice"), (80, "Bob")], schema=["height", "name"]) + + Write a DataFrame into a Parquet file and read it back. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="parquet1") as d: + ... # Write a DataFrame into a Parquet file. + ... df.write.mode("overwrite").format("parquet").save(d) + ... + ... # Read the Parquet file as a DataFrame. + ... spark.read.parquet(d).orderBy("name").show() + +---+-----+ + |age| name| + +---+-----+ + | 10|Alice| + | 15| Bob| + | 20| Tom| + +---+-----+ + + Read a Parquet file with a specific column. + + >>> with tempfile.TemporaryDirectory(prefix="parquet2") as d: + ... df.write.mode("overwrite").format("parquet").save(d) + ... + ... # Read the Parquet file with only the 'name' column. + ... spark.read.schema("name string").parquet(d).orderBy("name").show() + +-----+ + | name| + +-----+ + |Alice| + | Bob| + | Tom| + +-----+ + + Read multiple Parquet files and merge schema. + + >>> with tempfile.TemporaryDirectory(prefix="parquet3") as d1: + ... with tempfile.TemporaryDirectory(prefix="parquet4") as d2: + ... df.write.mode("overwrite").format("parquet").save(d1) + ... df2.write.mode("overwrite").format("parquet").save(d2) + ... + ... spark.read.option( + ... "mergeSchema", "true" + ... ).parquet(d1, d2).select( + ... "name", "age", "height" + ... ).orderBy("name", "age").show() + +-----+----+------+ + | name| age|height| + +-----+----+------+ + |Alice|NULL| 70| + |Alice| 10| NULL| + | Bob|NULL| 80| + | Bob| 15| NULL| + | Tom| 20| NULL| + +-----+----+------+ + """ + from pyspark.sql.classic.column import _to_seq + + mergeSchema = options.get("mergeSchema", None) + pathGlobFilter = options.get("pathGlobFilter", None) + modifiedBefore = options.get("modifiedBefore", None) + modifiedAfter = options.get("modifiedAfter", None) + recursiveFileLookup = options.get("recursiveFileLookup", None) + datetimeRebaseMode = options.get("datetimeRebaseMode", None) + int96RebaseMode = options.get("int96RebaseMode", None) + self._set_opts( + mergeSchema=mergeSchema, + pathGlobFilter=pathGlobFilter, + recursiveFileLookup=recursiveFileLookup, + modifiedBefore=modifiedBefore, + modifiedAfter=modifiedAfter, + datetimeRebaseMode=datetimeRebaseMode, + int96RebaseMode=int96RebaseMode, + ) + + return self._df(self._jreader.parquet(_to_seq(self._spark._sc, paths))) + + def text( + self, + paths: PathOrPaths, + wholetext: bool = False, + lineSep: Optional[str] = None, + pathGlobFilter: Optional[Union[bool, str]] = None, + recursiveFileLookup: Optional[Union[bool, str]] = None, + modifiedBefore: Optional[Union[bool, str]] = None, + modifiedAfter: Optional[Union[bool, str]] = None, + ) -> "DataFrame": + """ + Loads text files and returns a :class:`DataFrame` whose schema starts with a + string column named "value", and followed by partitioned columns if there + are any. + The text files must be encoded as UTF-8. + + By default, each line in the text file is a new row in the resulting DataFrame. + + .. versionadded:: 1.6.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + paths : str or list + string, or list of strings, for input path(s). + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ + for the version you use. + + .. # noqa + + Examples + -------- + Write a DataFrame into a text file and read it back. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="text") as d: + ... # Write a DataFrame into a text file + ... df = spark.createDataFrame([("a",), ("b",), ("c",)], schema=["alphabets"]) + ... df.write.mode("overwrite").format("text").save(d) + ... + ... # Read the text file as a DataFrame. + ... spark.read.schema(df.schema).text(d).sort("alphabets").show() + +---------+ + |alphabets| + +---------+ + | a| + | b| + | c| + +---------+ + """ + self._set_opts( + wholetext=wholetext, + lineSep=lineSep, + pathGlobFilter=pathGlobFilter, + recursiveFileLookup=recursiveFileLookup, + modifiedBefore=modifiedBefore, + modifiedAfter=modifiedAfter, + ) + + if isinstance(paths, str): + paths = [paths] + assert self._spark._sc._jvm is not None + return self._df(self._jreader.text(self._spark._sc._jvm.PythonUtils.toSeq(paths))) + + def csv( + self, + path: Union[str, List[str], "RDD[str]", "DataFrame"], + schema: Optional[Union[StructType, str]] = None, + sep: Optional[str] = None, + encoding: Optional[str] = None, + quote: Optional[str] = None, + escape: Optional[str] = None, + comment: Optional[str] = None, + header: Optional[Union[bool, str]] = None, + inferSchema: Optional[Union[bool, str]] = None, + ignoreLeadingWhiteSpace: Optional[Union[bool, str]] = None, + ignoreTrailingWhiteSpace: Optional[Union[bool, str]] = None, + nullValue: Optional[str] = None, + nanValue: Optional[str] = None, + positiveInf: Optional[str] = None, + negativeInf: Optional[str] = None, + dateFormat: Optional[str] = None, + timestampFormat: Optional[str] = None, + maxColumns: Optional[Union[int, str]] = None, + maxCharsPerColumn: Optional[Union[int, str]] = None, + maxMalformedLogPerPartition: Optional[Union[int, str]] = None, + mode: Optional[str] = None, + columnNameOfCorruptRecord: Optional[str] = None, + multiLine: Optional[Union[bool, str]] = None, + charToEscapeQuoteEscaping: Optional[str] = None, + samplingRatio: Optional[Union[float, str]] = None, + enforceSchema: Optional[Union[bool, str]] = None, + emptyValue: Optional[str] = None, + locale: Optional[str] = None, + lineSep: Optional[str] = None, + pathGlobFilter: Optional[Union[bool, str]] = None, + recursiveFileLookup: Optional[Union[bool, str]] = None, + modifiedBefore: Optional[Union[bool, str]] = None, + modifiedAfter: Optional[Union[bool, str]] = None, + unescapedQuoteHandling: Optional[str] = None, + ) -> "DataFrame": + r"""Loads a CSV file and returns the result as a :class:`DataFrame`. + + This function will go through the input once to determine the input schema if + ``inferSchema`` is enabled. To avoid going through the entire data once, disable + ``inferSchema`` option or specify the schema explicitly using ``schema``. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + .. versionchanged:: 4.2.0 + Supports DataFrame input. + + Parameters + ---------- + path : str, list, :class:`RDD`, or :class:`DataFrame` + string, or list of strings, for input path(s), + or RDD of Strings storing CSV rows, + or a DataFrame with a single string column containing CSV rows. + schema : :class:`pyspark.sql.types.StructType` or str, optional + an optional :class:`pyspark.sql.types.StructType` for the input schema + or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ + for the version you use. + + .. # noqa + + Examples + -------- + Write a DataFrame into a CSV file and read it back. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="csv") as d: + ... # Write a DataFrame into a CSV file + ... df = spark.createDataFrame([{"age": 100, "name": "Hyukjin Kwon"}]) + ... df.write.mode("overwrite").format("csv").save(d) + ... + ... # Read the CSV file as a DataFrame with 'nullValue' option set to 'Hyukjin Kwon'. + ... spark.read.csv(d, schema=df.schema, nullValue="Hyukjin Kwon").show() + +---+----+ + |age|name| + +---+----+ + |100|NULL| + +---+----+ + """ + self._set_opts( + schema=schema, + sep=sep, + encoding=encoding, + quote=quote, + escape=escape, + comment=comment, + header=header, + inferSchema=inferSchema, + ignoreLeadingWhiteSpace=ignoreLeadingWhiteSpace, + ignoreTrailingWhiteSpace=ignoreTrailingWhiteSpace, + nullValue=nullValue, + nanValue=nanValue, + positiveInf=positiveInf, + negativeInf=negativeInf, + dateFormat=dateFormat, + timestampFormat=timestampFormat, + maxColumns=maxColumns, + maxCharsPerColumn=maxCharsPerColumn, + maxMalformedLogPerPartition=maxMalformedLogPerPartition, + mode=mode, + columnNameOfCorruptRecord=columnNameOfCorruptRecord, + multiLine=multiLine, + charToEscapeQuoteEscaping=charToEscapeQuoteEscaping, + samplingRatio=samplingRatio, + enforceSchema=enforceSchema, + emptyValue=emptyValue, + locale=locale, + lineSep=lineSep, + pathGlobFilter=pathGlobFilter, + recursiveFileLookup=recursiveFileLookup, + modifiedBefore=modifiedBefore, + modifiedAfter=modifiedAfter, + unescapedQuoteHandling=unescapedQuoteHandling, + ) + if isinstance(path, str): + path = [path] + if isinstance(path, list): + assert self._spark._sc._jvm is not None + return self._df(self._jreader.csv(self._spark._sc._jvm.PythonUtils.toSeq(path))) + + if not is_remote_only(): + from pyspark.core.rdd import RDD + + if not is_remote_only() and isinstance(path, RDD): + + def func(iterator: Iterable) -> Iterable: + for x in iterator: + if not isinstance(x, str): + x = str(x) + if isinstance(x, str): + x = x.encode("utf-8") + yield x + + keyed = path.mapPartitions(func) + keyed._bypass_serializer = True # type: ignore[attr-defined] + assert self._spark._jvm is not None + jrdd = keyed._jrdd.map(self._spark._jvm.BytesToString()) + # see SPARK-22112 + # There aren't any jvm api for creating a dataframe from rdd storing csv. + # We can do it through creating a jvm dataset firstly and using the jvm api + # for creating a dataframe from dataset storing csv. + jdataset = self._spark._jsparkSession.createDataset( + jrdd.rdd(), self._spark._jvm.Encoders.STRING() + ) + return self._df(self._jreader.csv(jdataset)) + + from pyspark.sql.dataframe import DataFrame + + if isinstance(path, DataFrame): + assert self._spark._jvm is not None + return self._df( + self._spark._jvm.PythonSQLUtils.csvFromDataFrame(self._jreader, path._jdf) + ) + else: + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "path", + "expected_type": "str, list, RDD, or DataFrame", + "arg_type": type(path).__name__, + }, + ) + + def xml( + self, + path: Union[str, List[str], "RDD[str]", "DataFrame"], + rowTag: Optional[str] = None, + schema: Optional[Union[StructType, str]] = None, + excludeAttribute: Optional[Union[bool, str]] = None, + attributePrefix: Optional[str] = None, + valueTag: Optional[str] = None, + ignoreSurroundingSpaces: Optional[Union[bool, str]] = None, + rowValidationXSDPath: Optional[str] = None, + ignoreNamespace: Optional[Union[bool, str]] = None, + wildcardColName: Optional[str] = None, + encoding: Optional[str] = None, + inferSchema: Optional[Union[bool, str]] = None, + nullValue: Optional[str] = None, + dateFormat: Optional[str] = None, + timestampFormat: Optional[str] = None, + mode: Optional[str] = None, + columnNameOfCorruptRecord: Optional[str] = None, + multiLine: Optional[Union[bool, str]] = None, + samplingRatio: Optional[Union[float, str]] = None, + locale: Optional[str] = None, + ) -> "DataFrame": + r"""Loads a XML file and returns the result as a :class:`DataFrame`. + + If the ``schema`` parameter is not specified, this function goes + through the input once to determine the input schema. + + .. versionadded:: 4.0.0 + + .. versionchanged:: 4.2.0 + Supports DataFrame input. + + Parameters + ---------- + path : str, list, :class:`RDD`, or :class:`DataFrame` + string, or list of strings, for input path(s), + or RDD of Strings storing XML rows, + or a DataFrame with a single string column containing XML strings. + schema : :class:`pyspark.sql.types.StructType` or str, optional + an optional :class:`pyspark.sql.types.StructType` for the input schema + or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ + for the version you use. + + .. # noqa + + Examples + -------- + Example 1: Write a DataFrame into a XML file and read it back. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="xml") as d: + ... # Write a DataFrame into a XML file + ... spark.createDataFrame( + ... [{"age": 100, "name": "Hyukjin Kwon"}] + ... ).write.mode("overwrite").option("rowTag", "person").format("xml").save(d) + ... + ... # Read the XML file as a DataFrame. + ... spark.read.option("rowTag", "person").xml(d).show() + +---+------------+ + |age| name| + +---+------------+ + |100|Hyukjin Kwon| + +---+------------+ + + Example 2: Parse XML from a DataFrame with a single string column. + + >>> xml_df = spark.createDataFrame( + ... [('Alice25',), + ... ('Bob30',)], + ... schema="value STRING", + ... ) + >>> spark.read.option("rowTag", "person").xml(xml_df).sort("name").show() + +---+-----+ + |age| name| + +---+-----+ + | 25|Alice| + | 30| Bob| + +---+-----+ + """ + self._set_opts( + rowTag=rowTag, + schema=schema, + excludeAttribute=excludeAttribute, + attributePrefix=attributePrefix, + valueTag=valueTag, + ignoreSurroundingSpaces=ignoreSurroundingSpaces, + rowValidationXSDPath=rowValidationXSDPath, + ignoreNamespace=ignoreNamespace, + wildcardColName=wildcardColName, + encoding=encoding, + inferSchema=inferSchema, + nullValue=nullValue, + dateFormat=dateFormat, + timestampFormat=timestampFormat, + mode=mode, + columnNameOfCorruptRecord=columnNameOfCorruptRecord, + multiLine=multiLine, + samplingRatio=samplingRatio, + locale=locale, + ) + if isinstance(path, str): + path = [path] + if isinstance(path, list): + assert self._spark._sc._jvm is not None + return self._df(self._jreader.xml(self._spark._sc._jvm.PythonUtils.toSeq(path))) + + if not is_remote_only(): + from pyspark.core.rdd import RDD + + if not is_remote_only() and isinstance(path, RDD): + + def func(iterator: Iterable) -> Iterable: + for x in iterator: + if not isinstance(x, str): + x = str(x) + if isinstance(x, str): + x = x.encode("utf-8") + yield x + + keyed = path.mapPartitions(func) + keyed._bypass_serializer = True # type: ignore[attr-defined] + assert self._spark._jvm is not None + jrdd = keyed._jrdd.map(self._spark._jvm.BytesToString()) + # There isn't any jvm api for creating a dataframe from rdd storing XML. + # We can do it through creating a jvm dataset first and using the jvm api + # for creating a dataframe from dataset storing XML. + jdataset = self._spark._jsparkSession.createDataset( + jrdd.rdd(), self._spark._jvm.Encoders.STRING() + ) + return self._df(self._jreader.xml(jdataset)) + + from pyspark.sql.dataframe import DataFrame + + if isinstance(path, DataFrame): + assert self._spark._jvm is not None + return self._df( + self._spark._jvm.PythonSQLUtils.xmlFromDataFrame(self._jreader, path._jdf) + ) + else: + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "path", + "expected_type": "str, list, RDD, or DataFrame", + "arg_type": type(path).__name__, + }, + ) + + def orc( + self, + path: PathOrPaths, + mergeSchema: Optional[bool] = None, + pathGlobFilter: Optional[Union[bool, str]] = None, + recursiveFileLookup: Optional[Union[bool, str]] = None, + modifiedBefore: Optional[Union[bool, str]] = None, + modifiedAfter: Optional[Union[bool, str]] = None, + ) -> "DataFrame": + """Loads ORC files, returning the result as a :class:`DataFrame`. + + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + path : str or list + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ + for the version you use. + + .. # noqa + + Examples + -------- + Write a DataFrame into a ORC file and read it back. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="orc") as d: + ... # Write a DataFrame into a ORC file + ... spark.createDataFrame( + ... [{"age": 100, "name": "Hyukjin Kwon"}] + ... ).write.mode("overwrite").format("orc").save(d) + ... + ... # Read the Parquet file as a DataFrame. + ... spark.read.orc(d).show() + +---+------------+ + |age| name| + +---+------------+ + |100|Hyukjin Kwon| + +---+------------+ + """ + from pyspark.sql.classic.column import _to_seq + + self._set_opts( + mergeSchema=mergeSchema, + pathGlobFilter=pathGlobFilter, + modifiedBefore=modifiedBefore, + modifiedAfter=modifiedAfter, + recursiveFileLookup=recursiveFileLookup, + ) + if isinstance(path, str): + path = [path] + return self._df(self._jreader.orc(_to_seq(self._spark._sc, path))) + + @overload + def jdbc( + self, url: str, table: str, *, properties: Optional[Dict[str, str]] = None + ) -> "DataFrame": ... + + @overload + def jdbc( + self, + url: str, + table: str, + column: str, + lowerBound: Union[int, str], + upperBound: Union[int, str], + numPartitions: int, + *, + properties: Optional[Dict[str, str]] = None, + ) -> "DataFrame": ... + + @overload + def jdbc( + self, + url: str, + table: str, + *, + predicates: List[str], + properties: Optional[Dict[str, str]] = None, + ) -> "DataFrame": ... + + def jdbc( + self, + url: str, + table: str, + column: Optional[str] = None, + lowerBound: Optional[Union[int, str]] = None, + upperBound: Optional[Union[int, str]] = None, + numPartitions: Optional[int] = None, + predicates: Optional[List[str]] = None, + properties: Optional[Dict[str, str]] = None, + ) -> "DataFrame": + """ + Construct a :class:`DataFrame` representing the database table named ``table`` + accessible via JDBC URL ``url`` and connection ``properties``. + + Partitions of the table will be retrieved in parallel if either ``column`` or + ``predicates`` is specified. ``lowerBound``, ``upperBound`` and ``numPartitions`` + is needed when ``column`` is specified. + + If both ``column`` and ``predicates`` are specified, ``column`` will be used. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + table : str + the name of the table + column : str, optional + alias of ``partitionColumn`` option. Refer to ``partitionColumn`` in + `Data Source Option `_ + for the version you use. + predicates : list, optional + a list of expressions suitable for inclusion in WHERE clauses; + each one defines one partition of the :class:`DataFrame` + properties : dict, optional + a dictionary of JDBC database connection arguments. Normally at + least properties "user" and "password" with their corresponding values. + For example { 'user' : 'SYSTEM', 'password' : 'mypassword' } + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ + for the version you use. + + .. # noqa + + Notes + ----- + Don't create too many partitions in parallel on a large cluster; + otherwise Spark might crash your external database systems. + + Returns + ------- + :class:`DataFrame` + """ + from py4j.java_gateway import JavaClass + + if properties is None: + properties = dict() + assert self._spark._sc._gateway is not None + jprop = JavaClass( + "java.util.Properties", + self._spark._sc._gateway._gateway_client, + )() + for k in properties: + jprop.setProperty(k, properties[k]) + if column is not None: + assert lowerBound is not None, "lowerBound can not be None when ``column`` is specified" + assert upperBound is not None, "upperBound can not be None when ``column`` is specified" + assert numPartitions is not None, ( + "numPartitions can not be None when ``column`` is specified" + ) + return self._df( + self._jreader.jdbc( + url, table, column, int(lowerBound), int(upperBound), int(numPartitions), jprop + ) + ) + if predicates is not None: + gateway = self._spark._sc._gateway + assert gateway is not None + jpredicates = utils.to_java_array( + gateway, getattr(gateway.jvm, "java.lang.String"), predicates + ) + return self._df(self._jreader.jdbc(url, table, jpredicates, jprop)) + return self._df(self._jreader.jdbc(url, table, jprop)) + + +class DataFrameWriter(OptionUtils): + """ + Interface used to write a :class:`DataFrame` to external storage systems + (e.g. file systems, key-value stores, etc). Use :attr:`DataFrame.write` + to access this. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + """ + + def __init__(self, df: "DataFrame"): + self._df = df + self._spark = df.sparkSession + self._jwrite = df._jdf.write() + + def _sq(self, jsq: "JavaObject") -> "StreamingQuery": + from pyspark.sql.streaming import StreamingQuery + + return StreamingQuery(jsq) + + def mode(self, saveMode: Optional[str]) -> "DataFrameWriter": + """Specifies the behavior when data or table already exists. + + Options include: + + * `append`: Append contents of this :class:`DataFrame` to existing data. + * `overwrite`: Overwrite existing data. + * `error` or `errorifexists`: Throw an exception if data already exists. + * `ignore`: Silently ignore this operation if data already exists. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Examples + -------- + Raise an error when writing to an existing path. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="mode1") as d: + ... spark.createDataFrame( + ... [{"age": 80, "name": "Xinrong Meng"}] + ... ).write.mode("error").format("parquet").save(d) # doctest: +SKIP + Traceback (most recent call last): + ... + ...AnalysisException: ... + + Write a Parquet file back with various options, and read it back. + + >>> with tempfile.TemporaryDirectory(prefix="mode2") as d: + ... # Overwrite the path with a new Parquet file + ... spark.createDataFrame( + ... [{"age": 100, "name": "Hyukjin Kwon"}] + ... ).write.mode("overwrite").format("parquet").save(d) + ... + ... # Append another DataFrame into the Parquet file + ... spark.createDataFrame( + ... [{"age": 120, "name": "Takuya Ueshin"}] + ... ).write.mode("append").format("parquet").save(d) + ... + ... # Append another DataFrame into the Parquet file + ... spark.createDataFrame( + ... [{"age": 140, "name": "Haejoon Lee"}] + ... ).write.mode("ignore").format("parquet").save(d) + ... + ... # Read the Parquet file as a DataFrame. + ... spark.read.parquet(d).show() + +---+-------------+ + |age| name| + +---+-------------+ + |120|Takuya Ueshin| + |100| Hyukjin Kwon| + +---+-------------+ + """ + # At the JVM side, the default value of mode is already set to "error". + # So, if the given saveMode is None, we will not call JVM-side's mode method. + if saveMode is not None: + self._jwrite = self._jwrite.mode(saveMode) + return self + + def format(self, source: str) -> "DataFrameWriter": + """Specifies the underlying output data source. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + source : str + string, name of the data source, e.g. 'json', 'parquet'. + + Examples + -------- + >>> spark.range(1).write.format('parquet') + <...readwriter.DataFrameWriter object ...> + + Write a DataFrame into a Parquet file and read it back. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="format") as d: + ... # Write a DataFrame into a Parquet file + ... spark.createDataFrame( + ... [{"age": 100, "name": "Hyukjin Kwon"}] + ... ).write.mode("overwrite").format("parquet").save(d) + ... + ... # Read the Parquet file as a DataFrame. + ... spark.read.format('parquet').load(d).show() + +---+------------+ + |age| name| + +---+------------+ + |100|Hyukjin Kwon| + +---+------------+ + """ + self._jwrite = self._jwrite.format(source) + return self + + def option(self, key: str, value: "OptionalPrimitiveType") -> "DataFrameWriter": + """ + Adds an output option for the underlying data source. + + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + key : str + The key for the option to set. + value + The value for the option to set. + + Examples + -------- + >>> spark.range(1).write.option("key", "value") + <...readwriter.DataFrameWriter object ...> + + Specify the option 'nullValue' with writing a CSV file. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="option") as d: + ... # Write a DataFrame into a CSV file with 'nullValue' option set to 'Hyukjin Kwon'. + ... df = spark.createDataFrame([(100, None)], "age INT, name STRING") + ... df.write.option("nullValue", "Hyukjin Kwon").mode("overwrite").format("csv").save(d) + ... + ... # Read the CSV file as a DataFrame. + ... spark.read.schema(df.schema).format('csv').load(d).show() + +---+------------+ + |age| name| + +---+------------+ + |100|Hyukjin Kwon| + +---+------------+ + """ + + if value is None: + return self + self._jwrite = self._jwrite.option(key, to_str(value)) + return self + + def options(self, **options: "OptionalPrimitiveType") -> "DataFrameWriter": + """ + Adds output options for the underlying data source. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + **options : dict + The dictionary of string keys and primitive-type values. + + Examples + -------- + >>> spark.range(1).write.options(key="value") + <...readwriter.DataFrameWriter object ...> + + Specify options in a dictionary. + + >>> spark.range(1).write.options(**{"k1": "v1", "k2": "v2"}) + <...readwriter.DataFrameWriter object ...> + + Specify the option 'nullValue' and 'header' with writing a CSV file. + + >>> from pyspark.sql.types import StructType,StructField, StringType, IntegerType + >>> schema = StructType([ + ... StructField("age",IntegerType(),True), + ... StructField("name",StringType(),True), + ... ]) + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="options") as d: + ... # Write a DataFrame into a CSV file with 'nullValue' option set to 'Hyukjin Kwon', + ... # and 'header' option set to `True`. + ... df = spark.createDataFrame([(100, None)], schema=schema) + ... df.write.options(nullValue="Hyukjin Kwon", header=True).mode( + ... "overwrite").format("csv").save(d) + ... + ... # Read the CSV file as a DataFrame. + ... spark.read.option("header", True).format('csv').load(d).show() + +---+------------+ + |age| name| + +---+------------+ + |100|Hyukjin Kwon| + +---+------------+ + """ + for k, v in options.items(): + if v is not None: + self._jwrite = self._jwrite.option(k, to_str(v)) + return self + + @overload + def partitionBy(self, *cols: str) -> "DataFrameWriter": ... + + @overload + def partitionBy(self, *cols: List[str]) -> "DataFrameWriter": ... + + def partitionBy(self, *cols: Union[str, List[str]]) -> "DataFrameWriter": + """Partitions the output by the given columns on the file system. + + If specified, the output is laid out on the file system similar + to Hive's partitioning scheme. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + cols : str or list + name of columns + + Examples + -------- + Write a DataFrame into a Parquet file in a partitioned manner, and read it back. + + >>> import tempfile + >>> import os + >>> with tempfile.TemporaryDirectory(prefix="partitionBy") as d: + ... # Write a DataFrame into a Parquet file in a partitioned manner. + ... spark.createDataFrame( + ... [{"age": 100, "name": "Hyukjin Kwon"}, {"age": 120, "name": "Ruifeng Zheng"}] + ... ).write.partitionBy("name").mode("overwrite").format("parquet").save(d) + ... + ... # Read the Parquet file as a DataFrame. + ... spark.read.parquet(d).sort("age").show() + ... + ... # Read one partition as a DataFrame. + ... spark.read.parquet(f"{d}{os.path.sep}name=Hyukjin Kwon").show() + +---+-------------+ + |age| name| + +---+-------------+ + |100| Hyukjin Kwon| + |120|Ruifeng Zheng| + +---+-------------+ + +---+ + |age| + +---+ + |100| + +---+ + """ + from pyspark.sql.classic.column import _to_seq + + if len(cols) == 1 and isinstance(cols[0], (list, tuple)): + cols = cols[0] # type: ignore[assignment] + self._jwrite = self._jwrite.partitionBy( + _to_seq(self._spark._sc, cast(Iterable["ColumnOrName"], cols)) + ) + return self + + @overload + def bucketBy(self, numBuckets: int, col: str, *cols: str) -> "DataFrameWriter": ... + + @overload + def bucketBy(self, numBuckets: int, col: TupleOrListOfString) -> "DataFrameWriter": ... + + def bucketBy( + self, numBuckets: int, col: Union[str, TupleOrListOfString], *cols: Optional[str] + ) -> "DataFrameWriter": + """Buckets the output by the given columns. If specified, + the output is laid out on the file system similar to Hive's bucketing scheme, + but with a different bucket hash function and is not compatible with Hive's bucketing. + + .. versionadded:: 2.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + numBuckets : int + the number of buckets to save + col : str, list or tuple + a name of a column, or a list of names. + cols : str + additional names (optional). If `col` is a list it should be empty. + + Notes + ----- + Applicable for file-based data sources in combination with + :py:meth:`DataFrameWriter.saveAsTable`. + + Examples + -------- + Write a DataFrame into a Parquet file in a buckted manner, and read it back. + + >>> from pyspark.sql.functions import input_file_name + >>> # Write a DataFrame into a Parquet file in a bucketed manner. + ... _ = spark.sql("DROP TABLE IF EXISTS bucketed_table") + >>> spark.createDataFrame([ + ... (100, "Hyukjin Kwon"), (120, "Hyukjin Kwon"), (140, "Haejoon Lee")], + ... schema=["age", "name"] + ... ).write.bucketBy(2, "name").mode("overwrite").saveAsTable("bucketed_table") + >>> # Read the Parquet file as a DataFrame. + ... spark.read.table("bucketed_table").sort("age").show() + +---+------------+ + |age| name| + +---+------------+ + |100|Hyukjin Kwon| + |120|Hyukjin Kwon| + |140| Haejoon Lee| + +---+------------+ + >>> _ = spark.sql("DROP TABLE bucketed_table") + """ + from pyspark.sql.classic.column import _to_seq + + if not isinstance(numBuckets, int): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "int", + "arg_name": "numBuckets", + "arg_type": type(numBuckets).__name__, + }, + ) + + if isinstance(col, (list, tuple)): + if cols: + raise PySparkValueError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={ + "arg_list": f"`col` of type {type(col).__name__} and `cols`", + }, + ) + + col, cols = col[0], col[1:] # type: ignore[assignment] + + for c in cols: + if not isinstance(c, str): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "cols", + "expected_type": "list[str]", + "arg_type": type(c).__name__, + }, + ) + if not isinstance(col, str): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "col", + "expected_type": "list[str]", + "arg_type": type(col).__name__, + }, + ) + + self._jwrite = self._jwrite.bucketBy( + numBuckets, col, _to_seq(self._spark._sc, cast(Iterable["ColumnOrName"], cols)) + ) + return self + + @overload + def sortBy(self, col: str, *cols: str) -> "DataFrameWriter": ... + + @overload + def sortBy(self, col: TupleOrListOfString) -> "DataFrameWriter": ... + + def sortBy( + self, col: Union[str, TupleOrListOfString], *cols: Optional[str] + ) -> "DataFrameWriter": + """Sorts the output in each bucket by the given columns on the file system. + + .. versionadded:: 2.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + col : str, tuple or list + a name of a column, or a list of names. + cols : str + additional names (optional). If `col` is a list it should be empty. + + Examples + -------- + Write a DataFrame into a Parquet file in a sorted-buckted manner, and read it back. + + >>> from pyspark.sql.functions import input_file_name + >>> # Write a DataFrame into a Parquet file in a sorted-bucketed manner. + ... _ = spark.sql("DROP TABLE IF EXISTS sorted_bucketed_table") + >>> spark.createDataFrame([ + ... (100, "Hyukjin Kwon"), (120, "Hyukjin Kwon"), (140, "Haejoon Lee")], + ... schema=["age", "name"] + ... ).write.bucketBy(1, "name").sortBy("age").mode( + ... "overwrite").saveAsTable("sorted_bucketed_table") + >>> # Read the Parquet file as a DataFrame. + ... spark.read.table("sorted_bucketed_table").sort("age").show() + +---+------------+ + |age| name| + +---+------------+ + |100|Hyukjin Kwon| + |120|Hyukjin Kwon| + |140| Haejoon Lee| + +---+------------+ + >>> _ = spark.sql("DROP TABLE sorted_bucketed_table") + """ + from pyspark.sql.classic.column import _to_seq + + if isinstance(col, (list, tuple)): + if cols: + raise PySparkValueError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={ + "arg_list": f"`col` of type {type(col).__name__} and `cols`", + }, + ) + + col, cols = col[0], col[1:] # type: ignore[assignment] + + for c in cols: + if not isinstance(c, str): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "cols", + "expected_type": "list[str]", + "arg_type": type(c).__name__, + }, + ) + if not isinstance(col, str): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "col", + "expected_type": "list[str]", + "arg_type": type(col).__name__, + }, + ) + + self._jwrite = self._jwrite.sortBy( + col, _to_seq(self._spark._sc, cast(Iterable["ColumnOrName"], cols)) + ) + return self + + @overload + def clusterBy(self, *cols: str) -> "DataFrameWriter": ... + + @overload + def clusterBy(self, *cols: List[str]) -> "DataFrameWriter": ... + + def clusterBy(self, *cols: Union[str, List[str]]) -> "DataFrameWriter": + """Clusters the data by the given columns to optimize query performance. + + .. versionadded:: 4.0.0 + + Parameters + ---------- + cols : str or list + name of columns + + Examples + -------- + Write a DataFrame into a Parquet file with clustering. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="clusterBy") as d: + ... spark.createDataFrame( + ... [{"age": 100, "name": "Hyukjin Kwon"}, {"age": 120, "name": "Ruifeng Zheng"}] + ... ).write.clusterBy("name").mode("overwrite").format("parquet").save(d) + """ + from pyspark.sql.classic.column import _to_seq + + if len(cols) == 1 and isinstance(cols[0], (list, tuple)): + cols = cols[0] # type: ignore[assignment] + assert len(cols) > 0, "clusterBy needs one or more clustering columns." + self._jwrite = self._jwrite.clusterBy(cols[0], _to_seq(self._spark._sc, cols[1:])) + return self + + def save( + self, + path: Optional[str] = None, + format: Optional[str] = None, + mode: Optional[str] = None, + partitionBy: Optional[Union[str, List[str]]] = None, + **options: "OptionalPrimitiveType", + ) -> None: + """Saves the contents of the :class:`DataFrame` to a data source. + + The data source is specified by the ``format`` and a set of ``options``. + If ``format`` is not specified, the default data source configured by + ``spark.sql.sources.default`` will be used. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + path : str, optional + the path in a Hadoop supported file system + format : str, optional + the format used to save + mode : str, optional + specifies the behavior of the save operation when data already exists. + + * ``append``: Append contents of this :class:`DataFrame` to existing data. + * ``overwrite``: Overwrite existing data. + * ``ignore``: Silently ignore this operation if data already exists. + * ``error`` or ``errorifexists`` (default case): Throw an exception if data already \ + exists. + partitionBy : list, optional + names of partitioning columns + **options : dict + all other string options + + Examples + -------- + Write a DataFrame into a JSON file and read it back. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="save") as d: + ... # Write a DataFrame into a JSON file + ... spark.createDataFrame( + ... [{"age": 100, "name": "Hyukjin Kwon"}] + ... ).write.mode("overwrite").format("json").save(d) + ... + ... # Read the JSON file as a DataFrame. + ... spark.read.format('json').load(d).show() + +---+------------+ + |age| name| + +---+------------+ + |100|Hyukjin Kwon| + +---+------------+ + """ + self.mode(mode).options(**options) + if partitionBy is not None: + self.partitionBy(partitionBy) + if format is not None: + self.format(format) + if path is None: + self._jwrite.save() + else: + self._jwrite.save(path) + + def insertInto(self, tableName: str, overwrite: Optional[bool] = None) -> None: + """Inserts the content of the :class:`DataFrame` to the specified table. + + It requires that the schema of the :class:`DataFrame` is the same as the + schema of the table. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + overwrite : bool, optional + If true, overwrites existing data. Disabled by default + + Notes + ----- + Unlike :meth:`DataFrameWriter.saveAsTable`, :meth:`DataFrameWriter.insertInto` ignores + the column names and just uses position-based resolution. + + Examples + -------- + >>> _ = spark.sql("DROP TABLE IF EXISTS tblA") + >>> df = spark.createDataFrame([ + ... (100, "Hyukjin Kwon"), (120, "Hyukjin Kwon"), (140, "Haejoon Lee")], + ... schema=["age", "name"] + ... ) + >>> df.write.saveAsTable("tblA") + + Insert the data into 'tblA' table but with different column names. + + >>> df.selectExpr("age AS col1", "name AS col2").write.insertInto("tblA") + >>> spark.read.table("tblA").sort("age").show() + +---+------------+ + |age| name| + +---+------------+ + |100|Hyukjin Kwon| + |100|Hyukjin Kwon| + |120|Hyukjin Kwon| + |120|Hyukjin Kwon| + |140| Haejoon Lee| + |140| Haejoon Lee| + +---+------------+ + >>> _ = spark.sql("DROP TABLE tblA") + """ + if overwrite is not None: + self.mode("overwrite" if overwrite else "append") + self._jwrite.insertInto(tableName) + + def saveAsTable( + self, + name: str, + format: Optional[str] = None, + mode: Optional[str] = None, + partitionBy: Optional[Union[str, List[str]]] = None, + **options: "OptionalPrimitiveType", + ) -> None: + """Saves the content of the :class:`DataFrame` as the specified table. + + In the case the table already exists, behavior of this function depends on the + save mode, specified by the `mode` function (default to throwing an exception). + When `mode` is `Overwrite`, the schema of the :class:`DataFrame` does not need to be + the same as that of the existing table. + + * `append`: Append contents of this :class:`DataFrame` to existing data. + * `overwrite`: Overwrite existing data. + * `error` or `errorifexists`: Throw an exception if data already exists. + * `ignore`: Silently ignore this operation if data already exists. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Notes + ----- + When `mode` is `Append`, if there is an existing table, we will use the format and + options of the existing table. The column order in the schema of the :class:`DataFrame` + doesn't need to be the same as that of the existing table. Unlike + :meth:`DataFrameWriter.insertInto`, :meth:`DataFrameWriter.saveAsTable` will use the + column names to find the correct column positions. + + Parameters + ---------- + name : str + the table name + format : str, optional + the format used to save + mode : str, optional + one of `append`, `overwrite`, `error`, `errorifexists`, `ignore` \ + (default: error) + partitionBy : str or list + names of partitioning columns + **options : dict + all other string options + + Examples + -------- + Creates a table from a DataFrame, and read it back. + + >>> _ = spark.sql("DROP TABLE IF EXISTS tblA") + >>> spark.createDataFrame([ + ... (100, "Hyukjin Kwon"), (120, "Hyukjin Kwon"), (140, "Haejoon Lee")], + ... schema=["age", "name"] + ... ).write.saveAsTable("tblA") + >>> spark.read.table("tblA").sort("age").show() + +---+------------+ + |age| name| + +---+------------+ + |100|Hyukjin Kwon| + |120|Hyukjin Kwon| + |140| Haejoon Lee| + +---+------------+ + >>> _ = spark.sql("DROP TABLE tblA") + """ + self.mode(mode).options(**options) + if partitionBy is not None: + self.partitionBy(partitionBy) + if format is not None: + self.format(format) + self._jwrite.saveAsTable(name) + + def json( + self, + path: str, + mode: Optional[str] = None, + compression: Optional[str] = None, + dateFormat: Optional[str] = None, + timestampFormat: Optional[str] = None, + lineSep: Optional[str] = None, + encoding: Optional[str] = None, + ignoreNullFields: Optional[Union[bool, str]] = None, + ) -> None: + """Saves the content of the :class:`DataFrame` in JSON format + (`JSON Lines text format or newline-delimited JSON `_) at the + specified path. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + path : str + the path in any Hadoop supported file system + mode : str, optional + specifies the behavior of the save operation when data already exists. + + * ``append``: Append contents of this :class:`DataFrame` to existing data. + * ``overwrite``: Overwrite existing data. + * ``ignore``: Silently ignore this operation if data already exists. + * ``error`` or ``errorifexists`` (default case): Throw an exception if data already \ + exists. + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ + for the version you use. + + .. # noqa + + Examples + -------- + Write a DataFrame into a JSON file and read it back. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="json") as d: + ... # Write a DataFrame into a JSON file + ... spark.createDataFrame( + ... [{"age": 100, "name": "Hyukjin Kwon"}] + ... ).write.json(d, mode="overwrite") + ... + ... # Read the JSON file as a DataFrame. + ... spark.read.format("json").load(d).show() + +---+------------+ + |age| name| + +---+------------+ + |100|Hyukjin Kwon| + +---+------------+ + """ + self.mode(mode) + self._set_opts( + compression=compression, + dateFormat=dateFormat, + timestampFormat=timestampFormat, + lineSep=lineSep, + encoding=encoding, + ignoreNullFields=ignoreNullFields, + ) + self._jwrite.json(path) + + def parquet( + self, + path: str, + mode: Optional[str] = None, + partitionBy: Optional[Union[str, List[str]]] = None, + compression: Optional[str] = None, + ) -> None: + """Saves the content of the :class:`DataFrame` in Parquet format at the specified path. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + path : str + the path in any Hadoop supported file system + mode : str, optional + specifies the behavior of the save operation when data already exists. + + * ``append``: Append contents of this :class:`DataFrame` to existing data. + * ``overwrite``: Overwrite existing data. + * ``ignore``: Silently ignore this operation if data already exists. + * ``error`` or ``errorifexists`` (default case): Throw an exception if data already \ + exists. + partitionBy : str or list, optional + names of partitioning columns + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ + for the version you use. + + .. # noqa + + Examples + -------- + Write a DataFrame into a Parquet file and read it back. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="parquet") as d: + ... # Write a DataFrame into a Parquet file + ... spark.createDataFrame( + ... [{"age": 100, "name": "Hyukjin Kwon"}] + ... ).write.parquet(d, mode="overwrite") + ... + ... # Read the Parquet file as a DataFrame. + ... spark.read.format("parquet").load(d).show() + +---+------------+ + |age| name| + +---+------------+ + |100|Hyukjin Kwon| + +---+------------+ + """ + self.mode(mode) + if partitionBy is not None: + self.partitionBy(partitionBy) + self._set_opts(compression=compression) + self._jwrite.parquet(path) + + def text( + self, path: str, compression: Optional[str] = None, lineSep: Optional[str] = None + ) -> None: + """Saves the content of the DataFrame in a text file at the specified path. + The text files will be encoded as UTF-8. + + .. versionadded:: 1.6.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + path : str + the path in any Hadoop supported file system + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ + for the version you use. + + .. # noqa + + Notes + ----- + The DataFrame must have only one column that is of string type. + Each row becomes a new line in the output file. + + Examples + -------- + Write a DataFrame into a text file and read it back. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="text") as d: + ... # Write a DataFrame into a text file + ... df = spark.createDataFrame([("a",), ("b",), ("c",)], schema=["alphabets"]) + ... df.write.mode("overwrite").text(d) + ... + ... # Read the text file as a DataFrame. + ... spark.read.schema(df.schema).format("text").load(d).sort("alphabets").show() + +---------+ + |alphabets| + +---------+ + | a| + | b| + | c| + +---------+ + """ + self._set_opts(compression=compression, lineSep=lineSep) + self._jwrite.text(path) + + def csv( + self, + path: str, + mode: Optional[str] = None, + compression: Optional[str] = None, + sep: Optional[str] = None, + quote: Optional[str] = None, + escape: Optional[str] = None, + header: Optional[Union[bool, str]] = None, + nullValue: Optional[str] = None, + escapeQuotes: Optional[Union[bool, str]] = None, + quoteAll: Optional[Union[bool, str]] = None, + dateFormat: Optional[str] = None, + timestampFormat: Optional[str] = None, + ignoreLeadingWhiteSpace: Optional[Union[bool, str]] = None, + ignoreTrailingWhiteSpace: Optional[Union[bool, str]] = None, + charToEscapeQuoteEscaping: Optional[str] = None, + encoding: Optional[str] = None, + emptyValue: Optional[str] = None, + lineSep: Optional[str] = None, + ) -> None: + r"""Saves the content of the :class:`DataFrame` in CSV format at the specified path. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + path : str + the path in any Hadoop supported file system + mode : str, optional + specifies the behavior of the save operation when data already exists. + + * ``append``: Append contents of this :class:`DataFrame` to existing data. + * ``overwrite``: Overwrite existing data. + * ``ignore``: Silently ignore this operation if data already exists. + * ``error`` or ``errorifexists`` (default case): Throw an exception if data already \ + exists. + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ + for the version you use. + + .. # noqa + + Examples + -------- + Write a DataFrame into a CSV file and read it back. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="csv") as d: + ... # Write a DataFrame into a CSV file + ... df = spark.createDataFrame([{"age": 100, "name": "Hyukjin Kwon"}]) + ... df.write.csv(d, mode="overwrite") + ... + ... # Read the CSV file as a DataFrame with 'nullValue' option set to 'Hyukjin Kwon'. + ... spark.read.schema(df.schema).format("csv").option( + ... "nullValue", "Hyukjin Kwon").load(d).show() + +---+----+ + |age|name| + +---+----+ + |100|NULL| + +---+----+ + """ + self.mode(mode) + self._set_opts( + compression=compression, + sep=sep, + quote=quote, + escape=escape, + header=header, + nullValue=nullValue, + escapeQuotes=escapeQuotes, + quoteAll=quoteAll, + dateFormat=dateFormat, + timestampFormat=timestampFormat, + ignoreLeadingWhiteSpace=ignoreLeadingWhiteSpace, + ignoreTrailingWhiteSpace=ignoreTrailingWhiteSpace, + charToEscapeQuoteEscaping=charToEscapeQuoteEscaping, + encoding=encoding, + emptyValue=emptyValue, + lineSep=lineSep, + ) + self._jwrite.csv(path) + + def xml( + self, + path: str, + rowTag: Optional[str] = None, + mode: Optional[str] = None, + attributePrefix: Optional[str] = None, + valueTag: Optional[str] = None, + rootTag: Optional[str] = None, + declaration: Optional[bool] = None, + arrayElementName: Optional[str] = None, + nullValue: Optional[str] = None, + dateFormat: Optional[str] = None, + timestampFormat: Optional[str] = None, + compression: Optional[str] = None, + encoding: Optional[str] = None, + validateName: Optional[bool] = None, + ) -> None: + r"""Saves the content of the :class:`DataFrame` in XML format at the specified path. + + .. versionadded:: 4.0.0 + + Parameters + ---------- + path : str + the path in any Hadoop supported file system + mode : str, optional + specifies the behavior of the save operation when data already exists. + + * ``append``: Append contents of this :class:`DataFrame` to existing data. + * ``overwrite``: Overwrite existing data. + * ``ignore``: Silently ignore this operation if data already exists. + * ``error`` or ``errorifexists`` (default case): Throw an exception if data already \ + exists. + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ + for the version you use. + + .. # noqa + + Examples + -------- + Write a DataFrame into a XML file and read it back. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="xml") as d: + ... # Write a DataFrame into a XML file + ... spark.createDataFrame( + ... [{"age": 100, "name": "Hyukjin Kwon"}] + ... ).write.mode("overwrite").option("rowTag", "person").xml(d) + ... + ... # Read the XML file as a DataFrame. + ... spark.read.option("rowTag", "person").format("xml").load(d).show() + +---+------------+ + |age| name| + +---+------------+ + |100|Hyukjin Kwon| + +---+------------+ + """ + self.mode(mode) + self._set_opts( + rowTag=rowTag, + attributePrefix=attributePrefix, + valueTag=valueTag, + rootTag=rootTag, + declaration=declaration, + arrayElementName=arrayElementName, + nullValue=nullValue, + dateFormat=dateFormat, + timestampFormat=timestampFormat, + compression=compression, + encoding=encoding, + validateName=validateName, + ) + self._jwrite.xml(path) + + def orc( + self, + path: str, + mode: Optional[str] = None, + partitionBy: Optional[Union[str, List[str]]] = None, + compression: Optional[str] = None, + ) -> None: + """Saves the content of the :class:`DataFrame` in ORC format at the specified path. + + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + path : str + the path in any Hadoop supported file system + mode : str, optional + specifies the behavior of the save operation when data already exists. + + * ``append``: Append contents of this :class:`DataFrame` to existing data. + * ``overwrite``: Overwrite existing data. + * ``ignore``: Silently ignore this operation if data already exists. + * ``error`` or ``errorifexists`` (default case): Throw an exception if data already \ + exists. + partitionBy : str or list, optional + names of partitioning columns + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ + for the version you use. + + .. # noqa + + Examples + -------- + Write a DataFrame into a ORC file and read it back. + + >>> import tempfile + >>> with tempfile.TemporaryDirectory(prefix="orc") as d: + ... # Write a DataFrame into a ORC file + ... spark.createDataFrame( + ... [{"age": 100, "name": "Hyukjin Kwon"}] + ... ).write.orc(d, mode="overwrite") + ... + ... # Read the Parquet file as a DataFrame. + ... spark.read.format("orc").load(d).show() + +---+------------+ + |age| name| + +---+------------+ + |100|Hyukjin Kwon| + +---+------------+ + """ + self.mode(mode) + if partitionBy is not None: + self.partitionBy(partitionBy) + self._set_opts(compression=compression) + self._jwrite.orc(path) + + def jdbc( + self, + url: str, + table: str, + mode: Optional[str] = None, + properties: Optional[Dict[str, str]] = None, + ) -> None: + """Saves the content of the :class:`DataFrame` to an external database table via JDBC. + + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + table : str + Name of the table in the external database. + mode : str, optional + specifies the behavior of the save operation when data already exists. + + * ``append``: Append contents of this :class:`DataFrame` to existing data. + * ``overwrite``: Overwrite existing data. + * ``ignore``: Silently ignore this operation if data already exists. + * ``error`` or ``errorifexists`` (default case): Throw an exception if data already \ + exists. + properties : dict + a dictionary of JDBC database connection arguments. Normally at + least properties "user" and "password" with their corresponding values. + For example { 'user' : 'SYSTEM', 'password' : 'mypassword' } + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ + for the version you use. + + .. # noqa + + Notes + ----- + Don't create too many partitions in parallel on a large cluster; + otherwise Spark might crash your external database systems. + """ + from py4j.java_gateway import JavaClass + + if properties is None: + properties = dict() + + assert self._spark._sc._gateway is not None + jprop = JavaClass( + "java.util.Properties", + self._spark._sc._gateway._gateway_client, + )() + for k in properties: + jprop.setProperty(k, properties[k]) + self.mode(mode)._jwrite.jdbc(url, table, jprop) + + +class DataFrameWriterV2: + """ + Interface used to write a class:`pyspark.sql.dataframe.DataFrame` + to external storage using the v2 API. + + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + """ + + def __init__(self, df: "DataFrame", table: str): + self._df = df + self._spark = df.sparkSession + self._jwriter = df._jdf.writeTo(table) + + def using(self, provider: str) -> "DataFrameWriterV2": + """ + Specifies a provider for the underlying output data source. + Spark's default catalog supports "parquet", "json", etc. + + .. versionadded:: 3.1.0 + """ + self._jwriter.using(provider) + return self + + def option(self, key: str, value: "OptionalPrimitiveType") -> "DataFrameWriterV2": + """ + Add a write option. + + .. versionadded:: 3.1.0 + """ + if value is None: + return self + self._jwriter.option(key, to_str(value)) + return self + + def options(self, **options: "OptionalPrimitiveType") -> "DataFrameWriterV2": + """ + Add write options. + + .. versionadded:: 3.1.0 + """ + options = {k: to_str(v) for k, v in options.items() if v is not None} + self._jwriter.options(options) + return self + + def tableProperty(self, property: str, value: str) -> "DataFrameWriterV2": + """ + Add table property. + + .. versionadded:: 3.1.0 + """ + self._jwriter.tableProperty(property, value) + return self + + def partitionedBy(self, col: "ColumnOrName", *cols: "ColumnOrName") -> "DataFrameWriterV2": + """ + Partition the output table created by `create`, `createOrReplace`, or `replace` using + the given columns or transforms. + + When specified, the table data will be stored by these values for efficient reads. + + For example, when a table is partitioned by day, it may be stored + in a directory layout like: + + * `table/day=2019-06-01/` + * `table/day=2019-06-02/` + + Partitioning is one of the most widely used techniques to optimize physical data layout. + It provides a coarse-grained index for skipping unnecessary data reads when queries have + predicates on the partitioned columns. In order for partitioning to work well, the number + of distinct values in each column should typically be less than tens of thousands. + + `col` and `cols` support only the following functions: + + * :py:func:`pyspark.sql.functions.years` + * :py:func:`pyspark.sql.functions.months` + * :py:func:`pyspark.sql.functions.days` + * :py:func:`pyspark.sql.functions.hours` + * :py:func:`pyspark.sql.functions.bucket` + + .. versionadded:: 3.1.0 + """ + from pyspark.sql.classic.column import _to_seq, _to_java_column + + col = _to_java_column(col) + cols = _to_seq(self._spark._sc, [_to_java_column(c) for c in cols]) + self._jwriter.partitionedBy(col, cols) + return self + + def clusterBy(self, col: str, *cols: str) -> "DataFrameWriterV2": + """ + Clusters the data by the given columns to optimize query performance. + """ + from pyspark.sql.classic.column import _to_seq + + self._jwriter.clusterBy(col, _to_seq(self._spark._sc, cols)) + return self + + def create(self) -> None: + """ + Create a new table from the contents of the data frame. + + The new table's schema, partition layout, properties, and other configuration will be + based on the configuration set on this writer. + + .. versionadded:: 3.1.0 + """ + self._jwriter.create() + + def replace(self) -> None: + """ + Replace an existing table with the contents of the data frame. + + The existing table's schema, partition layout, properties, and other configuration will be + replaced with the contents of the data frame and the configuration set on this writer. + + .. versionadded:: 3.1.0 + """ + self._jwriter.replace() + + def createOrReplace(self) -> None: + """ + Create a new table or replace an existing table with the contents of the data frame. + + The output table's schema, partition layout, properties, + and other configuration will be based on the contents of the data frame + and the configuration set on this writer. + If the table exists, its configuration and data will be replaced. + + .. versionadded:: 3.1.0 + """ + self._jwriter.createOrReplace() + + def append(self) -> None: + """ + Append the contents of the data frame to the output table. + + .. versionadded:: 3.1.0 + """ + self._jwriter.append() + + def overwrite(self, condition: "ColumnOrName") -> None: + """ + Overwrite rows matching the given filter condition with the contents of the data frame in + the output table. + + .. versionadded:: 3.1.0 + """ + from pyspark.sql.classic.column import _to_java_column + + condition = _to_java_column(condition) + self._jwriter.overwrite(condition) + + def overwritePartitions(self) -> None: + """ + Overwrite all partition for which the data frame contains at least one row with the contents + of the data frame in the output table. + + This operation is equivalent to Hive's `INSERT OVERWRITE ... PARTITION`, which replaces + partitions dynamically depending on the contents of the data frame. + + .. versionadded:: 3.1.0 + """ + self._jwriter.overwritePartitions() + + +def _test() -> None: + import doctest + import os + import py4j + from pyspark.core.context import SparkContext + from pyspark.sql import SparkSession + import pyspark.sql.readwriter + + os.chdir(os.environ["SPARK_HOME"]) + + globs = pyspark.sql.readwriter.__dict__.copy() + sc = SparkContext("local[4]", "PythonTest") + try: + spark = SparkSession._getActiveSessionOrCreate() + except py4j.protocol.Py4JError: + spark = SparkSession(sc) + + globs["spark"] = spark + failure_count, test_count = doctest.testmod( + pyspark.sql.readwriter, + globs=globs, + optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF, + ) + spark.stop() + if failure_count: + sys.exit(-1) + + +if __name__ == "__main__": + _test() diff --git a/python/pyspark/sql/readwriter.pyi b/python/pyspark/sql/readwriter.pyi deleted file mode 100644 index b4b56670..00000000 --- a/python/pyspark/sql/readwriter.pyi +++ /dev/null @@ -1,1979 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Generated by scripts/gen_type_stubs.py from python/pyspark/sql/readwriter.py of Apache Spark v4.3.0: -# do not edit. Type checkers read this stub instead of the module of the drop-in. - -from typing import overload, Dict, List, Optional, Tuple, Union -from pyspark.sql.types import StructType -from pyspark.sql.utils import to_str -from pyspark.errors import PySparkTypeError -from py4j.java_gateway import JavaObject -from pyspark.core.rdd import RDD -from pyspark.sql._typing import OptionalPrimitiveType, ColumnOrName -from pyspark.sql.session import SparkSession -from pyspark.sql.dataframe import DataFrame -from pyspark.sql.streaming import StreamingQuery - -__all__ = ['DataFrameReader', 'DataFrameWriter', 'DataFrameWriterV2'] -PathOrPaths = Union[str, List[str]] -TupleOrListOfString = Union[List[str], Tuple[str, ...]] - -class OptionUtils: - def _set_opts(self, schema: Optional[Union[StructType, str]]=None, **options: 'OptionalPrimitiveType') -> None: - """Set named options (filter out those the value is None)""" - -class DataFrameReader(OptionUtils): - """Interface used to load a :class:`DataFrame` from external storage systems - (e.g. file systems, key-value stores, etc). Use :attr:`SparkSession.read` - to access this. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - """ - def __init__(self, spark: 'SparkSession'): - self._jreader = spark._jsparkSession.read() - self._spark = spark - def _df(self, jdf: 'JavaObject') -> 'DataFrame': ... - def format(self, source: str) -> 'DataFrameReader': - """Specifies the input data source format. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - source : str - string, name of the data source, e.g. 'json', 'parquet'. - - Examples - -------- - >>> spark.read.format('json') - <...readwriter.DataFrameReader object ...> - - Write a DataFrame into a JSON file and read it back. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="format") as d: - ... # Write a DataFrame into a JSON file - ... spark.createDataFrame( - ... [{"age": 100, "name": "Hyukjin Kwon"}] - ... ).write.mode("overwrite").format("json").save(d) - ... - ... # Read the JSON file as a DataFrame. - ... spark.read.format('json').load(d).show() - +---+------------+ - |age| name| - +---+------------+ - |100|Hyukjin Kwon| - +---+------------+ - """ - self._jreader = self._jreader.format(source) - return self - def schema(self, schema: Union[StructType, str]) -> 'DataFrameReader': - """Specifies the input schema. - - Some data sources (e.g. JSON) can infer the input schema automatically from data. - By specifying the schema here, the underlying data source can skip the schema - inference step, and thus speed up data loading. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - schema : :class:`pyspark.sql.types.StructType` or str - a :class:`pyspark.sql.types.StructType` object or a DDL-formatted string - (For example ``col0 INT, col1 DOUBLE``). - - Examples - -------- - >>> spark.read.schema("col0 INT, col1 DOUBLE") - <...readwriter.DataFrameReader object ...> - - Specify the schema with reading a CSV file. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="schema") as d: - ... spark.read.schema("col0 INT, col1 DOUBLE").format("csv").load(d).printSchema() - root - |-- col0: integer (nullable = true) - |-- col1: double (nullable = true) - """ - from pyspark.sql import SparkSession - - spark = SparkSession._getActiveSessionOrCreate() - if isinstance(schema, StructType): - jschema = spark._jsparkSession.parseDataType(schema.json()) - self._jreader = self._jreader.schema(jschema) - elif isinstance(schema, str): - self._jreader = self._jreader.schema(schema) - else: - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "arg_name": "schema", - "expected_type": "str or struct type", - "arg_type": type(schema).__name__, - }, - ) - return self - def option(self, key: str, value: 'OptionalPrimitiveType') -> 'DataFrameReader': - """Adds an input option for the underlying data source. - - .. versionadded:: 1.5.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - key : str - The key for the option to set. - value - The value for the option to set. - - Examples - -------- - >>> spark.read.option("key", "value") - <...readwriter.DataFrameReader object ...> - - Specify the option 'nullValue' with reading a CSV file. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="option") as d: - ... # Write a DataFrame into a CSV file - ... df = spark.createDataFrame([{"age": 100, "name": "Hyukjin Kwon"}]) - ... df.write.mode("overwrite").format("csv").save(d) - ... - ... # Read the CSV file as a DataFrame with 'nullValue' option set to 'Hyukjin Kwon'. - ... spark.read.schema(df.schema).option( - ... "nullValue", "Hyukjin Kwon").format('csv').load(d).show() - +---+----+ - |age|name| - +---+----+ - |100|NULL| - +---+----+ - """ - if value is None: - return self - self._jreader = self._jreader.option(key, to_str(value)) - return self - def options(self, **options: 'OptionalPrimitiveType') -> 'DataFrameReader': - """Adds input options for the underlying data source. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - **options : dict - The dictionary of string keys and primitive-type values. - - Examples - -------- - >>> spark.read.options(key="value") - <...readwriter.DataFrameReader object ...> - - Specify options in a dictionary. - - >>> spark.read.options(**{"k1": "v1", "k2": "v2"}) - <...readwriter.DataFrameReader object ...> - - Specify the option 'nullValue' and 'header' with reading a CSV file. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="options") as d: - ... # Write a DataFrame into a CSV file with a header. - ... df = spark.createDataFrame([{"age": 100, "name": "Hyukjin Kwon"}]) - ... df.write.option("header", True).mode("overwrite").format("csv").save(d) - ... - ... # Read the CSV file as a DataFrame with 'nullValue' option set to 'Hyukjin Kwon', - ... # and 'header' option set to `True`. - ... spark.read.options( - ... nullValue="Hyukjin Kwon", - ... header=True - ... ).format('csv').load(d).show() - +---+----+ - |age|name| - +---+----+ - |100|NULL| - +---+----+ - """ - for k, v in options.items(): - if v is not None: - self._jreader = self._jreader.option(k, to_str(v)) - return self - def load(self, path: Optional[PathOrPaths]=None, format: Optional[str]=None, schema: Optional[Union[StructType, str]]=None, **options: 'OptionalPrimitiveType') -> 'DataFrame': - """Loads data from a data source and returns it as a :class:`DataFrame`. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - path : str or list, optional - optional string or a list of string for file-system backed data sources. - format : str, optional - optional string for format of the data source. Default to 'parquet'. - schema : :class:`pyspark.sql.types.StructType` or str, optional - optional :class:`pyspark.sql.types.StructType` for the input schema - or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). - **options : dict - all other string options - - Examples - -------- - Load a CSV file with format, schema and options specified. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="load") as d: - ... # Write a DataFrame into a CSV file with a header - ... df = spark.createDataFrame([{"age": 100, "name": "Hyukjin Kwon"}]) - ... df.write.option("header", True).mode("overwrite").format("csv").save(d) - ... - ... # Read the CSV file as a DataFrame with 'nullValue' option set to 'Hyukjin Kwon', - ... # and 'header' option set to `True`. - ... df = spark.read.load( - ... d, schema=df.schema, format="csv", nullValue="Hyukjin Kwon", header=True) - ... df.printSchema() - ... df.show() - root - |-- age: long (nullable = true) - |-- name: string (nullable = true) - +---+----+ - |age|name| - +---+----+ - |100|NULL| - +---+----+ - """ - def json(self, path: Union[str, List[str], 'RDD[str]', 'DataFrame'], schema: Optional[Union[StructType, str]]=None, primitivesAsString: Optional[Union[bool, str]]=None, prefersDecimal: Optional[Union[bool, str]]=None, allowComments: Optional[Union[bool, str]]=None, allowUnquotedFieldNames: Optional[Union[bool, str]]=None, allowSingleQuotes: Optional[Union[bool, str]]=None, allowNumericLeadingZero: Optional[Union[bool, str]]=None, allowBackslashEscapingAnyCharacter: Optional[Union[bool, str]]=None, mode: Optional[str]=None, columnNameOfCorruptRecord: Optional[str]=None, dateFormat: Optional[str]=None, timestampFormat: Optional[str]=None, multiLine: Optional[Union[bool, str]]=None, allowUnquotedControlChars: Optional[Union[bool, str]]=None, lineSep: Optional[str]=None, samplingRatio: Optional[Union[float, str]]=None, dropFieldIfAllNull: Optional[Union[bool, str]]=None, encoding: Optional[str]=None, locale: Optional[str]=None, pathGlobFilter: Optional[Union[bool, str]]=None, recursiveFileLookup: Optional[Union[bool, str]]=None, modifiedBefore: Optional[Union[bool, str]]=None, modifiedAfter: Optional[Union[bool, str]]=None, allowNonNumericNumbers: Optional[Union[bool, str]]=None, useUnsafeRow: Optional[Union[bool, str]]=None) -> 'DataFrame': - """Loads JSON files and returns the results as a :class:`DataFrame`. - - `JSON Lines `_ (newline-delimited JSON) is supported by default. - For JSON (one record per file), set the ``multiLine`` parameter to ``true``. - - If the ``schema`` parameter is not specified, this function goes - through the input once to determine the input schema. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - .. versionchanged:: 4.2.0 - Supports DataFrame input. - - Parameters - ---------- - path : str, list, :class:`RDD`, or :class:`DataFrame` - string represents path to the JSON dataset, or a list of paths, - or RDD of Strings storing JSON objects, - or a DataFrame with a single string column containing JSON strings. - schema : :class:`pyspark.sql.types.StructType` or str, optional - an optional :class:`pyspark.sql.types.StructType` for the input schema or - a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - for the version you use. - - .. # noqa - - Examples - -------- - Example 1: Write a DataFrame into a JSON file and read it back. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="json1") as d: - ... # Write a DataFrame into a JSON file - ... spark.createDataFrame( - ... [{"age": 100, "name": "Hyukjin"}] - ... ).write.mode("overwrite").format("json").save(d) - ... - ... # Read the JSON file as a DataFrame. - ... spark.read.json(d).show() - +---+-------+ - |age| name| - +---+-------+ - |100|Hyukjin| - +---+-------+ - - Example 2: Read JSON from multiple files in a directory - - >>> from tempfile import TemporaryDirectory - >>> with TemporaryDirectory(prefix="json2") as d1, TemporaryDirectory(prefix="json3") as d2: - ... # Write a DataFrame into a JSON file - ... spark.createDataFrame( - ... [{"age": 30, "name": "Bob"}] - ... ).write.mode("overwrite").format("json").save(d1) - ... - ... # Read the JSON files as a DataFrame. - ... spark.createDataFrame( - ... [{"age": 25, "name": "Alice"}] - ... ).write.mode("overwrite").format("json").save(d2) - ... spark.read.json([d1, d2]).show() - +---+-----+ - |age| name| - +---+-----+ - | 25|Alice| - | 30| Bob| - +---+-----+ - - Example 3: Read JSON with a custom schema - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="json4") as d: - ... # Write a DataFrame into a JSON file - ... spark.createDataFrame( - ... [{"age": 30, "name": "Bob"}] - ... ).write.mode("overwrite").format("json").save(d) - ... custom_schema = "name STRING, age INT" - ... spark.read.json(d, schema=custom_schema).show() - +----+---+ - |name|age| - +----+---+ - | Bob| 30| - +----+---+ - - Example 4: Parse JSON from a DataFrame with a single string column. - - >>> json_df = spark.createDataFrame( - ... [('{"name": "Alice", "age": 25}',), ('{"name": "Bob", "age": 30}',)], - ... schema="value STRING", - ... ) - >>> spark.read.json(json_df).sort("name").show() - +---+-----+ - |age| name| - +---+-----+ - | 25|Alice| - | 30| Bob| - +---+-----+ - """ - def table(self, tableName: str) -> 'DataFrame': - """Returns the specified table as a :class:`DataFrame`. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - tableName : str - string, name of the table. - - Examples - -------- - >>> df = spark.range(10) - >>> df.createOrReplaceTempView('tblA') - >>> spark.read.table('tblA').show() - +---+ - | id| - +---+ - | 0| - | 1| - | 2| - | 3| - | 4| - | 5| - | 6| - | 7| - | 8| - | 9| - +---+ - >>> _ = spark.sql("DROP TABLE tblA") - """ - def changes(self, tableName: str) -> 'DataFrame': - """Returns the row-level changes (Change Data Capture) from the specified table - as a :class:`DataFrame`. Currently this API is only supported for Data Source V2 - tables whose catalog implements ``TableCatalog.loadChangelog()``. - - Use :meth:`option` to specify the version/timestamp range and processing options. - - .. versionadded:: 4.2.0 - - Parameters - ---------- - tableName : str - string, name of the table. - - Returns - ------- - :class:`DataFrame` - - Examples - -------- - >>> spark.read.option("startingVersion", "10").option( - ... "endingVersion", "20" - ... ).changes("my_table") # doctest: +SKIP - """ - def parquet(self, *paths: str, **options: 'OptionalPrimitiveType') -> 'DataFrame': - """Loads Parquet files, returning the result as a :class:`DataFrame`. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - paths : str - One or more file paths to read the Parquet files from. - - Other Parameters - ---------------- - **options - For the extra options, refer to - `Data Source Option `_ - for the version you use. - - .. # noqa - - Returns - ------- - :class:`DataFrame` - A DataFrame containing the data from the Parquet files. - - Examples - -------- - Create sample dataframes. - - >>> df = spark.createDataFrame( - ... [(10, "Alice"), (15, "Bob"), (20, "Tom")], schema=["age", "name"]) - >>> df2 = spark.createDataFrame([(70, "Alice"), (80, "Bob")], schema=["height", "name"]) - - Write a DataFrame into a Parquet file and read it back. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="parquet1") as d: - ... # Write a DataFrame into a Parquet file. - ... df.write.mode("overwrite").format("parquet").save(d) - ... - ... # Read the Parquet file as a DataFrame. - ... spark.read.parquet(d).orderBy("name").show() - +---+-----+ - |age| name| - +---+-----+ - | 10|Alice| - | 15| Bob| - | 20| Tom| - +---+-----+ - - Read a Parquet file with a specific column. - - >>> with tempfile.TemporaryDirectory(prefix="parquet2") as d: - ... df.write.mode("overwrite").format("parquet").save(d) - ... - ... # Read the Parquet file with only the 'name' column. - ... spark.read.schema("name string").parquet(d).orderBy("name").show() - +-----+ - | name| - +-----+ - |Alice| - | Bob| - | Tom| - +-----+ - - Read multiple Parquet files and merge schema. - - >>> with tempfile.TemporaryDirectory(prefix="parquet3") as d1: - ... with tempfile.TemporaryDirectory(prefix="parquet4") as d2: - ... df.write.mode("overwrite").format("parquet").save(d1) - ... df2.write.mode("overwrite").format("parquet").save(d2) - ... - ... spark.read.option( - ... "mergeSchema", "true" - ... ).parquet(d1, d2).select( - ... "name", "age", "height" - ... ).orderBy("name", "age").show() - +-----+----+------+ - | name| age|height| - +-----+----+------+ - |Alice|NULL| 70| - |Alice| 10| NULL| - | Bob|NULL| 80| - | Bob| 15| NULL| - | Tom| 20| NULL| - +-----+----+------+ - """ - def text(self, paths: PathOrPaths, wholetext: bool=False, lineSep: Optional[str]=None, pathGlobFilter: Optional[Union[bool, str]]=None, recursiveFileLookup: Optional[Union[bool, str]]=None, modifiedBefore: Optional[Union[bool, str]]=None, modifiedAfter: Optional[Union[bool, str]]=None) -> 'DataFrame': - """Loads text files and returns a :class:`DataFrame` whose schema starts with a - string column named "value", and followed by partitioned columns if there - are any. - The text files must be encoded as UTF-8. - - By default, each line in the text file is a new row in the resulting DataFrame. - - .. versionadded:: 1.6.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - paths : str or list - string, or list of strings, for input path(s). - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - for the version you use. - - .. # noqa - - Examples - -------- - Write a DataFrame into a text file and read it back. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="text") as d: - ... # Write a DataFrame into a text file - ... df = spark.createDataFrame([("a",), ("b",), ("c",)], schema=["alphabets"]) - ... df.write.mode("overwrite").format("text").save(d) - ... - ... # Read the text file as a DataFrame. - ... spark.read.schema(df.schema).text(d).sort("alphabets").show() - +---------+ - |alphabets| - +---------+ - | a| - | b| - | c| - +---------+ - """ - def csv(self, path: Union[str, List[str], 'RDD[str]', 'DataFrame'], schema: Optional[Union[StructType, str]]=None, sep: Optional[str]=None, encoding: Optional[str]=None, quote: Optional[str]=None, escape: Optional[str]=None, comment: Optional[str]=None, header: Optional[Union[bool, str]]=None, inferSchema: Optional[Union[bool, str]]=None, ignoreLeadingWhiteSpace: Optional[Union[bool, str]]=None, ignoreTrailingWhiteSpace: Optional[Union[bool, str]]=None, nullValue: Optional[str]=None, nanValue: Optional[str]=None, positiveInf: Optional[str]=None, negativeInf: Optional[str]=None, dateFormat: Optional[str]=None, timestampFormat: Optional[str]=None, maxColumns: Optional[Union[int, str]]=None, maxCharsPerColumn: Optional[Union[int, str]]=None, maxMalformedLogPerPartition: Optional[Union[int, str]]=None, mode: Optional[str]=None, columnNameOfCorruptRecord: Optional[str]=None, multiLine: Optional[Union[bool, str]]=None, charToEscapeQuoteEscaping: Optional[str]=None, samplingRatio: Optional[Union[float, str]]=None, enforceSchema: Optional[Union[bool, str]]=None, emptyValue: Optional[str]=None, locale: Optional[str]=None, lineSep: Optional[str]=None, pathGlobFilter: Optional[Union[bool, str]]=None, recursiveFileLookup: Optional[Union[bool, str]]=None, modifiedBefore: Optional[Union[bool, str]]=None, modifiedAfter: Optional[Union[bool, str]]=None, unescapedQuoteHandling: Optional[str]=None) -> 'DataFrame': - """Loads a CSV file and returns the result as a :class:`DataFrame`. - - This function will go through the input once to determine the input schema if - ``inferSchema`` is enabled. To avoid going through the entire data once, disable - ``inferSchema`` option or specify the schema explicitly using ``schema``. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - .. versionchanged:: 4.2.0 - Supports DataFrame input. - - Parameters - ---------- - path : str, list, :class:`RDD`, or :class:`DataFrame` - string, or list of strings, for input path(s), - or RDD of Strings storing CSV rows, - or a DataFrame with a single string column containing CSV rows. - schema : :class:`pyspark.sql.types.StructType` or str, optional - an optional :class:`pyspark.sql.types.StructType` for the input schema - or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - for the version you use. - - .. # noqa - - Examples - -------- - Write a DataFrame into a CSV file and read it back. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="csv") as d: - ... # Write a DataFrame into a CSV file - ... df = spark.createDataFrame([{"age": 100, "name": "Hyukjin Kwon"}]) - ... df.write.mode("overwrite").format("csv").save(d) - ... - ... # Read the CSV file as a DataFrame with 'nullValue' option set to 'Hyukjin Kwon'. - ... spark.read.csv(d, schema=df.schema, nullValue="Hyukjin Kwon").show() - +---+----+ - |age|name| - +---+----+ - |100|NULL| - +---+----+ - """ - def xml(self, path: Union[str, List[str], 'RDD[str]', 'DataFrame'], rowTag: Optional[str]=None, schema: Optional[Union[StructType, str]]=None, excludeAttribute: Optional[Union[bool, str]]=None, attributePrefix: Optional[str]=None, valueTag: Optional[str]=None, ignoreSurroundingSpaces: Optional[Union[bool, str]]=None, rowValidationXSDPath: Optional[str]=None, ignoreNamespace: Optional[Union[bool, str]]=None, wildcardColName: Optional[str]=None, encoding: Optional[str]=None, inferSchema: Optional[Union[bool, str]]=None, nullValue: Optional[str]=None, dateFormat: Optional[str]=None, timestampFormat: Optional[str]=None, mode: Optional[str]=None, columnNameOfCorruptRecord: Optional[str]=None, multiLine: Optional[Union[bool, str]]=None, samplingRatio: Optional[Union[float, str]]=None, locale: Optional[str]=None) -> 'DataFrame': - """Loads a XML file and returns the result as a :class:`DataFrame`. - - If the ``schema`` parameter is not specified, this function goes - through the input once to determine the input schema. - - .. versionadded:: 4.0.0 - - .. versionchanged:: 4.2.0 - Supports DataFrame input. - - Parameters - ---------- - path : str, list, :class:`RDD`, or :class:`DataFrame` - string, or list of strings, for input path(s), - or RDD of Strings storing XML rows, - or a DataFrame with a single string column containing XML strings. - schema : :class:`pyspark.sql.types.StructType` or str, optional - an optional :class:`pyspark.sql.types.StructType` for the input schema - or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - for the version you use. - - .. # noqa - - Examples - -------- - Example 1: Write a DataFrame into a XML file and read it back. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="xml") as d: - ... # Write a DataFrame into a XML file - ... spark.createDataFrame( - ... [{"age": 100, "name": "Hyukjin Kwon"}] - ... ).write.mode("overwrite").option("rowTag", "person").format("xml").save(d) - ... - ... # Read the XML file as a DataFrame. - ... spark.read.option("rowTag", "person").xml(d).show() - +---+------------+ - |age| name| - +---+------------+ - |100|Hyukjin Kwon| - +---+------------+ - - Example 2: Parse XML from a DataFrame with a single string column. - - >>> xml_df = spark.createDataFrame( - ... [('Alice25',), - ... ('Bob30',)], - ... schema="value STRING", - ... ) - >>> spark.read.option("rowTag", "person").xml(xml_df).sort("name").show() - +---+-----+ - |age| name| - +---+-----+ - | 25|Alice| - | 30| Bob| - +---+-----+ - """ - def orc(self, path: PathOrPaths, mergeSchema: Optional[bool]=None, pathGlobFilter: Optional[Union[bool, str]]=None, recursiveFileLookup: Optional[Union[bool, str]]=None, modifiedBefore: Optional[Union[bool, str]]=None, modifiedAfter: Optional[Union[bool, str]]=None) -> 'DataFrame': - """Loads ORC files, returning the result as a :class:`DataFrame`. - - .. versionadded:: 1.5.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - path : str or list - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - for the version you use. - - .. # noqa - - Examples - -------- - Write a DataFrame into a ORC file and read it back. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="orc") as d: - ... # Write a DataFrame into a ORC file - ... spark.createDataFrame( - ... [{"age": 100, "name": "Hyukjin Kwon"}] - ... ).write.mode("overwrite").format("orc").save(d) - ... - ... # Read the Parquet file as a DataFrame. - ... spark.read.orc(d).show() - +---+------------+ - |age| name| - +---+------------+ - |100|Hyukjin Kwon| - +---+------------+ - """ - @overload - def jdbc(self, url: str, table: str, *, properties: Optional[Dict[str, str]]=None) -> 'DataFrame': - """Construct a :class:`DataFrame` representing the database table named ``table`` - accessible via JDBC URL ``url`` and connection ``properties``. - - Partitions of the table will be retrieved in parallel if either ``column`` or - ``predicates`` is specified. ``lowerBound``, ``upperBound`` and ``numPartitions`` - is needed when ``column`` is specified. - - If both ``column`` and ``predicates`` are specified, ``column`` will be used. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - table : str - the name of the table - column : str, optional - alias of ``partitionColumn`` option. Refer to ``partitionColumn`` in - `Data Source Option `_ - for the version you use. - predicates : list, optional - a list of expressions suitable for inclusion in WHERE clauses; - each one defines one partition of the :class:`DataFrame` - properties : dict, optional - a dictionary of JDBC database connection arguments. Normally at - least properties "user" and "password" with their corresponding values. - For example { 'user' : 'SYSTEM', 'password' : 'mypassword' } - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - for the version you use. - - .. # noqa - - Notes - ----- - Don't create too many partitions in parallel on a large cluster; - otherwise Spark might crash your external database systems. - - Returns - ------- - :class:`DataFrame` - """ - @overload - def jdbc(self, url: str, table: str, column: str, lowerBound: Union[int, str], upperBound: Union[int, str], numPartitions: int, *, properties: Optional[Dict[str, str]]=None) -> 'DataFrame': - """Construct a :class:`DataFrame` representing the database table named ``table`` - accessible via JDBC URL ``url`` and connection ``properties``. - - Partitions of the table will be retrieved in parallel if either ``column`` or - ``predicates`` is specified. ``lowerBound``, ``upperBound`` and ``numPartitions`` - is needed when ``column`` is specified. - - If both ``column`` and ``predicates`` are specified, ``column`` will be used. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - table : str - the name of the table - column : str, optional - alias of ``partitionColumn`` option. Refer to ``partitionColumn`` in - `Data Source Option `_ - for the version you use. - predicates : list, optional - a list of expressions suitable for inclusion in WHERE clauses; - each one defines one partition of the :class:`DataFrame` - properties : dict, optional - a dictionary of JDBC database connection arguments. Normally at - least properties "user" and "password" with their corresponding values. - For example { 'user' : 'SYSTEM', 'password' : 'mypassword' } - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - for the version you use. - - .. # noqa - - Notes - ----- - Don't create too many partitions in parallel on a large cluster; - otherwise Spark might crash your external database systems. - - Returns - ------- - :class:`DataFrame` - """ - @overload - def jdbc(self, url: str, table: str, *, predicates: List[str], properties: Optional[Dict[str, str]]=None) -> 'DataFrame': - """Construct a :class:`DataFrame` representing the database table named ``table`` - accessible via JDBC URL ``url`` and connection ``properties``. - - Partitions of the table will be retrieved in parallel if either ``column`` or - ``predicates`` is specified. ``lowerBound``, ``upperBound`` and ``numPartitions`` - is needed when ``column`` is specified. - - If both ``column`` and ``predicates`` are specified, ``column`` will be used. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - table : str - the name of the table - column : str, optional - alias of ``partitionColumn`` option. Refer to ``partitionColumn`` in - `Data Source Option `_ - for the version you use. - predicates : list, optional - a list of expressions suitable for inclusion in WHERE clauses; - each one defines one partition of the :class:`DataFrame` - properties : dict, optional - a dictionary of JDBC database connection arguments. Normally at - least properties "user" and "password" with their corresponding values. - For example { 'user' : 'SYSTEM', 'password' : 'mypassword' } - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - for the version you use. - - .. # noqa - - Notes - ----- - Don't create too many partitions in parallel on a large cluster; - otherwise Spark might crash your external database systems. - - Returns - ------- - :class:`DataFrame` - """ - -class DataFrameWriter(OptionUtils): - """Interface used to write a :class:`DataFrame` to external storage systems - (e.g. file systems, key-value stores, etc). Use :attr:`DataFrame.write` - to access this. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - """ - def __init__(self, df: 'DataFrame'): - self._df = df - self._spark = df.sparkSession - self._jwrite = df._jdf.write() - def _sq(self, jsq: 'JavaObject') -> 'StreamingQuery': ... - def mode(self, saveMode: Optional[str]) -> 'DataFrameWriter': - """Specifies the behavior when data or table already exists. - - Options include: - - * `append`: Append contents of this :class:`DataFrame` to existing data. - * `overwrite`: Overwrite existing data. - * `error` or `errorifexists`: Throw an exception if data already exists. - * `ignore`: Silently ignore this operation if data already exists. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Examples - -------- - Raise an error when writing to an existing path. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="mode1") as d: - ... spark.createDataFrame( - ... [{"age": 80, "name": "Xinrong Meng"}] - ... ).write.mode("error").format("parquet").save(d) # doctest: +SKIP - Traceback (most recent call last): - ... - ...AnalysisException: ... - - Write a Parquet file back with various options, and read it back. - - >>> with tempfile.TemporaryDirectory(prefix="mode2") as d: - ... # Overwrite the path with a new Parquet file - ... spark.createDataFrame( - ... [{"age": 100, "name": "Hyukjin Kwon"}] - ... ).write.mode("overwrite").format("parquet").save(d) - ... - ... # Append another DataFrame into the Parquet file - ... spark.createDataFrame( - ... [{"age": 120, "name": "Takuya Ueshin"}] - ... ).write.mode("append").format("parquet").save(d) - ... - ... # Append another DataFrame into the Parquet file - ... spark.createDataFrame( - ... [{"age": 140, "name": "Haejoon Lee"}] - ... ).write.mode("ignore").format("parquet").save(d) - ... - ... # Read the Parquet file as a DataFrame. - ... spark.read.parquet(d).show() - +---+-------------+ - |age| name| - +---+-------------+ - |120|Takuya Ueshin| - |100| Hyukjin Kwon| - +---+-------------+ - """ - if saveMode is not None: - self._jwrite = self._jwrite.mode(saveMode) - return self - def format(self, source: str) -> 'DataFrameWriter': - """Specifies the underlying output data source. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - source : str - string, name of the data source, e.g. 'json', 'parquet'. - - Examples - -------- - >>> spark.range(1).write.format('parquet') - <...readwriter.DataFrameWriter object ...> - - Write a DataFrame into a Parquet file and read it back. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="format") as d: - ... # Write a DataFrame into a Parquet file - ... spark.createDataFrame( - ... [{"age": 100, "name": "Hyukjin Kwon"}] - ... ).write.mode("overwrite").format("parquet").save(d) - ... - ... # Read the Parquet file as a DataFrame. - ... spark.read.format('parquet').load(d).show() - +---+------------+ - |age| name| - +---+------------+ - |100|Hyukjin Kwon| - +---+------------+ - """ - self._jwrite = self._jwrite.format(source) - return self - def option(self, key: str, value: 'OptionalPrimitiveType') -> 'DataFrameWriter': - """Adds an output option for the underlying data source. - - .. versionadded:: 1.5.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - key : str - The key for the option to set. - value - The value for the option to set. - - Examples - -------- - >>> spark.range(1).write.option("key", "value") - <...readwriter.DataFrameWriter object ...> - - Specify the option 'nullValue' with writing a CSV file. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="option") as d: - ... # Write a DataFrame into a CSV file with 'nullValue' option set to 'Hyukjin Kwon'. - ... df = spark.createDataFrame([(100, None)], "age INT, name STRING") - ... df.write.option("nullValue", "Hyukjin Kwon").mode("overwrite").format("csv").save(d) - ... - ... # Read the CSV file as a DataFrame. - ... spark.read.schema(df.schema).format('csv').load(d).show() - +---+------------+ - |age| name| - +---+------------+ - |100|Hyukjin Kwon| - +---+------------+ - """ - if value is None: - return self - self._jwrite = self._jwrite.option(key, to_str(value)) - return self - def options(self, **options: 'OptionalPrimitiveType') -> 'DataFrameWriter': - """Adds output options for the underlying data source. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - **options : dict - The dictionary of string keys and primitive-type values. - - Examples - -------- - >>> spark.range(1).write.options(key="value") - <...readwriter.DataFrameWriter object ...> - - Specify options in a dictionary. - - >>> spark.range(1).write.options(**{"k1": "v1", "k2": "v2"}) - <...readwriter.DataFrameWriter object ...> - - Specify the option 'nullValue' and 'header' with writing a CSV file. - - >>> from pyspark.sql.types import StructType,StructField, StringType, IntegerType - >>> schema = StructType([ - ... StructField("age",IntegerType(),True), - ... StructField("name",StringType(),True), - ... ]) - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="options") as d: - ... # Write a DataFrame into a CSV file with 'nullValue' option set to 'Hyukjin Kwon', - ... # and 'header' option set to `True`. - ... df = spark.createDataFrame([(100, None)], schema=schema) - ... df.write.options(nullValue="Hyukjin Kwon", header=True).mode( - ... "overwrite").format("csv").save(d) - ... - ... # Read the CSV file as a DataFrame. - ... spark.read.option("header", True).format('csv').load(d).show() - +---+------------+ - |age| name| - +---+------------+ - |100|Hyukjin Kwon| - +---+------------+ - """ - for k, v in options.items(): - if v is not None: - self._jwrite = self._jwrite.option(k, to_str(v)) - return self - @overload - def partitionBy(self, *cols: str) -> 'DataFrameWriter': - """Partitions the output by the given columns on the file system. - - If specified, the output is laid out on the file system similar - to Hive's partitioning scheme. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - cols : str or list - name of columns - - Examples - -------- - Write a DataFrame into a Parquet file in a partitioned manner, and read it back. - - >>> import tempfile - >>> import os - >>> with tempfile.TemporaryDirectory(prefix="partitionBy") as d: - ... # Write a DataFrame into a Parquet file in a partitioned manner. - ... spark.createDataFrame( - ... [{"age": 100, "name": "Hyukjin Kwon"}, {"age": 120, "name": "Ruifeng Zheng"}] - ... ).write.partitionBy("name").mode("overwrite").format("parquet").save(d) - ... - ... # Read the Parquet file as a DataFrame. - ... spark.read.parquet(d).sort("age").show() - ... - ... # Read one partition as a DataFrame. - ... spark.read.parquet(f"{d}{os.path.sep}name=Hyukjin Kwon").show() - +---+-------------+ - |age| name| - +---+-------------+ - |100| Hyukjin Kwon| - |120|Ruifeng Zheng| - +---+-------------+ - +---+ - |age| - +---+ - |100| - +---+ - """ - @overload - def partitionBy(self, *cols: List[str]) -> 'DataFrameWriter': - """Partitions the output by the given columns on the file system. - - If specified, the output is laid out on the file system similar - to Hive's partitioning scheme. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - cols : str or list - name of columns - - Examples - -------- - Write a DataFrame into a Parquet file in a partitioned manner, and read it back. - - >>> import tempfile - >>> import os - >>> with tempfile.TemporaryDirectory(prefix="partitionBy") as d: - ... # Write a DataFrame into a Parquet file in a partitioned manner. - ... spark.createDataFrame( - ... [{"age": 100, "name": "Hyukjin Kwon"}, {"age": 120, "name": "Ruifeng Zheng"}] - ... ).write.partitionBy("name").mode("overwrite").format("parquet").save(d) - ... - ... # Read the Parquet file as a DataFrame. - ... spark.read.parquet(d).sort("age").show() - ... - ... # Read one partition as a DataFrame. - ... spark.read.parquet(f"{d}{os.path.sep}name=Hyukjin Kwon").show() - +---+-------------+ - |age| name| - +---+-------------+ - |100| Hyukjin Kwon| - |120|Ruifeng Zheng| - +---+-------------+ - +---+ - |age| - +---+ - |100| - +---+ - """ - @overload - def bucketBy(self, numBuckets: int, col: str, *cols: str) -> 'DataFrameWriter': - """Buckets the output by the given columns. If specified, - the output is laid out on the file system similar to Hive's bucketing scheme, - but with a different bucket hash function and is not compatible with Hive's bucketing. - - .. versionadded:: 2.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - numBuckets : int - the number of buckets to save - col : str, list or tuple - a name of a column, or a list of names. - cols : str - additional names (optional). If `col` is a list it should be empty. - - Notes - ----- - Applicable for file-based data sources in combination with - :py:meth:`DataFrameWriter.saveAsTable`. - - Examples - -------- - Write a DataFrame into a Parquet file in a buckted manner, and read it back. - - >>> from pyspark.sql.functions import input_file_name - >>> # Write a DataFrame into a Parquet file in a bucketed manner. - ... _ = spark.sql("DROP TABLE IF EXISTS bucketed_table") - >>> spark.createDataFrame([ - ... (100, "Hyukjin Kwon"), (120, "Hyukjin Kwon"), (140, "Haejoon Lee")], - ... schema=["age", "name"] - ... ).write.bucketBy(2, "name").mode("overwrite").saveAsTable("bucketed_table") - >>> # Read the Parquet file as a DataFrame. - ... spark.read.table("bucketed_table").sort("age").show() - +---+------------+ - |age| name| - +---+------------+ - |100|Hyukjin Kwon| - |120|Hyukjin Kwon| - |140| Haejoon Lee| - +---+------------+ - >>> _ = spark.sql("DROP TABLE bucketed_table") - """ - @overload - def bucketBy(self, numBuckets: int, col: TupleOrListOfString) -> 'DataFrameWriter': - """Buckets the output by the given columns. If specified, - the output is laid out on the file system similar to Hive's bucketing scheme, - but with a different bucket hash function and is not compatible with Hive's bucketing. - - .. versionadded:: 2.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - numBuckets : int - the number of buckets to save - col : str, list or tuple - a name of a column, or a list of names. - cols : str - additional names (optional). If `col` is a list it should be empty. - - Notes - ----- - Applicable for file-based data sources in combination with - :py:meth:`DataFrameWriter.saveAsTable`. - - Examples - -------- - Write a DataFrame into a Parquet file in a buckted manner, and read it back. - - >>> from pyspark.sql.functions import input_file_name - >>> # Write a DataFrame into a Parquet file in a bucketed manner. - ... _ = spark.sql("DROP TABLE IF EXISTS bucketed_table") - >>> spark.createDataFrame([ - ... (100, "Hyukjin Kwon"), (120, "Hyukjin Kwon"), (140, "Haejoon Lee")], - ... schema=["age", "name"] - ... ).write.bucketBy(2, "name").mode("overwrite").saveAsTable("bucketed_table") - >>> # Read the Parquet file as a DataFrame. - ... spark.read.table("bucketed_table").sort("age").show() - +---+------------+ - |age| name| - +---+------------+ - |100|Hyukjin Kwon| - |120|Hyukjin Kwon| - |140| Haejoon Lee| - +---+------------+ - >>> _ = spark.sql("DROP TABLE bucketed_table") - """ - @overload - def sortBy(self, col: str, *cols: str) -> 'DataFrameWriter': - """Sorts the output in each bucket by the given columns on the file system. - - .. versionadded:: 2.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - col : str, tuple or list - a name of a column, or a list of names. - cols : str - additional names (optional). If `col` is a list it should be empty. - - Examples - -------- - Write a DataFrame into a Parquet file in a sorted-buckted manner, and read it back. - - >>> from pyspark.sql.functions import input_file_name - >>> # Write a DataFrame into a Parquet file in a sorted-bucketed manner. - ... _ = spark.sql("DROP TABLE IF EXISTS sorted_bucketed_table") - >>> spark.createDataFrame([ - ... (100, "Hyukjin Kwon"), (120, "Hyukjin Kwon"), (140, "Haejoon Lee")], - ... schema=["age", "name"] - ... ).write.bucketBy(1, "name").sortBy("age").mode( - ... "overwrite").saveAsTable("sorted_bucketed_table") - >>> # Read the Parquet file as a DataFrame. - ... spark.read.table("sorted_bucketed_table").sort("age").show() - +---+------------+ - |age| name| - +---+------------+ - |100|Hyukjin Kwon| - |120|Hyukjin Kwon| - |140| Haejoon Lee| - +---+------------+ - >>> _ = spark.sql("DROP TABLE sorted_bucketed_table") - """ - @overload - def sortBy(self, col: TupleOrListOfString) -> 'DataFrameWriter': - """Sorts the output in each bucket by the given columns on the file system. - - .. versionadded:: 2.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - col : str, tuple or list - a name of a column, or a list of names. - cols : str - additional names (optional). If `col` is a list it should be empty. - - Examples - -------- - Write a DataFrame into a Parquet file in a sorted-buckted manner, and read it back. - - >>> from pyspark.sql.functions import input_file_name - >>> # Write a DataFrame into a Parquet file in a sorted-bucketed manner. - ... _ = spark.sql("DROP TABLE IF EXISTS sorted_bucketed_table") - >>> spark.createDataFrame([ - ... (100, "Hyukjin Kwon"), (120, "Hyukjin Kwon"), (140, "Haejoon Lee")], - ... schema=["age", "name"] - ... ).write.bucketBy(1, "name").sortBy("age").mode( - ... "overwrite").saveAsTable("sorted_bucketed_table") - >>> # Read the Parquet file as a DataFrame. - ... spark.read.table("sorted_bucketed_table").sort("age").show() - +---+------------+ - |age| name| - +---+------------+ - |100|Hyukjin Kwon| - |120|Hyukjin Kwon| - |140| Haejoon Lee| - +---+------------+ - >>> _ = spark.sql("DROP TABLE sorted_bucketed_table") - """ - @overload - def clusterBy(self, *cols: str) -> 'DataFrameWriter': - """Clusters the data by the given columns to optimize query performance. - - .. versionadded:: 4.0.0 - - Parameters - ---------- - cols : str or list - name of columns - - Examples - -------- - Write a DataFrame into a Parquet file with clustering. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="clusterBy") as d: - ... spark.createDataFrame( - ... [{"age": 100, "name": "Hyukjin Kwon"}, {"age": 120, "name": "Ruifeng Zheng"}] - ... ).write.clusterBy("name").mode("overwrite").format("parquet").save(d) - """ - @overload - def clusterBy(self, *cols: List[str]) -> 'DataFrameWriter': - """Clusters the data by the given columns to optimize query performance. - - .. versionadded:: 4.0.0 - - Parameters - ---------- - cols : str or list - name of columns - - Examples - -------- - Write a DataFrame into a Parquet file with clustering. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="clusterBy") as d: - ... spark.createDataFrame( - ... [{"age": 100, "name": "Hyukjin Kwon"}, {"age": 120, "name": "Ruifeng Zheng"}] - ... ).write.clusterBy("name").mode("overwrite").format("parquet").save(d) - """ - def save(self, path: Optional[str]=None, format: Optional[str]=None, mode: Optional[str]=None, partitionBy: Optional[Union[str, List[str]]]=None, **options: 'OptionalPrimitiveType') -> None: - """Saves the contents of the :class:`DataFrame` to a data source. - - The data source is specified by the ``format`` and a set of ``options``. - If ``format`` is not specified, the default data source configured by - ``spark.sql.sources.default`` will be used. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - path : str, optional - the path in a Hadoop supported file system - format : str, optional - the format used to save - mode : str, optional - specifies the behavior of the save operation when data already exists. - - * ``append``: Append contents of this :class:`DataFrame` to existing data. - * ``overwrite``: Overwrite existing data. - * ``ignore``: Silently ignore this operation if data already exists. - * ``error`` or ``errorifexists`` (default case): Throw an exception if data already exists. - partitionBy : list, optional - names of partitioning columns - **options : dict - all other string options - - Examples - -------- - Write a DataFrame into a JSON file and read it back. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="save") as d: - ... # Write a DataFrame into a JSON file - ... spark.createDataFrame( - ... [{"age": 100, "name": "Hyukjin Kwon"}] - ... ).write.mode("overwrite").format("json").save(d) - ... - ... # Read the JSON file as a DataFrame. - ... spark.read.format('json').load(d).show() - +---+------------+ - |age| name| - +---+------------+ - |100|Hyukjin Kwon| - +---+------------+ - """ - def insertInto(self, tableName: str, overwrite: Optional[bool]=None) -> None: - """Inserts the content of the :class:`DataFrame` to the specified table. - - It requires that the schema of the :class:`DataFrame` is the same as the - schema of the table. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - overwrite : bool, optional - If true, overwrites existing data. Disabled by default - - Notes - ----- - Unlike :meth:`DataFrameWriter.saveAsTable`, :meth:`DataFrameWriter.insertInto` ignores - the column names and just uses position-based resolution. - - Examples - -------- - >>> _ = spark.sql("DROP TABLE IF EXISTS tblA") - >>> df = spark.createDataFrame([ - ... (100, "Hyukjin Kwon"), (120, "Hyukjin Kwon"), (140, "Haejoon Lee")], - ... schema=["age", "name"] - ... ) - >>> df.write.saveAsTable("tblA") - - Insert the data into 'tblA' table but with different column names. - - >>> df.selectExpr("age AS col1", "name AS col2").write.insertInto("tblA") - >>> spark.read.table("tblA").sort("age").show() - +---+------------+ - |age| name| - +---+------------+ - |100|Hyukjin Kwon| - |100|Hyukjin Kwon| - |120|Hyukjin Kwon| - |120|Hyukjin Kwon| - |140| Haejoon Lee| - |140| Haejoon Lee| - +---+------------+ - >>> _ = spark.sql("DROP TABLE tblA") - """ - def saveAsTable(self, name: str, format: Optional[str]=None, mode: Optional[str]=None, partitionBy: Optional[Union[str, List[str]]]=None, **options: 'OptionalPrimitiveType') -> None: - """Saves the content of the :class:`DataFrame` as the specified table. - - In the case the table already exists, behavior of this function depends on the - save mode, specified by the `mode` function (default to throwing an exception). - When `mode` is `Overwrite`, the schema of the :class:`DataFrame` does not need to be - the same as that of the existing table. - - * `append`: Append contents of this :class:`DataFrame` to existing data. - * `overwrite`: Overwrite existing data. - * `error` or `errorifexists`: Throw an exception if data already exists. - * `ignore`: Silently ignore this operation if data already exists. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Notes - ----- - When `mode` is `Append`, if there is an existing table, we will use the format and - options of the existing table. The column order in the schema of the :class:`DataFrame` - doesn't need to be the same as that of the existing table. Unlike - :meth:`DataFrameWriter.insertInto`, :meth:`DataFrameWriter.saveAsTable` will use the - column names to find the correct column positions. - - Parameters - ---------- - name : str - the table name - format : str, optional - the format used to save - mode : str, optional - one of `append`, `overwrite`, `error`, `errorifexists`, `ignore` (default: error) - partitionBy : str or list - names of partitioning columns - **options : dict - all other string options - - Examples - -------- - Creates a table from a DataFrame, and read it back. - - >>> _ = spark.sql("DROP TABLE IF EXISTS tblA") - >>> spark.createDataFrame([ - ... (100, "Hyukjin Kwon"), (120, "Hyukjin Kwon"), (140, "Haejoon Lee")], - ... schema=["age", "name"] - ... ).write.saveAsTable("tblA") - >>> spark.read.table("tblA").sort("age").show() - +---+------------+ - |age| name| - +---+------------+ - |100|Hyukjin Kwon| - |120|Hyukjin Kwon| - |140| Haejoon Lee| - +---+------------+ - >>> _ = spark.sql("DROP TABLE tblA") - """ - def json(self, path: str, mode: Optional[str]=None, compression: Optional[str]=None, dateFormat: Optional[str]=None, timestampFormat: Optional[str]=None, lineSep: Optional[str]=None, encoding: Optional[str]=None, ignoreNullFields: Optional[Union[bool, str]]=None) -> None: - """Saves the content of the :class:`DataFrame` in JSON format - (`JSON Lines text format or newline-delimited JSON `_) at the - specified path. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - path : str - the path in any Hadoop supported file system - mode : str, optional - specifies the behavior of the save operation when data already exists. - - * ``append``: Append contents of this :class:`DataFrame` to existing data. - * ``overwrite``: Overwrite existing data. - * ``ignore``: Silently ignore this operation if data already exists. - * ``error`` or ``errorifexists`` (default case): Throw an exception if data already exists. - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - for the version you use. - - .. # noqa - - Examples - -------- - Write a DataFrame into a JSON file and read it back. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="json") as d: - ... # Write a DataFrame into a JSON file - ... spark.createDataFrame( - ... [{"age": 100, "name": "Hyukjin Kwon"}] - ... ).write.json(d, mode="overwrite") - ... - ... # Read the JSON file as a DataFrame. - ... spark.read.format("json").load(d).show() - +---+------------+ - |age| name| - +---+------------+ - |100|Hyukjin Kwon| - +---+------------+ - """ - def parquet(self, path: str, mode: Optional[str]=None, partitionBy: Optional[Union[str, List[str]]]=None, compression: Optional[str]=None) -> None: - """Saves the content of the :class:`DataFrame` in Parquet format at the specified path. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - path : str - the path in any Hadoop supported file system - mode : str, optional - specifies the behavior of the save operation when data already exists. - - * ``append``: Append contents of this :class:`DataFrame` to existing data. - * ``overwrite``: Overwrite existing data. - * ``ignore``: Silently ignore this operation if data already exists. - * ``error`` or ``errorifexists`` (default case): Throw an exception if data already exists. - partitionBy : str or list, optional - names of partitioning columns - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - for the version you use. - - .. # noqa - - Examples - -------- - Write a DataFrame into a Parquet file and read it back. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="parquet") as d: - ... # Write a DataFrame into a Parquet file - ... spark.createDataFrame( - ... [{"age": 100, "name": "Hyukjin Kwon"}] - ... ).write.parquet(d, mode="overwrite") - ... - ... # Read the Parquet file as a DataFrame. - ... spark.read.format("parquet").load(d).show() - +---+------------+ - |age| name| - +---+------------+ - |100|Hyukjin Kwon| - +---+------------+ - """ - def text(self, path: str, compression: Optional[str]=None, lineSep: Optional[str]=None) -> None: - """Saves the content of the DataFrame in a text file at the specified path. - The text files will be encoded as UTF-8. - - .. versionadded:: 1.6.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - path : str - the path in any Hadoop supported file system - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - for the version you use. - - .. # noqa - - Notes - ----- - The DataFrame must have only one column that is of string type. - Each row becomes a new line in the output file. - - Examples - -------- - Write a DataFrame into a text file and read it back. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="text") as d: - ... # Write a DataFrame into a text file - ... df = spark.createDataFrame([("a",), ("b",), ("c",)], schema=["alphabets"]) - ... df.write.mode("overwrite").text(d) - ... - ... # Read the text file as a DataFrame. - ... spark.read.schema(df.schema).format("text").load(d).sort("alphabets").show() - +---------+ - |alphabets| - +---------+ - | a| - | b| - | c| - +---------+ - """ - def csv(self, path: str, mode: Optional[str]=None, compression: Optional[str]=None, sep: Optional[str]=None, quote: Optional[str]=None, escape: Optional[str]=None, header: Optional[Union[bool, str]]=None, nullValue: Optional[str]=None, escapeQuotes: Optional[Union[bool, str]]=None, quoteAll: Optional[Union[bool, str]]=None, dateFormat: Optional[str]=None, timestampFormat: Optional[str]=None, ignoreLeadingWhiteSpace: Optional[Union[bool, str]]=None, ignoreTrailingWhiteSpace: Optional[Union[bool, str]]=None, charToEscapeQuoteEscaping: Optional[str]=None, encoding: Optional[str]=None, emptyValue: Optional[str]=None, lineSep: Optional[str]=None) -> None: - r"""Saves the content of the :class:`DataFrame` in CSV format at the specified path. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - path : str - the path in any Hadoop supported file system - mode : str, optional - specifies the behavior of the save operation when data already exists. - - * ``append``: Append contents of this :class:`DataFrame` to existing data. - * ``overwrite``: Overwrite existing data. - * ``ignore``: Silently ignore this operation if data already exists. - * ``error`` or ``errorifexists`` (default case): Throw an exception if data already \ - exists. - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - for the version you use. - - .. # noqa - - Examples - -------- - Write a DataFrame into a CSV file and read it back. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="csv") as d: - ... # Write a DataFrame into a CSV file - ... df = spark.createDataFrame([{"age": 100, "name": "Hyukjin Kwon"}]) - ... df.write.csv(d, mode="overwrite") - ... - ... # Read the CSV file as a DataFrame with 'nullValue' option set to 'Hyukjin Kwon'. - ... spark.read.schema(df.schema).format("csv").option( - ... "nullValue", "Hyukjin Kwon").load(d).show() - +---+----+ - |age|name| - +---+----+ - |100|NULL| - +---+----+ - """ - def xml(self, path: str, rowTag: Optional[str]=None, mode: Optional[str]=None, attributePrefix: Optional[str]=None, valueTag: Optional[str]=None, rootTag: Optional[str]=None, declaration: Optional[bool]=None, arrayElementName: Optional[str]=None, nullValue: Optional[str]=None, dateFormat: Optional[str]=None, timestampFormat: Optional[str]=None, compression: Optional[str]=None, encoding: Optional[str]=None, validateName: Optional[bool]=None) -> None: - r"""Saves the content of the :class:`DataFrame` in XML format at the specified path. - - .. versionadded:: 4.0.0 - - Parameters - ---------- - path : str - the path in any Hadoop supported file system - mode : str, optional - specifies the behavior of the save operation when data already exists. - - * ``append``: Append contents of this :class:`DataFrame` to existing data. - * ``overwrite``: Overwrite existing data. - * ``ignore``: Silently ignore this operation if data already exists. - * ``error`` or ``errorifexists`` (default case): Throw an exception if data already \ - exists. - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - for the version you use. - - .. # noqa - - Examples - -------- - Write a DataFrame into a XML file and read it back. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="xml") as d: - ... # Write a DataFrame into a XML file - ... spark.createDataFrame( - ... [{"age": 100, "name": "Hyukjin Kwon"}] - ... ).write.mode("overwrite").option("rowTag", "person").xml(d) - ... - ... # Read the XML file as a DataFrame. - ... spark.read.option("rowTag", "person").format("xml").load(d).show() - +---+------------+ - |age| name| - +---+------------+ - |100|Hyukjin Kwon| - +---+------------+ - """ - def orc(self, path: str, mode: Optional[str]=None, partitionBy: Optional[Union[str, List[str]]]=None, compression: Optional[str]=None) -> None: - """Saves the content of the :class:`DataFrame` in ORC format at the specified path. - - .. versionadded:: 1.5.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - path : str - the path in any Hadoop supported file system - mode : str, optional - specifies the behavior of the save operation when data already exists. - - * ``append``: Append contents of this :class:`DataFrame` to existing data. - * ``overwrite``: Overwrite existing data. - * ``ignore``: Silently ignore this operation if data already exists. - * ``error`` or ``errorifexists`` (default case): Throw an exception if data already exists. - partitionBy : str or list, optional - names of partitioning columns - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - for the version you use. - - .. # noqa - - Examples - -------- - Write a DataFrame into a ORC file and read it back. - - >>> import tempfile - >>> with tempfile.TemporaryDirectory(prefix="orc") as d: - ... # Write a DataFrame into a ORC file - ... spark.createDataFrame( - ... [{"age": 100, "name": "Hyukjin Kwon"}] - ... ).write.orc(d, mode="overwrite") - ... - ... # Read the Parquet file as a DataFrame. - ... spark.read.format("orc").load(d).show() - +---+------------+ - |age| name| - +---+------------+ - |100|Hyukjin Kwon| - +---+------------+ - """ - def jdbc(self, url: str, table: str, mode: Optional[str]=None, properties: Optional[Dict[str, str]]=None) -> None: - """Saves the content of the :class:`DataFrame` to an external database table via JDBC. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - table : str - Name of the table in the external database. - mode : str, optional - specifies the behavior of the save operation when data already exists. - - * ``append``: Append contents of this :class:`DataFrame` to existing data. - * ``overwrite``: Overwrite existing data. - * ``ignore``: Silently ignore this operation if data already exists. - * ``error`` or ``errorifexists`` (default case): Throw an exception if data already exists. - properties : dict - a dictionary of JDBC database connection arguments. Normally at - least properties "user" and "password" with their corresponding values. - For example { 'user' : 'SYSTEM', 'password' : 'mypassword' } - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - for the version you use. - - .. # noqa - - Notes - ----- - Don't create too many partitions in parallel on a large cluster; - otherwise Spark might crash your external database systems. - """ - -class DataFrameWriterV2: - """Interface used to write a class:`pyspark.sql.dataframe.DataFrame` - to external storage using the v2 API. - - .. versionadded:: 3.1.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - """ - def __init__(self, df: 'DataFrame', table: str): - self._df = df - self._spark = df.sparkSession - self._jwriter = df._jdf.writeTo(table) - def using(self, provider: str) -> 'DataFrameWriterV2': - """Specifies a provider for the underlying output data source. - Spark's default catalog supports "parquet", "json", etc. - - .. versionadded:: 3.1.0 - """ - def option(self, key: str, value: 'OptionalPrimitiveType') -> 'DataFrameWriterV2': - """Add a write option. - - .. versionadded:: 3.1.0 - """ - def options(self, **options: 'OptionalPrimitiveType') -> 'DataFrameWriterV2': - """Add write options. - - .. versionadded:: 3.1.0 - """ - def tableProperty(self, property: str, value: str) -> 'DataFrameWriterV2': - """Add table property. - - .. versionadded:: 3.1.0 - """ - def partitionedBy(self, col: 'ColumnOrName', *cols: 'ColumnOrName') -> 'DataFrameWriterV2': - """Partition the output table created by `create`, `createOrReplace`, or `replace` using - the given columns or transforms. - - When specified, the table data will be stored by these values for efficient reads. - - For example, when a table is partitioned by day, it may be stored - in a directory layout like: - - * `table/day=2019-06-01/` - * `table/day=2019-06-02/` - - Partitioning is one of the most widely used techniques to optimize physical data layout. - It provides a coarse-grained index for skipping unnecessary data reads when queries have - predicates on the partitioned columns. In order for partitioning to work well, the number - of distinct values in each column should typically be less than tens of thousands. - - `col` and `cols` support only the following functions: - - * :py:func:`pyspark.sql.functions.years` - * :py:func:`pyspark.sql.functions.months` - * :py:func:`pyspark.sql.functions.days` - * :py:func:`pyspark.sql.functions.hours` - * :py:func:`pyspark.sql.functions.bucket` - - .. versionadded:: 3.1.0 - """ - def clusterBy(self, col: str, *cols: str) -> 'DataFrameWriterV2': - """Clusters the data by the given columns to optimize query performance.""" - def create(self) -> None: - """Create a new table from the contents of the data frame. - - The new table's schema, partition layout, properties, and other configuration will be - based on the configuration set on this writer. - - .. versionadded:: 3.1.0 - """ - def replace(self) -> None: - """Replace an existing table with the contents of the data frame. - - The existing table's schema, partition layout, properties, and other configuration will be - replaced with the contents of the data frame and the configuration set on this writer. - - .. versionadded:: 3.1.0 - """ - def createOrReplace(self) -> None: - """Create a new table or replace an existing table with the contents of the data frame. - - The output table's schema, partition layout, properties, - and other configuration will be based on the contents of the data frame - and the configuration set on this writer. - If the table exists, its configuration and data will be replaced. - - .. versionadded:: 3.1.0 - """ - def append(self) -> None: - """Append the contents of the data frame to the output table. - - .. versionadded:: 3.1.0 - """ - def overwrite(self, condition: 'ColumnOrName') -> None: - """Overwrite rows matching the given filter condition with the contents of the data frame in - the output table. - - .. versionadded:: 3.1.0 - """ - def overwritePartitions(self) -> None: - """Overwrite all partition for which the data frame contains at least one row with the contents - of the data frame in the output table. - - This operation is equivalent to Hive's `INSERT OVERWRITE ... PARTITION`, which replaces - partitions dynamically depending on the contents of the data frame. - - .. versionadded:: 3.1.0 - """ diff --git a/python/pyspark/sql/streaming/query.py b/python/pyspark/sql/streaming/query.py index 5707dea3..03d04cbb 100644 --- a/python/pyspark/sql/streaming/query.py +++ b/python/pyspark/sql/streaming/query.py @@ -15,106 +15,714 @@ # limitations under the License. # -""" -Streaming query management. - -This module re-exports the PyO3-implemented streaming query classes that provide -the client-side API for managing and inspecting streaming queries, and extends them -with Python-side listener bus implementation. -""" - import json -import warnings -from threading import Lock, Thread -from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union - -from pyspark import cloudpickle -from pyspark._pyspark import ( - StreamingQuery as _StreamingQuery, -) -from pyspark._pyspark import ( - StreamingQueryManager as _StreamingQueryManager, -) +from typing import Any, Dict, List, Optional, TYPE_CHECKING -# As PySpark's, the exception of a failed query is that of pyspark.errors. -from pyspark.errors import ( - PySparkValueError, - StreamingQueryException, # noqa: F401 +from pyspark.errors import StreamingQueryException, PySparkValueError +from pyspark.errors.exceptions.captured import ( + StreamingQueryException as CapturedStreamingQueryException, ) from pyspark.sql.streaming.listener import ( - QueryIdleEvent, - QueryProgressEvent, - QueryStartedEvent, - QueryTerminatedEvent, StreamingQueryListener, + StreamingQueryProgress, ) if TYPE_CHECKING: from py4j.java_gateway import JavaObject - from pyspark.sql.session import SparkSession +# TODO: Add StreamingCheckpointManager to __all__ once we add streaming checkpoint manager to the +# public API +__all__ = ["StreamingQuery", "StreamingQueryManager"] -# StreamingQueryEventType values (spark.connect.StreamingQueryEventType); the Rust -# listener stream yields these as the event_type int alongside each event's JSON. The started -# event of a query comes with the result of its start, which the Rust core posts as -# spark_connect::streaming::QUERY_STARTED_EVENT, of no StreamingQueryEventType. -_QUERY_STARTED_EVENT = -1 -_QUERY_PROGRESS_EVENT = 1 -_QUERY_TERMINATED_EVENT = 2 -_QUERY_IDLE_EVENT = 3 -# Re-export the core classes from the Rust extension -StreamingQuery = _StreamingQuery +class StreamingQuery: + """ + A handle to a query that is executing continuously in the background as new data arrives. + All these methods are thread-safe. + .. versionadded:: 2.0.0 -def _deserialize_listener_event(event_type, event_json, *, started=False): - """Decode a protocol event before entering callback exception isolation. + .. versionchanged:: 3.5.0 + Supports Spark Connect. - Started events belong to the synchronous start response, not the asynchronous - protocol enum. Malformed JSON and missing fields propagate to the caller, as in - reference PySpark's writer and listener-bus deserializers. + Notes + ----- + This API is evolving. """ - if started and event_type == _QUERY_STARTED_EVENT: - return QueryStartedEvent.fromJson(json.loads(event_json)) - if event_type == _QUERY_PROGRESS_EVENT: - return QueryProgressEvent.fromJson(json.loads(event_json)) - if event_type == _QUERY_TERMINATED_EVENT: - return QueryTerminatedEvent.fromJson(json.loads(event_json)) - if event_type == _QUERY_IDLE_EVENT: - return QueryIdleEvent.fromJson(json.loads(event_json)) - raise PySparkValueError( - errorClass="UNKNOWN_VALUE_FOR", - messageParameters={"var": f"proto.StreamingQueryEventType: {event_type}"}, - ) + def __init__(self, jsq: "JavaObject") -> None: + self._jsq = jsq -def _dispatch_listener_event(listener, event): - """Warn on callback Exception; BaseException and warning-as-error escape to the bus.""" - try: - if isinstance(event, QueryStartedEvent): - listener.onQueryStarted(event) - elif isinstance(event, QueryProgressEvent): - listener.onQueryProgress(event) - elif isinstance(event, QueryIdleEvent): - listener.onQueryIdle(event) - elif isinstance(event, QueryTerminatedEvent): - listener.onQueryTerminated(event) + @property + def id(self) -> str: + """ + Returns the unique id of this query that persists across restarts from checkpoint data. + That is, this id is generated when a query is started for the first time, and + will be the same every time it is restarted from checkpoint data. + There can only be one query with the same id active in a Spark cluster. + Also see, `runId`. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Returns + ------- + str + The unique id of query that persists across restarts from checkpoint data. + + Examples + -------- + >>> sdf = spark.readStream.format("rate").load() + >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() + + Get the unique id of this query that persists across restarts from checkpoint data + + >>> sq.id + '...' + + >>> sq.stop() + """ + return self._jsq.id().toString() + + @property + def runId(self) -> str: + """ + Returns the unique id of this query that does not persist across restarts. That is, every + query that is started (or restarted from checkpoint) will have a different runId. + + .. versionadded:: 2.1.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Returns + ------- + str + The unique id of query that does not persist across restarts. + + Examples + -------- + >>> sdf = spark.readStream.format("rate").load() + >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() + + Get the unique id of this query that does not persist across restarts + + >>> sq.runId + '...' + + >>> sq.stop() + """ + return self._jsq.runId().toString() + + @property + def name(self) -> str: + """ + Returns the user-specified name of the query, or None if not specified. + This name can be specified in the `org.apache.spark.sql.streaming.DataStreamWriter` + as `dataframe.writeStream.queryName("query").start()`. + This name, if set, must be unique across all active queries. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Returns + ------- + str + The user-specified name of the query, or None if not specified. + + Examples + -------- + >>> sdf = spark.readStream.format("rate").load() + >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() + + Get the user-specified name of the query, or None if not specified. + + >>> sq.name + 'this_query' + + >>> sq.stop() + """ + return self._jsq.name() + + @property + def isActive(self) -> bool: + """ + Whether this streaming query is currently active or not. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Returns + ------- + bool + The result whether specified streaming query is currently active or not. + + Examples + -------- + >>> sdf = spark.readStream.format("rate").load() + >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() + >>> sq.isActive + True + + >>> sq.stop() + """ + return self._jsq.isActive() + + def awaitTermination(self, timeout: Optional[int] = None) -> Optional[bool]: + """ + Waits for the termination of `this` query, either by :func:`query.stop()` or by an + exception. If the query has terminated with an exception, then the exception will be thrown. + If `timeout` is set, it returns whether the query has terminated or not within the + `timeout` seconds. + + If the query has terminated, then all subsequent calls to this method will either return + immediately (if the query was terminated by :func:`stop()`), or throw the exception + immediately (if the query has terminated with exception). + + throws :class:`StreamingQueryException`, if `this` query has terminated with an exception + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + timeout : int, optional + default ``None``. The waiting time for specified streaming query to terminate. + + Returns + ------- + bool, optional + The result whether specified streaming query has terminated or not within the `timeout` + seconds if `timeout` is set. The :class:`StreamingQueryException` will be thrown if the + query has terminated with an exception. + + Examples + -------- + >>> sdf = spark.readStream.format("rate").load() + >>> sq = sdf.writeStream.format('memory').queryName('query_awaitTermination').start() + + Return whether the query has terminated or not within 5 seconds + + >>> sq.awaitTermination(5) + False + + >>> sq.stop() + """ + if timeout is not None: + if not isinstance(timeout, (int, float)) or timeout <= 0: + raise PySparkValueError( + errorClass="VALUE_NOT_POSITIVE", + messageParameters={"arg_name": "timeout", "arg_value": type(timeout).__name__}, + ) + return self._jsq.awaitTermination(int(timeout * 1000)) + else: + return self._jsq.awaitTermination() + + @property + def status(self) -> Dict[str, Any]: + """ + Returns the current status of the query. + + .. versionadded:: 2.1.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Returns + ------- + dict + The current status of the specified query. + + Examples + -------- + >>> sdf = spark.readStream.format("rate").load() + >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() + + Get the current status of the query + + >>> sq.status + {'message': '...', 'isDataAvailable': ..., 'isTriggerActive': ...} + + >>> sq.stop() + """ + return json.loads(self._jsq.status().json()) + + @property + def recentProgress(self) -> List[StreamingQueryProgress]: + """ + Returns an array of the most recent [[StreamingQueryProgress]] updates for this query. + The number of progress updates retained for each stream is configured by Spark session + configuration `spark.sql.streaming.numRecentProgressUpdates`. + + .. versionadded:: 2.1.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Returns + ------- + list + List of dict which is the most recent :class:`StreamingQueryProgress` updates + for this query. + + Examples + -------- + >>> sdf = spark.readStream.format("rate").load() + >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() + + Get an array of the most recent query progress updates for this query + + >>> sq.recentProgress + [...] + + >>> sq.stop() + """ + return [ + StreamingQueryProgress.fromJson(json.loads(p.json())) + for p in self._jsq.recentProgress() + ] + + @property + def lastProgress(self) -> Optional[StreamingQueryProgress]: + """ + Returns the most recent :class:`StreamingQueryProgress` update of this streaming query or + None if there were no progress updates + + .. versionadded:: 2.1.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Returns + ------- + dict, optional + The most recent :class:`StreamingQueryProgress` update of this streaming query or + None if there were no progress updates. + + Examples + -------- + >>> sdf = spark.readStream.format("rate").load() + >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() + + Get the most recent query progress updates for this query + + >>> sq.lastProgress + >>> sq.stop() + """ + lastProgress = self._jsq.lastProgress() + if lastProgress: + return StreamingQueryProgress.fromJson(json.loads(lastProgress.json())) + else: + return None + + def processAllAvailable(self) -> None: + """ + Blocks until all available data in the source has been processed and committed to the + sink. This method is intended for testing. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Notes + ----- + In the case of continually arriving data, this method may block forever. + Additionally, this method is only guaranteed to block until data that has been + synchronously appended data to a stream source prior to invocation. + (i.e. `getOffset` must immediately reflect the addition). + + Examples + -------- + >>> sdf = spark.readStream.format("rate").load() + >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() + + Blocks query until all available data in the source + has been processed and committed to the sink + + >>> sq.processAllAvailable + + + >>> sq.stop() + """ + return self._jsq.processAllAvailable() + + def stop(self) -> None: + """ + Stop this streaming query. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Examples + -------- + >>> sdf = spark.readStream.format("rate").load() + >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() + >>> sq.isActive + True + + Stop streaming query + + >>> sq.stop() + + >>> sq.isActive + False + """ + self._jsq.stop() + + def explain(self, extended: bool = False) -> None: + """ + Prints the (logical and physical) plans to the console for debugging purpose. + + .. versionadded:: 2.1.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + extended : bool, optional + default ``False``. If ``False``, prints only the physical plan. + + Examples + -------- + >>> sdf = spark.readStream.format("rate").load() + >>> sdf.printSchema() + root + |-- timestamp: timestamp (nullable = true) + |-- value: long (nullable = true) + + >>> sq = sdf.writeStream.format('memory').queryName('query_explain').start() + >>> sq.processAllAvailable() # Wait a bit to generate the runtime plans. + + Explain the runtime plans + + >>> sq.explain() + == Physical Plan == + ... + >>> sq.explain(True) + == Parsed Logical Plan == + ... + == Analyzed Logical Plan == + ... + == Optimized Logical Plan == + ... + == Physical Plan == + ... + >>> sq.stop() + """ + # Cannot call `_jsq.explain(...)` because it will print in the JVM process. + # We should print it in the Python process. + print(self._jsq.explainInternal(extended)) + + def exception(self) -> Optional[StreamingQueryException]: + """ + .. versionadded:: 2.1.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Returns + ------- + :class:`StreamingQueryException` + the StreamingQueryException if the query was terminated by an exception, or None. + """ + if self._jsq.exception().isDefined(): + je = self._jsq.exception().get() + msg = je.toString().split(": ", 1)[1] # Drop the Java StreamingQueryException type info + stackTrace = "\n\t at ".join(map(lambda x: x.toString(), je.getStackTrace())) + return CapturedStreamingQueryException(msg, stackTrace, je.getCause()) + else: + return None + + +class StreamingQueryManager: + """A class to manage all the :class:`StreamingQuery` StreamingQueries active. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Notes + ----- + This API is evolving. + """ + + def __init__(self, jsqm: "JavaObject") -> None: + self._jsqm = jsqm + + @property + def active(self) -> List[StreamingQuery]: + """ + Returns a list of active queries associated with this SparkSession + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Returns + ------- + list + The active queries associated with this :class:`SparkSession`. + + Examples + -------- + >>> sdf = spark.readStream.format("rate").load() + >>> sdf.printSchema() + root + |-- timestamp: timestamp (nullable = true) + |-- value: long (nullable = true) + + >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() + >>> sqm = spark.streams + + Get the list of active streaming queries + + >>> [q.name for q in sqm.active] + ['this_query'] + >>> sq.stop() + """ + return [StreamingQuery(jsq) for jsq in self._jsqm.active()] + + def get(self, id: str) -> Optional["StreamingQuery"]: + """ + Returns an active query from this :class:`SparkSession`. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + id : str + The unique id of specified query. + + Returns + ------- + :class:`StreamingQuery` + An active query with `id` from this SparkSession. + + Notes + ----- + Exception will be thrown if an active query with this id does not exist. + + Examples + -------- + >>> sdf = spark.readStream.format("rate").load() + >>> sdf.printSchema() + root + |-- timestamp: timestamp (nullable = true) + |-- value: long (nullable = true) + + >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() + >>> sq.name + 'this_query' + + Get an active query by id + + >>> sq = spark.streams.get(sq.id) + >>> sq.isActive + True + >>> sq.stop() + """ + query = self._jsqm.get(id) + if query is not None: + return StreamingQuery(query) + else: + return None + + def awaitAnyTermination(self, timeout: Optional[int] = None) -> Optional[bool]: + """ + Wait until any of the queries on the associated SparkSession has terminated since the + creation of the context, or since :func:`resetTerminated()` was called. If any query was + terminated with an exception, then the exception will be thrown. + If `timeout` is set, it returns whether the query has terminated or not within the + `timeout` seconds. + + If a query has terminated, then subsequent calls to :func:`awaitAnyTermination()` will + either return immediately (if the query was terminated by :func:`query.stop()`), + or throw the exception immediately (if the query was terminated with exception). Use + :func:`resetTerminated()` to clear past terminations and wait for new terminations. + + In the case where multiple queries have terminated since :func:`resetTermination()` + was called, if any query has terminated with exception, then :func:`awaitAnyTermination()` + will throw any of the exception. For correctly documenting exceptions across multiple + queries, users need to stop all of them after any of them terminates with exception, and + then check the `query.exception()` for each query. + + throws :class:`StreamingQueryException`, if `this` query has terminated with an exception + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + timeout : int, optional + default ``None``. The waiting time for any streaming query to terminate. + + Returns + ------- + bool, optional + The result whether any streaming query has terminated or not within the `timeout` + seconds if `timeout` is set. The :class:`StreamingQueryException` will be thrown if any + query has terminated with an exception. + + Examples + -------- + >>> sdf = spark.readStream.format("rate").load() + >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() + + Return whether any of the query on the associated SparkSession + has terminated or not within 5 seconds + + >>> spark.streams.awaitAnyTermination(5) + True + >>> sq.stop() + """ + if timeout is not None: + if not isinstance(timeout, (int, float)) or timeout < 0: + raise PySparkValueError( + errorClass="VALUE_NOT_POSITIVE", + messageParameters={"arg_name": "timeout", "arg_value": type(timeout).__name__}, + ) + return self._jsqm.awaitAnyTermination(int(timeout * 1000)) else: - warnings.warn(f"Unknown StreamingQueryListener event: {event}") - except Exception as e: # noqa: BLE001 - warnings.warn(f"Listener {str(listener)} threw an exception\n{e}") + return self._jsqm.awaitAnyTermination() + + def resetTerminated(self) -> None: + """ + Forget about past terminated queries so that :func:`awaitAnyTermination()` can be used + again to wait for new terminations. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Examples + -------- + >>> spark.streams.resetTerminated() + """ + self._jsqm.resetTerminated() + + def addListener(self, listener: StreamingQueryListener) -> None: + """ + Register a :class:`StreamingQueryListener` to receive up-calls for life cycle events of + :class:`~pyspark.sql.streaming.StreamingQuery`. + + .. versionadded:: 3.4.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + listener : :class:`StreamingQueryListener` + A :class:`StreamingQueryListener` to receive up-calls for life cycle events of + :class:`~pyspark.sql.streaming.StreamingQuery`. + + Notes + ----- + This function behaves differently in Spark Connect mode. + In Connect, the provided functions doesn't have access to variables defined outside of it. + Also in Connect, you need to use `self.spark` to access spark session. + Using `spark` would throw an exception. + In short, if you want to use spark session inside the listener, + please use `self.spark` in Connect mode, and use `spark` otherwise. + Examples + -------- + >>> from pyspark.sql.streaming import StreamingQueryListener + >>> class TestListener(StreamingQueryListener): + ... def onQueryStarted(self, event): + ... pass + ... + ... def onQueryProgress(self, event): + ... pass + ... + ... def onQueryIdle(self, event): + ... pass + ... + ... def onQueryTerminated(self, event): + ... pass + ... + >>> test_listener = TestListener() -# ``spark.streams`` returns the native (Rust) StreamingQueryManager, which owns the -# client-side listener bus: addListener / removeListener / close are implemented in the -# Rust core (so Rust clients get the listener feature too). Re-export it as the public -# class. -StreamingQueryManager = _StreamingQueryManager + Register streaming query listener + + >>> spark.streams.addListener(test_listener) + + Deregister streaming query listener + + >>> spark.streams.removeListener(test_listener) + """ + from py4j.java_gateway import java_import + from pyspark import SparkContext + from pyspark.java_gateway import ensure_callback_server_started + + gw = SparkContext._gateway + assert gw is not None + java_import(gw.jvm, "org.apache.spark.sql.streaming.*") + ensure_callback_server_started(gw) + + self._jsqm.addListener(listener._jlistener) + + def removeListener(self, listener: StreamingQueryListener) -> None: + """ + Deregister a :class:`StreamingQueryListener`. + + .. versionadded:: 3.4.0 + + Parameters + ---------- + listener : :class:`StreamingQueryListener` + A :class:`StreamingQueryListener` to receive up-calls for life cycle events of + :class:`~pyspark.sql.streaming.StreamingQuery`. + + Examples + -------- + >>> from pyspark.sql.streaming import StreamingQueryListener + >>> class TestListener(StreamingQueryListener): + ... def onQueryStarted(self, event): + ... pass + ... + ... def onQueryProgress(self, event): + ... pass + ... + ... def onQueryIdle(self, event): + ... pass + ... + ... def onQueryTerminated(self, event): + ... pass + ... + >>> test_listener = TestListener() + + Register streaming query listener + + >>> spark.streams.addListener(test_listener) + + Deregister streaming query listener + + >>> spark.streams.removeListener(test_listener) + """ + self._jsqm.removeListener(listener._jlistener) -# PySpark's class, of its manager of the checkpoints of the JVM of Spark Classic, which the -# classic SparkSession._streamingCheckpointManager and SQLContext._streamingCheckpointManager -# annotate and return: neither is of a session of Spark Connect. class StreamingCheckpointManager: """ A class to manage operations on streaming query checkpoints. @@ -169,4 +777,36 @@ def repartition( self._jmanager.repartition(checkpoint_location, num_partitions, enforce_exactly_once_sink) -__all__ = ["StreamingQuery", "StreamingQueryManager"] +def _test() -> None: + import doctest + import os + import sys + from pyspark.core.context import SparkContext + from pyspark.sql import SparkSession + import pyspark.sql.streaming.query + from py4j.protocol import Py4JError + + os.chdir(os.environ["SPARK_HOME"]) + + globs = pyspark.sql.streaming.query.__dict__.copy() + sc = SparkContext("local[4]", "PythonTest") + try: + spark = SparkSession._getActiveSessionOrCreate() + except Py4JError: + spark = SparkSession(sc) + + globs["spark"] = spark + + failure_count, test_count = doctest.testmod( + pyspark.sql.streaming.query, + globs=globs, + optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF, + ) + globs["spark"].stop() + + if failure_count: + sys.exit(-1) + + +if __name__ == "__main__": + _test() diff --git a/python/pyspark/sql/streaming/query.pyi b/python/pyspark/sql/streaming/query.pyi deleted file mode 100644 index e8f96ced..00000000 --- a/python/pyspark/sql/streaming/query.pyi +++ /dev/null @@ -1,662 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Generated by scripts/gen_type_stubs.py from python/pyspark/sql/streaming/query.py of Apache Spark v4.3.0: -# do not edit. Type checkers read this stub instead of the module of the drop-in. - -from typing import Any, Dict, List, Optional -from pyspark.errors import StreamingQueryException as StreamingQueryException, PySparkValueError as PySparkValueError -from pyspark.sql.streaming.listener import StreamingQueryListener as StreamingQueryListener, StreamingQueryProgress -from py4j.java_gateway import JavaObject - -__all__ = ['StreamingQuery', 'StreamingQueryManager'] - -class StreamingQuery: - """A handle to a query that is executing continuously in the background as new data arrives. - All these methods are thread-safe. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Notes - ----- - This API is evolving. - """ - def __init__(self, jsq: 'JavaObject') -> None: - self._jsq = jsq - @property - def id(self) -> str: - """Returns the unique id of this query that persists across restarts from checkpoint data. - That is, this id is generated when a query is started for the first time, and - will be the same every time it is restarted from checkpoint data. - There can only be one query with the same id active in a Spark cluster. - Also see, `runId`. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Returns - ------- - str - The unique id of query that persists across restarts from checkpoint data. - - Examples - -------- - >>> sdf = spark.readStream.format("rate").load() - >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() - - Get the unique id of this query that persists across restarts from checkpoint data - - >>> sq.id - '...' - - >>> sq.stop() - """ - @property - def runId(self) -> str: - """Returns the unique id of this query that does not persist across restarts. That is, every - query that is started (or restarted from checkpoint) will have a different runId. - - .. versionadded:: 2.1.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Returns - ------- - str - The unique id of query that does not persist across restarts. - - Examples - -------- - >>> sdf = spark.readStream.format("rate").load() - >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() - - Get the unique id of this query that does not persist across restarts - - >>> sq.runId - '...' - - >>> sq.stop() - """ - @property - def name(self) -> str: - """Returns the user-specified name of the query, or None if not specified. - This name can be specified in the `org.apache.spark.sql.streaming.DataStreamWriter` - as `dataframe.writeStream.queryName("query").start()`. - This name, if set, must be unique across all active queries. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Returns - ------- - str - The user-specified name of the query, or None if not specified. - - Examples - -------- - >>> sdf = spark.readStream.format("rate").load() - >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() - - Get the user-specified name of the query, or None if not specified. - - >>> sq.name - 'this_query' - - >>> sq.stop() - """ - @property - def isActive(self) -> bool: - """Whether this streaming query is currently active or not. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Returns - ------- - bool - The result whether specified streaming query is currently active or not. - - Examples - -------- - >>> sdf = spark.readStream.format("rate").load() - >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() - >>> sq.isActive - True - - >>> sq.stop() - """ - def awaitTermination(self, timeout: Optional[int]=None) -> Optional[bool]: - """Waits for the termination of `this` query, either by :func:`query.stop()` or by an - exception. If the query has terminated with an exception, then the exception will be thrown. - If `timeout` is set, it returns whether the query has terminated or not within the - `timeout` seconds. - - If the query has terminated, then all subsequent calls to this method will either return - immediately (if the query was terminated by :func:`stop()`), or throw the exception - immediately (if the query has terminated with exception). - - throws :class:`StreamingQueryException`, if `this` query has terminated with an exception - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - timeout : int, optional - default ``None``. The waiting time for specified streaming query to terminate. - - Returns - ------- - bool, optional - The result whether specified streaming query has terminated or not within the `timeout` - seconds if `timeout` is set. The :class:`StreamingQueryException` will be thrown if the - query has terminated with an exception. - - Examples - -------- - >>> sdf = spark.readStream.format("rate").load() - >>> sq = sdf.writeStream.format('memory').queryName('query_awaitTermination').start() - - Return whether the query has terminated or not within 5 seconds - - >>> sq.awaitTermination(5) - False - - >>> sq.stop() - """ - @property - def status(self) -> Dict[str, Any]: - """Returns the current status of the query. - - .. versionadded:: 2.1.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Returns - ------- - dict - The current status of the specified query. - - Examples - -------- - >>> sdf = spark.readStream.format("rate").load() - >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() - - Get the current status of the query - - >>> sq.status - {'message': '...', 'isDataAvailable': ..., 'isTriggerActive': ...} - - >>> sq.stop() - """ - @property - def recentProgress(self) -> List[StreamingQueryProgress]: - """Returns an array of the most recent [[StreamingQueryProgress]] updates for this query. - The number of progress updates retained for each stream is configured by Spark session - configuration `spark.sql.streaming.numRecentProgressUpdates`. - - .. versionadded:: 2.1.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Returns - ------- - list - List of dict which is the most recent :class:`StreamingQueryProgress` updates - for this query. - - Examples - -------- - >>> sdf = spark.readStream.format("rate").load() - >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() - - Get an array of the most recent query progress updates for this query - - >>> sq.recentProgress - [...] - - >>> sq.stop() - """ - @property - def lastProgress(self) -> Optional[StreamingQueryProgress]: - """Returns the most recent :class:`StreamingQueryProgress` update of this streaming query or - None if there were no progress updates - - .. versionadded:: 2.1.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Returns - ------- - dict, optional - The most recent :class:`StreamingQueryProgress` update of this streaming query or - None if there were no progress updates. - - Examples - -------- - >>> sdf = spark.readStream.format("rate").load() - >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() - - Get the most recent query progress updates for this query - - >>> sq.lastProgress - >>> sq.stop() - """ - def processAllAvailable(self) -> None: - """Blocks until all available data in the source has been processed and committed to the - sink. This method is intended for testing. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Notes - ----- - In the case of continually arriving data, this method may block forever. - Additionally, this method is only guaranteed to block until data that has been - synchronously appended data to a stream source prior to invocation. - (i.e. `getOffset` must immediately reflect the addition). - - Examples - -------- - >>> sdf = spark.readStream.format("rate").load() - >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() - - Blocks query until all available data in the source - has been processed and committed to the sink - - >>> sq.processAllAvailable - - - >>> sq.stop() - """ - def stop(self) -> None: - """Stop this streaming query. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Examples - -------- - >>> sdf = spark.readStream.format("rate").load() - >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() - >>> sq.isActive - True - - Stop streaming query - - >>> sq.stop() - - >>> sq.isActive - False - """ - def explain(self, extended: bool=False) -> None: - """Prints the (logical and physical) plans to the console for debugging purpose. - - .. versionadded:: 2.1.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - extended : bool, optional - default ``False``. If ``False``, prints only the physical plan. - - Examples - -------- - >>> sdf = spark.readStream.format("rate").load() - >>> sdf.printSchema() - root - |-- timestamp: timestamp (nullable = true) - |-- value: long (nullable = true) - - >>> sq = sdf.writeStream.format('memory').queryName('query_explain').start() - >>> sq.processAllAvailable() # Wait a bit to generate the runtime plans. - - Explain the runtime plans - - >>> sq.explain() - == Physical Plan == - ... - >>> sq.explain(True) - == Parsed Logical Plan == - ... - == Analyzed Logical Plan == - ... - == Optimized Logical Plan == - ... - == Physical Plan == - ... - >>> sq.stop() - """ - def exception(self) -> Optional[StreamingQueryException]: - """.. versionadded:: 2.1.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Returns - ------- - :class:`StreamingQueryException` - the StreamingQueryException if the query was terminated by an exception, or None. - """ - -class StreamingQueryManager: - """A class to manage all the :class:`StreamingQuery` StreamingQueries active. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Notes - ----- - This API is evolving. - """ - def __init__(self, jsqm: 'JavaObject') -> None: - self._jsqm = jsqm - @property - def active(self) -> List[StreamingQuery]: - """Returns a list of active queries associated with this SparkSession - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Returns - ------- - list - The active queries associated with this :class:`SparkSession`. - - Examples - -------- - >>> sdf = spark.readStream.format("rate").load() - >>> sdf.printSchema() - root - |-- timestamp: timestamp (nullable = true) - |-- value: long (nullable = true) - - >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() - >>> sqm = spark.streams - - Get the list of active streaming queries - - >>> [q.name for q in sqm.active] - ['this_query'] - >>> sq.stop() - """ - def get(self, id: str) -> Optional['StreamingQuery']: - """Returns an active query from this :class:`SparkSession`. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - id : str - The unique id of specified query. - - Returns - ------- - :class:`StreamingQuery` - An active query with `id` from this SparkSession. - - Notes - ----- - Exception will be thrown if an active query with this id does not exist. - - Examples - -------- - >>> sdf = spark.readStream.format("rate").load() - >>> sdf.printSchema() - root - |-- timestamp: timestamp (nullable = true) - |-- value: long (nullable = true) - - >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() - >>> sq.name - 'this_query' - - Get an active query by id - - >>> sq = spark.streams.get(sq.id) - >>> sq.isActive - True - >>> sq.stop() - """ - def awaitAnyTermination(self, timeout: Optional[int]=None) -> Optional[bool]: - """Wait until any of the queries on the associated SparkSession has terminated since the - creation of the context, or since :func:`resetTerminated()` was called. If any query was - terminated with an exception, then the exception will be thrown. - If `timeout` is set, it returns whether the query has terminated or not within the - `timeout` seconds. - - If a query has terminated, then subsequent calls to :func:`awaitAnyTermination()` will - either return immediately (if the query was terminated by :func:`query.stop()`), - or throw the exception immediately (if the query was terminated with exception). Use - :func:`resetTerminated()` to clear past terminations and wait for new terminations. - - In the case where multiple queries have terminated since :func:`resetTermination()` - was called, if any query has terminated with exception, then :func:`awaitAnyTermination()` - will throw any of the exception. For correctly documenting exceptions across multiple - queries, users need to stop all of them after any of them terminates with exception, and - then check the `query.exception()` for each query. - - throws :class:`StreamingQueryException`, if `this` query has terminated with an exception - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - timeout : int, optional - default ``None``. The waiting time for any streaming query to terminate. - - Returns - ------- - bool, optional - The result whether any streaming query has terminated or not within the `timeout` - seconds if `timeout` is set. The :class:`StreamingQueryException` will be thrown if any - query has terminated with an exception. - - Examples - -------- - >>> sdf = spark.readStream.format("rate").load() - >>> sq = sdf.writeStream.format('memory').queryName('this_query').start() - - Return whether any of the query on the associated SparkSession - has terminated or not within 5 seconds - - >>> spark.streams.awaitAnyTermination(5) - True - >>> sq.stop() - """ - def resetTerminated(self) -> None: - """Forget about past terminated queries so that :func:`awaitAnyTermination()` can be used - again to wait for new terminations. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Examples - -------- - >>> spark.streams.resetTerminated() - """ - def addListener(self, listener: StreamingQueryListener) -> None: - """Register a :class:`StreamingQueryListener` to receive up-calls for life cycle events of - :class:`~pyspark.sql.streaming.StreamingQuery`. - - .. versionadded:: 3.4.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - listener : :class:`StreamingQueryListener` - A :class:`StreamingQueryListener` to receive up-calls for life cycle events of - :class:`~pyspark.sql.streaming.StreamingQuery`. - - Notes - ----- - This function behaves differently in Spark Connect mode. - In Connect, the provided functions doesn't have access to variables defined outside of it. - Also in Connect, you need to use `self.spark` to access spark session. - Using `spark` would throw an exception. - In short, if you want to use spark session inside the listener, - please use `self.spark` in Connect mode, and use `spark` otherwise. - - Examples - -------- - >>> from pyspark.sql.streaming import StreamingQueryListener - >>> class TestListener(StreamingQueryListener): - ... def onQueryStarted(self, event): - ... pass - ... - ... def onQueryProgress(self, event): - ... pass - ... - ... def onQueryIdle(self, event): - ... pass - ... - ... def onQueryTerminated(self, event): - ... pass - ... - >>> test_listener = TestListener() - - Register streaming query listener - - >>> spark.streams.addListener(test_listener) - - Deregister streaming query listener - - >>> spark.streams.removeListener(test_listener) - """ - def removeListener(self, listener: StreamingQueryListener) -> None: - """Deregister a :class:`StreamingQueryListener`. - - .. versionadded:: 3.4.0 - - Parameters - ---------- - listener : :class:`StreamingQueryListener` - A :class:`StreamingQueryListener` to receive up-calls for life cycle events of - :class:`~pyspark.sql.streaming.StreamingQuery`. - - Examples - -------- - >>> from pyspark.sql.streaming import StreamingQueryListener - >>> class TestListener(StreamingQueryListener): - ... def onQueryStarted(self, event): - ... pass - ... - ... def onQueryProgress(self, event): - ... pass - ... - ... def onQueryIdle(self, event): - ... pass - ... - ... def onQueryTerminated(self, event): - ... pass - ... - >>> test_listener = TestListener() - - Register streaming query listener - - >>> spark.streams.addListener(test_listener) - - Deregister streaming query listener - - >>> spark.streams.removeListener(test_listener) - """ - -class StreamingCheckpointManager: - """A class to manage operations on streaming query checkpoints. - - .. versionadded:: 4.2.0 - - Notes - ----- - This API is evolving and currently supported in Spark Classic. - """ - def __init__(self, jmanager: 'JavaObject') -> None: - self._jmanager = jmanager - def repartition(self, checkpoint_location: str, num_partitions: int, enforce_exactly_once_sink: bool=True) -> None: - """Repartition the stateful streaming operators state in the streaming checkpoint to have - `num_partitions` partitions. The streaming query MUST not be running. If `num_partitions` is - the same as the current number of partitions, this is a no-op, and an exception will be - thrown. - - This produces a new microbatch in the checkpoint that contains the repartitioned state i.e. - if the last streaming batch was batch `N`, this will create batch `N+1` with the - repartitioned state. Note that this new batch doesn't read input data from sources, it only - represents the repartition operation. The next time the streaming query is started, it will - pick up from this new batch. - - This will return only when the repartitioning is complete or fails. - - .. versionadded:: 4.2.0 - - Parameters - ---------- - checkpoint_location : str - The checkpoint location of the streaming query, should be the `checkpointLocation` option - on the DataStreamWriter. - num_partitions : int - The target number of state partitions. - enforce_exactly_once_sink : bool, optional - If we shouldn't allow skipping failed batches, to avoid duplicates in exactly once sinks. - default ``True``. - - Notes - ----- - This API is experimental. - - This operation should only be performed after the streaming query has been stopped. If not, - can lead to undefined behavior or checkpoint corruption. - """ - -# Names of the drop-in's module that PySpark's does not have. -from pyspark.sql.streaming.listener import QueryIdleEvent as QueryIdleEvent -from pyspark.sql.streaming.listener import QueryProgressEvent as QueryProgressEvent -from pyspark.sql.streaming.listener import QueryStartedEvent as QueryStartedEvent -from pyspark.sql.streaming.listener import QueryTerminatedEvent as QueryTerminatedEvent diff --git a/python/pyspark/sql/streaming/readwriter.py b/python/pyspark/sql/streaming/readwriter.py index ea008ec3..28ae1a9d 100644 --- a/python/pyspark/sql/streaming/readwriter.py +++ b/python/pyspark/sql/streaming/readwriter.py @@ -15,122 +15,1951 @@ # limitations under the License. # -""" -Streaming data readers and writers. - -Re-exports the PyO3-implemented streaming classes. `DataStreamWriter.foreachBatch` -and `.foreach` accept a Python callable and cloudpickle it inside the extension (with -``pyspark.serializers.CloudPickleSerializer``), so no Python subclass of the (non- -subclassable) PyO3 class is needed; `.foreach` sends the function of -`_construct_foreach_function`, as PySpark does. -""" - +import re +import sys from collections.abc import Iterator -from typing import Any, Callable, Union, cast +from typing import cast, overload, Any, Callable, List, Optional, TYPE_CHECKING, Union -from pyspark._pyspark import ( - DataStreamReader as _DataStreamReader, -) -from pyspark._pyspark import ( - DataStreamWriter as _DataStreamWriter, +from pyspark.sql.readwriter import OptionUtils, to_str +from pyspark.sql.streaming.query import StreamingQuery +from pyspark.sql.types import Row, StructType +from pyspark.sql.utils import ForeachBatchFunction +from pyspark.errors import ( + PySparkTypeError, + PySparkValueError, + PySparkAttributeError, + PySparkRuntimeError, ) -from pyspark._pyspark import ( - Trigger as _Trigger, -) -from pyspark.errors import PySparkAttributeError, PySparkRuntimeError -DataStreamReader = _DataStreamReader -DataStreamWriter = _DataStreamWriter -Trigger = _Trigger +if TYPE_CHECKING: + from py4j.java_gateway import JavaObject + from pyspark.sql.session import SparkSession + from pyspark.sql._typing import SupportsProcess, OptionalPrimitiveType + from pyspark.sql.dataframe import DataFrame + +__all__ = ["DataStreamReader", "DataStreamWriter"] -def _construct_foreach_function( - f: Union[Callable[[Any], None], Any], -) -> Callable[[Any, Iterator], Iterator]: +class DataStreamReader(OptionUtils): """ - The function of the partition and the rows of each partition that ``foreach(f)`` sends, - which the Python worker of the server runs, as PySpark's - ``DataStreamWriter._construct_foreach_function``: for a function ``f``, one that calls ``f`` - with each row, and for an object, one that calls its ``open(partition_id, epoch_id)``, its - ``process(row)`` with each row, and its ``close(error)``. + Interface used to load a streaming :class:`DataFrame ` from external + storage systems (e.g. file systems, key-value stores, etc). + Use :attr:`SparkSession.readStream ` to access this. + + .. versionadded:: 2.0.0 - Ported from PySpark, but for ``TaskContext``, which the function imports on the worker. + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Notes + ----- + This API is evolving. + + Examples + -------- + >>> spark.readStream + <...streaming.readwriter.DataStreamReader object ...> + + The example below uses Rate source that generates rows continuously. + After that, we operate a modulo by 3, and then writes the stream out to the console. + The streaming query stops in 3 seconds. + + >>> import time + >>> df = spark.readStream.format("rate").load() + >>> df = df.selectExpr("value % 3 as v") + >>> q = df.writeStream.format("console").start() + >>> time.sleep(3) + >>> q.stop() """ - if callable(f): - # The provided object is a callable function that is supposed to be called on each row. - # Construct a function that takes an iterator and calls the provided function on each - # row. - def func_without_process(_: Any, iterator: Iterator) -> Iterator: - for x in iterator: - f(x) - return iter([]) - - return func_without_process - - else: - # The provided object is not a callable function. Then it is expected to have a - # 'process(row)' method, and optional 'open(partition_id, epoch_id)' and - # 'close(error)' methods. - - if not hasattr(f, "process"): - raise PySparkAttributeError( - errorClass="ATTRIBUTE_NOT_CALLABLE", - messageParameters={"attr_name": "process", "obj_name": "f"}, + + def __init__(self, spark: "SparkSession") -> None: + self._jreader = spark._jsparkSession.readStream() + self._spark = spark + + def _df(self, jdf: "JavaObject") -> "DataFrame": + from pyspark.sql.dataframe import DataFrame + + return DataFrame(jdf, self._spark) + + def format(self, source: str) -> "DataStreamReader": + """Specifies the input data source format. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + source : str + name of the data source, e.g. 'json', 'parquet'. + + Notes + ----- + This API is evolving. + + Examples + -------- + >>> spark.readStream.format("text") + <...streaming.readwriter.DataStreamReader object ...> + + This API allows to configure other sources to read. The example below writes a small text + file, and reads it back via Text source. + + >>> import tempfile + >>> import time + >>> with tempfile.TemporaryDirectory(prefix="format") as d: + ... # Write a temporary text file to read it. + ... spark.createDataFrame( + ... [("hello",), ("this",)]).write.mode("overwrite").format("text").save(d) + ... + ... # Start a streaming query to read the text file. + ... q = spark.readStream.format("text").load(d).writeStream.format("console").start() + ... time.sleep(3) + ... q.stop() + """ + self._jreader = self._jreader.format(source) + return self + + def schema(self, schema: Union[StructType, str]) -> "DataStreamReader": + """Specifies the input schema. + + Some data sources (e.g. JSON) can infer the input schema automatically from data. + By specifying the schema here, the underlying data source can skip the schema + inference step, and thus speed up data loading. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + schema : :class:`pyspark.sql.types.StructType` or str + a :class:`pyspark.sql.types.StructType` object or a DDL-formatted string + (For example ``col0 INT, col1 DOUBLE``). + + Notes + ----- + This API is evolving. + + Examples + -------- + >>> from pyspark.sql.types import StructField, StructType, StringType + >>> spark.readStream.schema(StructType([StructField("data", StringType(), True)])) + <...streaming.readwriter.DataStreamReader object ...> + >>> spark.readStream.schema("col0 INT, col1 DOUBLE") + <...streaming.readwriter.DataStreamReader object ...> + + The example below specifies a different schema to CSV file. + + >>> import tempfile + >>> import time + >>> with tempfile.TemporaryDirectory(prefix="schema") as d: + ... # Start a streaming query to read the CSV file. + ... spark.readStream.schema("col0 INT, col1 STRING").format("csv").load(d).printSchema() + root + |-- col0: integer (nullable = true) + |-- col1: string (nullable = true) + """ + from pyspark.sql import SparkSession + + spark = SparkSession._getActiveSessionOrCreate() + if isinstance(schema, StructType): + jschema = spark._jsparkSession.parseDataType(schema.json()) + self._jreader = self._jreader.schema(jschema) + elif isinstance(schema, str): + self._jreader = self._jreader.schema(schema) + else: + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "schema", + "expected_type": "str or struct type", + "arg_type": type(schema).__name__, + }, + ) + return self + + def option(self, key: str, value: "OptionalPrimitiveType") -> "DataStreamReader": + """Adds an input option for the underlying data source. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Notes + ----- + This API is evolving. + + Examples + -------- + >>> spark.readStream.option("x", 1) + <...streaming.readwriter.DataStreamReader object ...> + + The example below specifies 'rowsPerSecond' option to Rate source in order to generate + 10 rows every second. + + >>> import time + >>> q = spark.readStream.format( + ... "rate").option("rowsPerSecond", 10).load().writeStream.format("console").start() + >>> time.sleep(3) + >>> q.stop() + """ + if value is None: + return self + self._jreader = self._jreader.option(key, to_str(value)) + return self + + def options(self, **options: "OptionalPrimitiveType") -> "DataStreamReader": + """Adds input options for the underlying data source. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Notes + ----- + This API is evolving. + + Examples + -------- + >>> spark.readStream.options(x="1", y=2) + <...streaming.readwriter.DataStreamReader object ...> + + Specify options in a dictionary. + + >>> spark.readStream.options(**{"k1": "v1", "k2": "v2"}) + <...streaming.readwriter.DataStreamReader object ...> + + The example below specifies 'rowsPerSecond' and 'numPartitions' options to + Rate source in order to generate 10 rows with 10 partitions every second. + + >>> import time + >>> q = spark.readStream.format("rate").options( + ... rowsPerSecond=10, numPartitions=10 + ... ).load().writeStream.format("console").start() + >>> time.sleep(3) + >>> q.stop() + """ + for k, v in options.items(): + if v is not None: + self._jreader = self._jreader.option(k, to_str(v)) + return self + + def name(self, source_name: str) -> "DataStreamReader": + """Specifies a name for the streaming source. + + This name is used to identify the source in checkpoint metadata and enables + stable checkpoint locations for source evolution. + + .. versionadded:: 4.2.0 + + Parameters + ---------- + source_name : str + the name to assign to this streaming source. Must contain only ASCII letters, + digits, and underscores. + + Returns + ------- + :class:`DataStreamReader` + + Notes + ----- + This API is experimental. + + Examples + -------- + >>> spark.readStream.format("rate").name("my_source") # doctest: +SKIP + <...streaming.readwriter.DataStreamReader object ...> + """ + if not isinstance(source_name, str): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "source_name", + "expected_type": "str", + "arg_type": type(source_name).__name__, + }, + ) + + # Validate that source_name contains only ASCII letters, digits, and underscores + if not re.match(r"^[a-zA-Z0-9_]+$", source_name): + raise PySparkValueError( + errorClass="INVALID_STREAMING_SOURCE_NAME", + messageParameters={"source_name": source_name}, ) - if not callable(f.process): - raise PySparkAttributeError( - errorClass="ATTRIBUTE_NOT_CALLABLE", - messageParameters={"attr_name": "process", "obj_name": "f"}, + self._jreader = self._jreader.name(source_name) + return self + + def load( + self, + path: Optional[str] = None, + format: Optional[str] = None, + schema: Optional[Union[StructType, str]] = None, + **options: "OptionalPrimitiveType", + ) -> "DataFrame": + """Loads a data stream from a data source and returns it as a + :class:`DataFrame `. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + path : str, optional + optional string for file-system backed data sources. + format : str, optional + optional string for format of the data source. Default to 'parquet'. + schema : :class:`pyspark.sql.types.StructType` or str, optional + optional :class:`pyspark.sql.types.StructType` for the input schema + or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). + **options : dict + all other string options + + Notes + ----- + This API is evolving. + + Examples + -------- + Load a data stream from a temporary JSON file. + + >>> import tempfile + >>> import time + >>> with tempfile.TemporaryDirectory(prefix="load") as d: + ... # Write a temporary JSON file to read it. + ... spark.createDataFrame( + ... [(100, "Hyukjin Kwon"),], ["age", "name"] + ... ).write.mode("overwrite").format("json").save(d) + ... + ... # Start a streaming query to read the JSON file. + ... q = spark.readStream.schema( + ... "age INT, name STRING" + ... ).format("json").load(d).writeStream.format("console").start() + ... time.sleep(3) + ... q.stop() + """ + if format is not None: + self.format(format) + if schema is not None: + self.schema(schema) + self.options(**options) + if path is not None: + if not isinstance(path, str) or len(path.strip()) == 0: + raise PySparkValueError( + errorClass="VALUE_NOT_NON_EMPTY_STR", + messageParameters={"arg_name": "path", "arg_value": str(path)}, + ) + return self._df(self._jreader.load(path)) + else: + return self._df(self._jreader.load()) + + def json( + self, + path: str, + schema: Optional[Union[StructType, str]] = None, + primitivesAsString: Optional[Union[bool, str]] = None, + prefersDecimal: Optional[Union[bool, str]] = None, + allowComments: Optional[Union[bool, str]] = None, + allowUnquotedFieldNames: Optional[Union[bool, str]] = None, + allowSingleQuotes: Optional[Union[bool, str]] = None, + allowNumericLeadingZero: Optional[Union[bool, str]] = None, + allowBackslashEscapingAnyCharacter: Optional[Union[bool, str]] = None, + mode: Optional[str] = None, + columnNameOfCorruptRecord: Optional[str] = None, + dateFormat: Optional[str] = None, + timestampFormat: Optional[str] = None, + multiLine: Optional[Union[bool, str]] = None, + allowUnquotedControlChars: Optional[Union[bool, str]] = None, + lineSep: Optional[str] = None, + locale: Optional[str] = None, + dropFieldIfAllNull: Optional[Union[bool, str]] = None, + encoding: Optional[str] = None, + pathGlobFilter: Optional[Union[bool, str]] = None, + recursiveFileLookup: Optional[Union[bool, str]] = None, + allowNonNumericNumbers: Optional[Union[bool, str]] = None, + useUnsafeRow: Optional[Union[bool, str]] = None, + ) -> "DataFrame": + """ + Loads a JSON file stream and returns the results as a :class:`DataFrame`. + + `JSON Lines `_ (newline-delimited JSON) is supported by default. + For JSON (one record per file), set the ``multiLine`` parameter to ``true``. + + If the ``schema`` parameter is not specified, this function goes + through the input once to determine the input schema. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + path : str + string represents path to the JSON dataset, + or RDD of Strings storing JSON objects. + schema : :class:`pyspark.sql.types.StructType` or str, optional + an optional :class:`pyspark.sql.types.StructType` for the input schema + or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ + in the version you use. + + .. # noqa + + Notes + ----- + This API is evolving. + + Examples + -------- + Load a data stream from a temporary JSON file. + + >>> import tempfile + >>> import time + >>> with tempfile.TemporaryDirectory(prefix="json") as d: + ... # Write a temporary JSON file to read it. + ... spark.createDataFrame( + ... [(100, "Hyukjin Kwon"),], ["age", "name"] + ... ).write.mode("overwrite").format("json").save(d) + ... + ... # Start a streaming query to read the JSON file. + ... q = spark.readStream.schema( + ... "age INT, name STRING" + ... ).json(d).writeStream.format("console").start() + ... time.sleep(3) + ... q.stop() + """ + self._set_opts( + schema=schema, + primitivesAsString=primitivesAsString, + prefersDecimal=prefersDecimal, + allowComments=allowComments, + allowUnquotedFieldNames=allowUnquotedFieldNames, + allowSingleQuotes=allowSingleQuotes, + allowNumericLeadingZero=allowNumericLeadingZero, + allowBackslashEscapingAnyCharacter=allowBackslashEscapingAnyCharacter, + mode=mode, + columnNameOfCorruptRecord=columnNameOfCorruptRecord, + dateFormat=dateFormat, + timestampFormat=timestampFormat, + multiLine=multiLine, + allowUnquotedControlChars=allowUnquotedControlChars, + lineSep=lineSep, + locale=locale, + dropFieldIfAllNull=dropFieldIfAllNull, + encoding=encoding, + pathGlobFilter=pathGlobFilter, + recursiveFileLookup=recursiveFileLookup, + allowNonNumericNumbers=allowNonNumericNumbers, + useUnsafeRow=useUnsafeRow, + ) + if isinstance(path, str): + return self._df(self._jreader.json(path)) + else: + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "path", + "expected_type": "str", + "arg_type": type(path).__name__, + }, + ) + + def orc( + self, + path: str, + mergeSchema: Optional[bool] = None, + pathGlobFilter: Optional[Union[bool, str]] = None, + recursiveFileLookup: Optional[Union[bool, str]] = None, + ) -> "DataFrame": + """Loads a ORC file stream, returning the result as a :class:`DataFrame`. + + .. versionadded:: 2.3.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ + in the version you use. + + .. # noqa + + Examples + -------- + Load a data stream from a temporary ORC file. + + >>> import tempfile + >>> import time + >>> with tempfile.TemporaryDirectory(prefix="orc") as d: + ... # Write a temporary ORC file to read it. + ... spark.range(10).write.mode("overwrite").format("orc").save(d) + ... + ... # Start a streaming query to read the ORC file. + ... q = spark.readStream.schema("id LONG").orc(d).writeStream.format("console").start() + ... time.sleep(3) + ... q.stop() + """ + self._set_opts( + mergeSchema=mergeSchema, + pathGlobFilter=pathGlobFilter, + recursiveFileLookup=recursiveFileLookup, + ) + if isinstance(path, str): + return self._df(self._jreader.orc(path)) + else: + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "path", + "expected_type": "str", + "arg_type": type(path).__name__, + }, + ) + + def parquet( + self, + path: str, + mergeSchema: Optional[bool] = None, + pathGlobFilter: Optional[Union[bool, str]] = None, + recursiveFileLookup: Optional[Union[bool, str]] = None, + datetimeRebaseMode: Optional[Union[bool, str]] = None, + int96RebaseMode: Optional[Union[bool, str]] = None, + ) -> "DataFrame": + """ + Loads a Parquet file stream, returning the result as a :class:`DataFrame`. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + path : str + the path in any Hadoop supported file system + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_. + in the version you use. + + .. # noqa + + Examples + -------- + Load a data stream from a temporary Parquet file. + + >>> import tempfile + >>> import time + >>> with tempfile.TemporaryDirectory(prefix="parquet") as d: + ... # Write a temporary Parquet file to read it. + ... spark.range(10).write.mode("overwrite").format("parquet").save(d) + ... + ... # Start a streaming query to read the Parquet file. + ... q = spark.readStream.schema( + ... "id LONG").parquet(d).writeStream.format("console").start() + ... time.sleep(3) + ... q.stop() + """ + self._set_opts( + mergeSchema=mergeSchema, + pathGlobFilter=pathGlobFilter, + recursiveFileLookup=recursiveFileLookup, + datetimeRebaseMode=datetimeRebaseMode, + int96RebaseMode=int96RebaseMode, + ) + if isinstance(path, str): + return self._df(self._jreader.parquet(path)) + else: + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "path", + "expected_type": "str", + "arg_type": type(path).__name__, + }, + ) + + def text( + self, + path: str, + wholetext: bool = False, + lineSep: Optional[str] = None, + pathGlobFilter: Optional[Union[bool, str]] = None, + recursiveFileLookup: Optional[Union[bool, str]] = None, + ) -> "DataFrame": + """ + Loads a text file stream and returns a :class:`DataFrame` whose schema starts with a + string column named "value", and followed by partitioned columns if there + are any. + The text files must be encoded as UTF-8. + + By default, each line in the text file is a new row in the resulting DataFrame. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + path : str + string for input path. + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ + in the version you use. + + .. # noqa + + Notes + ----- + This API is evolving. + + Examples + -------- + Load a data stream from a temporary text file. + + >>> import tempfile + >>> import time + >>> with tempfile.TemporaryDirectory(prefix="text") as d: + ... # Write a temporary text file to read it. + ... spark.createDataFrame( + ... [("hello",), ("this",)]).write.mode("overwrite").format("text").save(d) + ... + ... # Start a streaming query to read the text file. + ... q = spark.readStream.text(d).writeStream.format("console").start() + ... time.sleep(3) + ... q.stop() + """ + self._set_opts( + wholetext=wholetext, + lineSep=lineSep, + pathGlobFilter=pathGlobFilter, + recursiveFileLookup=recursiveFileLookup, + ) + if isinstance(path, str): + return self._df(self._jreader.text(path)) + else: + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "path", + "expected_type": "str", + "arg_type": type(path).__name__, + }, + ) + + def csv( + self, + path: str, + schema: Optional[Union[StructType, str]] = None, + sep: Optional[str] = None, + encoding: Optional[str] = None, + quote: Optional[str] = None, + escape: Optional[str] = None, + comment: Optional[str] = None, + header: Optional[Union[bool, str]] = None, + inferSchema: Optional[Union[bool, str]] = None, + ignoreLeadingWhiteSpace: Optional[Union[bool, str]] = None, + ignoreTrailingWhiteSpace: Optional[Union[bool, str]] = None, + nullValue: Optional[str] = None, + nanValue: Optional[str] = None, + positiveInf: Optional[str] = None, + negativeInf: Optional[str] = None, + dateFormat: Optional[str] = None, + timestampFormat: Optional[str] = None, + maxColumns: Optional[Union[int, str]] = None, + maxCharsPerColumn: Optional[Union[int, str]] = None, + maxMalformedLogPerPartition: Optional[Union[int, str]] = None, + mode: Optional[str] = None, + columnNameOfCorruptRecord: Optional[str] = None, + multiLine: Optional[Union[bool, str]] = None, + charToEscapeQuoteEscaping: Optional[Union[bool, str]] = None, + enforceSchema: Optional[Union[bool, str]] = None, + emptyValue: Optional[str] = None, + locale: Optional[str] = None, + lineSep: Optional[str] = None, + pathGlobFilter: Optional[Union[bool, str]] = None, + recursiveFileLookup: Optional[Union[bool, str]] = None, + unescapedQuoteHandling: Optional[str] = None, + ) -> "DataFrame": + r"""Loads a CSV file stream and returns the result as a :class:`DataFrame`. + + This function will go through the input once to determine the input schema if + ``inferSchema`` is enabled. To avoid going through the entire data once, disable + ``inferSchema`` option or specify the schema explicitly using ``schema``. + + Parameters + ---------- + path : str + string for input path. + schema : :class:`pyspark.sql.types.StructType` or str, optional + an optional :class:`pyspark.sql.types.StructType` for the input schema + or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ + in the version you use. + + .. # noqa + + Notes + ----- + This API is evolving. + + Examples + -------- + Load a data stream from a temporary CSV file. + + >>> import tempfile + >>> import time + >>> with tempfile.TemporaryDirectory(prefix="csv") as d: + ... # Write a temporary text file to read it. + ... spark.createDataFrame([(1, "2"),]).write.mode("overwrite").format("csv").save(d) + ... + ... # Start a streaming query to read the CSV file. + ... q = spark.readStream.schema( + ... "col0 INT, col1 STRING" + ... ).format("csv").load(d).writeStream.format("console").start() + ... time.sleep(3) + ... q.stop() + """ + self._set_opts( + schema=schema, + sep=sep, + encoding=encoding, + quote=quote, + escape=escape, + comment=comment, + header=header, + inferSchema=inferSchema, + ignoreLeadingWhiteSpace=ignoreLeadingWhiteSpace, + ignoreTrailingWhiteSpace=ignoreTrailingWhiteSpace, + nullValue=nullValue, + nanValue=nanValue, + positiveInf=positiveInf, + negativeInf=negativeInf, + dateFormat=dateFormat, + timestampFormat=timestampFormat, + maxColumns=maxColumns, + maxCharsPerColumn=maxCharsPerColumn, + maxMalformedLogPerPartition=maxMalformedLogPerPartition, + mode=mode, + columnNameOfCorruptRecord=columnNameOfCorruptRecord, + multiLine=multiLine, + charToEscapeQuoteEscaping=charToEscapeQuoteEscaping, + enforceSchema=enforceSchema, + emptyValue=emptyValue, + locale=locale, + lineSep=lineSep, + pathGlobFilter=pathGlobFilter, + recursiveFileLookup=recursiveFileLookup, + unescapedQuoteHandling=unescapedQuoteHandling, + ) + if isinstance(path, str): + return self._df(self._jreader.csv(path)) + else: + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "path", + "expected_type": "str", + "arg_type": type(path).__name__, + }, + ) + + def xml( + self, + path: str, + rowTag: Optional[str] = None, + schema: Optional[Union[StructType, str]] = None, + excludeAttribute: Optional[Union[bool, str]] = None, + attributePrefix: Optional[str] = None, + valueTag: Optional[str] = None, + ignoreSurroundingSpaces: Optional[Union[bool, str]] = None, + rowValidationXSDPath: Optional[str] = None, + ignoreNamespace: Optional[Union[bool, str]] = None, + wildcardColName: Optional[str] = None, + encoding: Optional[str] = None, + inferSchema: Optional[Union[bool, str]] = None, + nullValue: Optional[str] = None, + dateFormat: Optional[str] = None, + timestampFormat: Optional[str] = None, + mode: Optional[str] = None, + columnNameOfCorruptRecord: Optional[str] = None, + multiLine: Optional[Union[bool, str]] = None, + samplingRatio: Optional[Union[float, str]] = None, + locale: Optional[str] = None, + ) -> "DataFrame": + r"""Loads a XML file stream and returns the result as a :class:`DataFrame`. + + If the ``schema`` parameter is not specified, this function goes + through the input once to determine the input schema. + + Parameters + ---------- + path : str + string for input path. + schema : :class:`pyspark.sql.types.StructType` or str, optional + an optional :class:`pyspark.sql.types.StructType` for the input schema + or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). + + .. versionadded:: 4.0.0 + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ + in the version you use. + + .. # noqa + + Notes + ----- + This API is evolving. + + Examples + -------- + Write a DataFrame into a XML file and read it back. + + >>> import tempfile + >>> import time + >>> with tempfile.TemporaryDirectory(prefix="xml") as d: + ... # Write a DataFrame into a XML file + ... spark.createDataFrame( + ... [{"age": 100, "name": "Hyukjin Kwon"}] + ... ).write.mode("overwrite").option("rowTag", "person").xml(d) + ... + ... # Start a streaming query to read the XML file. + ... q = spark.readStream.schema( + ... "age INT, name STRING" + ... ).xml(d, rowTag="person").writeStream.format("console").start() + ... time.sleep(3) + ... q.stop() + """ + self._set_opts( + rowTag=rowTag, + schema=schema, + excludeAttribute=excludeAttribute, + attributePrefix=attributePrefix, + valueTag=valueTag, + ignoreSurroundingSpaces=ignoreSurroundingSpaces, + rowValidationXSDPath=rowValidationXSDPath, + ignoreNamespace=ignoreNamespace, + wildcardColName=wildcardColName, + encoding=encoding, + inferSchema=inferSchema, + nullValue=nullValue, + dateFormat=dateFormat, + timestampFormat=timestampFormat, + mode=mode, + columnNameOfCorruptRecord=columnNameOfCorruptRecord, + multiLine=multiLine, + samplingRatio=samplingRatio, + locale=locale, + ) + if isinstance(path, str): + return self._df(self._jreader.xml(path)) + else: + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "path", + "expected_type": "str", + "arg_type": type(path).__name__, + }, + ) + + def table(self, tableName: str) -> "DataFrame": + """Define a Streaming DataFrame on a Table. The DataSource corresponding to the table should + support streaming mode. + + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + tableName : str + string, for the name of the table. + + Returns + ------- + :class:`DataFrame` + + Notes + ----- + This API is evolving. + + Examples + -------- + Load a data stream from a table. + + >>> import tempfile + >>> import time + >>> _ = spark.sql("DROP TABLE IF EXISTS my_table") + >>> with tempfile.TemporaryDirectory(prefix="table") as d: + ... # Create a table with Rate source. + ... q1 = spark.readStream.format("rate").load().writeStream.toTable( + ... "my_table", checkpointLocation=d) + ... + ... # Read the table back and print out in the console. + ... q2 = spark.readStream.table("my_table").writeStream.format("console").start() + ... time.sleep(3) + ... q1.stop() + ... q2.stop() + ... _ = spark.sql("DROP TABLE my_table") + """ + if isinstance(tableName, str): + return self._df(self._jreader.table(tableName)) + else: + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "tableName", + "expected_type": "str", + "arg_type": type(tableName).__name__, + }, + ) + + def changes(self, tableName: str) -> "DataFrame": + """Returns the row-level changes (Change Data Capture) from the specified table + as a streaming :class:`DataFrame`. Currently this API is only supported for + Data Source V2 tables whose catalog implements ``TableCatalog.loadChangelog()``. + + Use :meth:`option` to specify the starting version/timestamp and processing options. + + .. versionadded:: 4.2.0 + + Parameters + ---------- + tableName : str + string, name of the table. + + Returns + ------- + :class:`DataFrame` + + Notes + ----- + This API is evolving. + + Examples + -------- + >>> spark.readStream.option("startingVersion", "10").changes( + ... "my_table" + ... ) # doctest: +SKIP + """ + if isinstance(tableName, str): + return self._df(self._jreader.changes(tableName)) + else: + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "tableName", + "expected_type": "str", + "arg_type": type(tableName).__name__, + }, + ) + + +class DataStreamWriter: + """ + Interface used to write a streaming :class:`DataFrame ` to external + storage systems (e.g. file systems, key-value stores, etc). + Use :attr:`DataFrame.writeStream ` + to access this. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Notes + ----- + This API is evolving. + + Examples + -------- + The example below uses Rate source that generates rows continuously. + After that, we operate a modulo by 3, and then writes the stream out to the console. + The streaming query stops in 3 seconds. + + >>> import time + >>> df = spark.readStream.format("rate").load() + >>> df = df.selectExpr("value % 3 as v") + >>> q = df.writeStream.format("console").start() + >>> time.sleep(3) + >>> q.stop() + """ + + def __init__(self, df: "DataFrame") -> None: + self._df = df + self._spark = df.sparkSession + self._jwrite = df._jdf.writeStream() + + def _sq(self, jsq: "JavaObject") -> StreamingQuery: + return StreamingQuery(jsq) + + def outputMode(self, outputMode: str) -> "DataStreamWriter": + """Specifies how data of a streaming DataFrame/Dataset is written to a streaming sink. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Options include: + + * `append`: Only the new rows in the streaming DataFrame/Dataset will be written to + the sink + * `complete`: All the rows in the streaming DataFrame/Dataset will be written to the sink + every time these are some updates + * `update`: only the rows that were updated in the streaming DataFrame/Dataset will be + written to the sink every time there are some updates. If the query doesn't contain + aggregations, it will be equivalent to `append` mode. + + Notes + ----- + This API is evolving. + + Examples + -------- + >>> df = spark.readStream.format("rate").load() + >>> df.writeStream.outputMode('append') + <...streaming.readwriter.DataStreamWriter object ...> + + The example below uses Complete mode that the entire aggregated counts are printed out. + + >>> import time + >>> df = spark.readStream.format("rate").option("rowsPerSecond", 10).load() + >>> df = df.groupby().count() + >>> q = df.writeStream.outputMode("complete").format("console").start() + >>> time.sleep(3) + >>> q.stop() + """ + if not outputMode or not isinstance(outputMode, str) or len(outputMode.strip()) == 0: + raise PySparkValueError( + errorClass="VALUE_NOT_NON_EMPTY_STR", + messageParameters={"arg_name": "outputMode", "arg_value": str(outputMode)}, ) + self._jwrite = self._jwrite.outputMode(outputMode) + return self - def doesMethodExist(method_name: str) -> bool: - exists = hasattr(f, method_name) - if exists and not callable(getattr(f, method_name)): + def format(self, source: str) -> "DataStreamWriter": + """Specifies the underlying output data source. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + source : str + string, name of the data source, which for now can be 'parquet'. + + Notes + ----- + This API is evolving. + + Examples + -------- + >>> df = spark.readStream.format("rate").load() + >>> df.writeStream.format("text") + <...streaming.readwriter.DataStreamWriter object ...> + + This API allows to configure the source to write. The example below writes a CSV + file from Rate source in a streaming manner. + + >>> import tempfile + >>> import time + >>> with tempfile.TemporaryDirectory(prefix="format1") as d: + ... with tempfile.TemporaryDirectory(prefix="format2") as cp: + ... df = spark.readStream.format("rate").load() + ... q = df.writeStream.format("csv").option("checkpointLocation", cp).start(d) + ... time.sleep(5) + ... q.stop() + ... spark.read.schema("timestamp TIMESTAMP, value STRING").csv(d).show() + +...---------+-----+ + |...timestamp|value| + +...---------+-----+ + ... + """ + self._jwrite = self._jwrite.format(source) + return self + + def option(self, key: str, value: "OptionalPrimitiveType") -> "DataStreamWriter": + """Adds an output option for the underlying data source. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Notes + ----- + This API is evolving. + + Examples + -------- + >>> df = spark.readStream.format("rate").load() + >>> df.writeStream.option("x", 1) + <...streaming.readwriter.DataStreamWriter object ...> + + The example below specifies 'numRows' option to Console source in order to print + 3 rows for every batch. + + >>> import time + >>> q = spark.readStream.format( + ... "rate").option("rowsPerSecond", 10).load().writeStream.format( + ... "console").option("numRows", 3).start() + >>> time.sleep(3) + >>> q.stop() + """ + if value is None: + return self + self._jwrite = self._jwrite.option(key, to_str(value)) + return self + + def options(self, **options: "OptionalPrimitiveType") -> "DataStreamWriter": + """Adds output options for the underlying data source. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Notes + ----- + This API is evolving. + + Examples + -------- + >>> df = spark.readStream.format("rate").load() + >>> df.writeStream.option("x", 1) + <...streaming.readwriter.DataStreamWriter object ...> + + Specify options in a dictionary. + + >>> df.writeStream.options(**{"k1": "v1", "k2": "v2"}) + <...streaming.readwriter.DataStreamWriter object ...> + + The example below specifies 'numRows' and 'truncate' options to Console source in order + to print 3 rows for every batch without truncating the results. + + >>> import time + >>> q = spark.readStream.format( + ... "rate").option("rowsPerSecond", 10).load().writeStream.format( + ... "console").options(numRows=3, truncate=False).start() + >>> time.sleep(3) + >>> q.stop() + """ + for k, v in options.items(): + if v is not None: + self._jwrite = self._jwrite.option(k, to_str(v)) + return self + + @overload + def partitionBy(self, *cols: str) -> "DataStreamWriter": ... + + @overload + def partitionBy(self, __cols: List[str]) -> "DataStreamWriter": ... + + def partitionBy(self, *cols: str) -> "DataStreamWriter": # type: ignore[misc] + """Partitions the output by the given columns on the file system. + + If specified, the output is laid out on the file system similar + to Hive's partitioning scheme. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + cols : str or list + name of columns + + Notes + ----- + This API is evolving. + + Examples + -------- + >>> df = spark.readStream.format("rate").load() + >>> df.writeStream.partitionBy("value") + <...streaming.readwriter.DataStreamWriter object ...> + + Partition-by timestamp column from Rate source. + + >>> import tempfile + >>> import time + >>> with tempfile.TemporaryDirectory(prefix="partitionBy1") as d: + ... with tempfile.TemporaryDirectory(prefix="partitionBy2") as cp: + ... df = spark.readStream.format("rate").option("rowsPerSecond", 10).load() + ... q = df.writeStream.partitionBy( + ... "timestamp").format("parquet").option("checkpointLocation", cp).start(d) + ... time.sleep(5) + ... q.stop() + ... spark.read.schema(df.schema).parquet(d).show() + +...---------+-----+ + |...timestamp|value| + +...---------+-----+ + ... + """ + from pyspark.sql.classic.column import _to_seq + + if len(cols) == 1 and isinstance(cols[0], (list, tuple)): + cols = cols[0] + self._jwrite = self._jwrite.partitionBy(_to_seq(self._spark._sc, cols)) + return self + + @overload + def clusterBy(self, *cols: str) -> "DataStreamWriter": ... + + @overload + def clusterBy(self, __cols: List[str]) -> "DataStreamWriter": ... + + def clusterBy(self, *cols: str) -> "DataStreamWriter": # type: ignore[misc] + """Clusters the output by the given columns. + + If specified, the output is laid out such that records with similar values on the clustering + column(s) are grouped together in the same file. + + Clustering improves query efficiency by allowing queries with predicates on the clustering + columns to skip unnecessary data. Unlike partitioning, clustering can be used on very high + cardinality columns. + + .. versionadded:: 4.0.0 + + Parameters + ---------- + cols : str or list + name of columns + + Notes + ----- + This API is evolving. + + Examples + -------- + >>> df = spark.readStream.format("rate").load() + >>> df.writeStream.clusterBy("value") + <...streaming.readwriter.DataStreamWriter object ...> + + Cluster-by timestamp column from Rate source. + + >>> import tempfile + >>> import time + >>> with tempfile.TemporaryDirectory(prefix="partitionBy1") as d: + ... with tempfile.TemporaryDirectory(prefix="partitionBy2") as cp: + ... df = spark.readStream.format("rate").option("rowsPerSecond", 10).load() + ... q = df.writeStream.clusterBy( + ... "timestamp").format("parquet").option("checkpointLocation", cp).start(d) + ... time.sleep(5) + ... q.stop() + ... spark.read.schema(df.schema).parquet(d).show() + +...---------+-----+ + |...timestamp|value| + +...---------+-----+ + ... + """ + from pyspark.sql.classic.column import _to_seq + + if len(cols) == 1 and isinstance(cols[0], (list, tuple)): + cols = cols[0] + self._jwrite = self._jwrite.clusterBy(_to_seq(self._spark._sc, cols)) + return self + + def queryName(self, queryName: str) -> "DataStreamWriter": + """Specifies the name of the :class:`StreamingQuery` that can be started with + :func:`start`. This name must be unique among all the currently active queries + in the associated SparkSession. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + queryName : str + unique name for the query + + Notes + ----- + This API is evolving. + + Examples + -------- + >>> import time + >>> df = spark.readStream.format("rate").load() + >>> q = df.writeStream.queryName("streaming_query").format("console").start() + >>> q.stop() + >>> q.name + 'streaming_query' + """ + if not queryName or not isinstance(queryName, str) or len(queryName.strip()) == 0: + raise PySparkValueError( + errorClass="VALUE_NOT_NON_EMPTY_STR", + messageParameters={"arg_name": "queryName", "arg_value": str(queryName)}, + ) + self._jwrite = self._jwrite.queryName(queryName) + return self + + @overload + def trigger(self, *, processingTime: str) -> "DataStreamWriter": ... + + @overload + def trigger(self, *, once: bool) -> "DataStreamWriter": ... + + @overload + def trigger(self, *, continuous: str) -> "DataStreamWriter": ... + + @overload + def trigger(self, *, availableNow: bool) -> "DataStreamWriter": ... + + def trigger( + self, + *, + processingTime: Optional[str] = None, + once: Optional[bool] = None, + continuous: Optional[str] = None, + availableNow: Optional[bool] = None, + realTime: Optional[str] = None, + ) -> "DataStreamWriter": + """Set the trigger for the stream query. If this is not set it will run the query as fast + as possible, which is equivalent to setting the trigger to ``processingTime='0 seconds'``. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + processingTime : str, optional + a processing time interval as a string, e.g. '5 seconds', '1 minute'. + Set a trigger that runs a microbatch query periodically based on the + processing time. Only one trigger can be set. + once : bool, optional + if set to True, set a trigger that processes only one batch of data in a + streaming query then terminates the query. Only one trigger can be set. + continuous : str, optional + a time interval as a string, e.g. '5 seconds', '1 minute'. + Set a trigger that runs a continuous query with a given checkpoint + interval. Only one trigger can be set. + availableNow : bool, optional + if set to True, set a trigger that processes all available data in multiple + batches then terminates the query. Only one trigger can be set. + realTime : str, optional + a batch duration as a string, e.g. '5 seconds', '1 minute'. + Set a trigger that runs a real time mode query with + batch at the specified duration. Only one trigger can be set. + + Notes + ----- + This API is evolving. + + Examples + -------- + >>> df = spark.readStream.format("rate").load() + + Trigger the query for execution every 5 seconds + + >>> df.writeStream.trigger(processingTime='5 seconds') + <...streaming.readwriter.DataStreamWriter object ...> + + Trigger the query for execution every 5 seconds + + >>> df.writeStream.trigger(continuous='5 seconds') + <...streaming.readwriter.DataStreamWriter object ...> + + Trigger the query for reading all available data with multiple batches + + >>> df.writeStream.trigger(availableNow=True) + <...streaming.readwriter.DataStreamWriter object ...> + + Trigger the query for real time mode execution every 5 seconds. + + >>> df.writeStream.trigger(realTime='5 seconds') + <...streaming.readwriter.DataStreamWriter object ...> + """ + params = [processingTime, once, continuous, availableNow, realTime] + + if params.count(None) == 5: + raise PySparkValueError( + errorClass="ONLY_ALLOW_SINGLE_TRIGGER", + messageParameters={}, + ) + elif params.count(None) < 4: + raise PySparkValueError( + errorClass="ONLY_ALLOW_SINGLE_TRIGGER", + messageParameters={}, + ) + + jTrigger = None + assert self._spark._sc._jvm is not None + if processingTime is not None: + if not isinstance(processingTime, str) or len(processingTime.strip()) == 0: + raise PySparkValueError( + errorClass="VALUE_NOT_NON_EMPTY_STR", + messageParameters={ + "arg_name": "processingTime", + "arg_value": str(processingTime), + }, + ) + interval = processingTime.strip() + jTrigger = getattr( + self._spark._sc._jvm, "org.apache.spark.sql.streaming.Trigger" + ).ProcessingTime(interval) + + elif once is not None: + if once is not True: + raise PySparkValueError( + errorClass="VALUE_NOT_ALLOWED", + messageParameters={"arg_name": "once", "allowed_values": "[True]"}, + ) + + jTrigger = getattr( + self._spark._sc._jvm, "org.apache.spark.sql.streaming.Trigger" + ).Once() + + elif continuous is not None: + if not isinstance(continuous, str) or len(continuous.strip()) == 0: + raise PySparkValueError( + errorClass="VALUE_NOT_NON_EMPTY_STR", + messageParameters={"arg_name": "continuous", "arg_value": str(continuous)}, + ) + interval = continuous.strip() + jTrigger = getattr( + self._spark._sc._jvm, "org.apache.spark.sql.streaming.Trigger" + ).Continuous(interval) + elif realTime is not None: + if not isinstance(realTime, str) or len(realTime.strip()) == 0: + raise PySparkValueError( + errorClass="VALUE_NOT_NON_EMPTY_STR", + messageParameters={"arg_name": "realTime", "arg_value": str(realTime)}, + ) + batch_duration = realTime.strip() + jTrigger = getattr( + self._spark._sc._jvm, "org.apache.spark.sql.streaming.Trigger" + ).RealTime(batch_duration) + else: + if availableNow is not True: + raise PySparkValueError( + errorClass="VALUE_NOT_ALLOWED", + messageParameters={"arg_name": "availableNow", "allowed_values": "[True]"}, + ) + jTrigger = getattr( + self._spark._sc._jvm, "org.apache.spark.sql.streaming.Trigger" + ).AvailableNow() + + self._jwrite = self._jwrite.trigger(jTrigger) + return self + + @staticmethod + def _construct_foreach_function( + f: Union[Callable[[Row], None], "SupportsProcess"], + ) -> Callable[[Any, Iterator], Iterator]: + from pyspark.taskcontext import TaskContext + + if callable(f): + # The provided object is a callable function that is supposed to be called on each row. + # Construct a function that takes an iterator and calls the provided function on each + # row. + def func_without_process(_: Any, iterator: Iterator) -> Iterator: + for x in iterator: + f(x) + return iter([]) + + return func_without_process + + else: + # The provided object is not a callable function. Then it is expected to have a + # 'process(row)' method, and optional 'open(partition_id, epoch_id)' and + # 'close(error)' methods. + + if not hasattr(f, "process"): raise PySparkAttributeError( errorClass="ATTRIBUTE_NOT_CALLABLE", - messageParameters={"attr_name": method_name, "obj_name": "f"}, + messageParameters={"attr_name": "process", "obj_name": "f"}, ) - return exists - open_exists = doesMethodExist("open") - close_exists = doesMethodExist("close") + if not callable(getattr(f, "process")): + raise PySparkAttributeError( + errorClass="ATTRIBUTE_NOT_CALLABLE", + messageParameters={"attr_name": "process", "obj_name": "f"}, + ) - def func_with_open_process_close(partition_id: Any, iterator: Iterator) -> Iterator: - from pyspark.taskcontext import TaskContext + def doesMethodExist(method_name: str) -> bool: + exists = hasattr(f, method_name) + if exists and not callable(getattr(f, method_name)): + raise PySparkAttributeError( + errorClass="ATTRIBUTE_NOT_CALLABLE", + messageParameters={"attr_name": method_name, "obj_name": "f"}, + ) + return exists - epoch_id = cast(TaskContext, TaskContext.get()).getLocalProperty( - "streaming.sql.batchId" - ) - if epoch_id: - int_epoch_id = int(epoch_id) - else: - raise PySparkRuntimeError( - errorClass="CANNOT_GET_BATCH_ID", - messageParameters={"obj_name": "TaskContext"}, + open_exists = doesMethodExist("open") + close_exists = doesMethodExist("close") + + def func_with_open_process_close(partition_id: Any, iterator: Iterator) -> Iterator: + epoch_id = cast(TaskContext, TaskContext.get()).getLocalProperty( + "streaming.sql.batchId" ) + if epoch_id: + int_epoch_id = int(epoch_id) + else: + raise PySparkRuntimeError( + errorClass="CANNOT_GET_BATCH_ID", + messageParameters={"obj_name": "TaskContext"}, + ) + + # Check if the data should be processed + should_process = True + if open_exists: + should_process = f.open( # type: ignore[attr-defined] + partition_id, int_epoch_id + ) + + error = None + + try: + if should_process: + for x in iterator: + f.process(x) + except Exception as ex: + error = ex + finally: + if close_exists: + f.close(error) # type: ignore[attr-defined] + if error: + raise error + + return iter([]) + + return func_with_open_process_close + + @overload + def foreach(self, f: Callable[[Row], None]) -> "DataStreamWriter": ... + + @overload + def foreach(self, f: "SupportsProcess") -> "DataStreamWriter": ... + + def foreach(self, f: Union[Callable[[Row], None], "SupportsProcess"]) -> "DataStreamWriter": + """ + Sets the output of the streaming query to be processed using the provided writer ``f``. + This is often used to write the output of a streaming query to arbitrary storage systems. + The processing logic can be specified in two ways. + + #. A **function** that takes a row as input. + This is a simple way to express your processing logic. Note that this does + not allow you to deduplicate generated data when failures cause reprocessing of + some input data. That would require you to specify the processing logic in the next + way. + + #. An **object** with a ``process`` method and optional ``open`` and ``close`` methods. + The object can have the following methods. + + * ``open(partition_id, epoch_id)``: *Optional* method that initializes the processing + (for example, open a connection, start a transaction, etc). Additionally, you can + use the `partition_id` and `epoch_id` to deduplicate regenerated data + (discussed later). + + * ``process(row)``: *Non-optional* method that processes each :class:`Row`. + + * ``close(error)``: *Optional* method that finalizes and cleans up (for example, + close connection, commit transaction, etc.) after all rows have been processed. + + The object will be used by Spark in the following way. + + * A single copy of this object is responsible of all the data generated by a + single task in a query. In other words, one instance is responsible for + processing one partition of the data generated in a distributed manner. + + * This object must be serializable because each task will get a fresh + serialized-deserialized copy of the provided object. Hence, it is strongly + recommended that any initialization for writing data (e.g. opening a + connection or starting a transaction) is done after the `open(...)` + method has been called, which signifies that the task is ready to generate data. + + * The lifecycle of the methods are as follows. + + For each partition with ``partition_id``: + + ... For each batch/epoch of streaming data with ``epoch_id``: + + ....... Method ``open(partitionId, epochId)`` is called. + + ....... If ``open(...)`` returns true, for each row in the partition and + batch/epoch, method ``process(row)`` is called. + + ....... Method ``close(errorOrNull)`` is called with error (if any) seen while + processing rows. + + Important points to note: + + * The `partitionId` and `epochId` can be used to deduplicate generated data when + failures cause reprocessing of some input data. This depends on the execution + mode of the query. If the streaming query is being executed in the micro-batch + mode, then every partition represented by a unique tuple (partition_id, epoch_id) + is guaranteed to have the same data. Hence, (partition_id, epoch_id) can be used + to deduplicate and/or transactionally commit data and achieve exactly-once + guarantees. However, if the streaming query is being executed in the continuous + mode, then this guarantee does not hold and therefore should not be used for + deduplication. + + * The ``close()`` method (if exists) will be called if `open()` method exists and + returns successfully (irrespective of the return value), except if the Python + crashes in the middle. + + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Notes + ----- + This API is evolving. + + Examples + -------- + >>> import time + >>> df = spark.readStream.format("rate").load() + + Print every row using a function + + >>> def print_row(row): + ... print(row) + ... + >>> q = df.writeStream.foreach(print_row).start() + >>> time.sleep(3) + >>> q.stop() + + Print every row using a object with process() method + + >>> class RowPrinter: + ... def open(self, partition_id, epoch_id): + ... print("Opened %d, %d" % (partition_id, epoch_id)) + ... return True + ... + ... def process(self, row): + ... print(row) + ... + ... def close(self, error): + ... print("Closed with error: %s" % str(error)) + ... + >>> q = df.writeStream.foreach(RowPrinter()).start() + >>> time.sleep(3) + >>> q.stop() + """ + + from pyspark.core.rdd import _wrap_function + from pyspark.serializers import CPickleSerializer, AutoBatchedSerializer + + func = self._construct_foreach_function(f) + serializer = AutoBatchedSerializer(CPickleSerializer()) + wrapped_func = _wrap_function(self._spark._sc, func, serializer, serializer) + assert self._spark._sc._jvm is not None + jForeachWriter = getattr( + self._spark._sc._jvm, + "org.apache.spark.sql.execution.python.streaming.PythonForeachWriter", + )(wrapped_func, self._df._jdf.schema()) + self._jwrite.foreach(jForeachWriter) + return self + + def foreachBatch(self, func: Callable[["DataFrame", int], None]) -> "DataStreamWriter": + """ + Sets the output of the streaming query to be processed using the provided + function. This is supported only the in the micro-batch execution modes (that is, when the + trigger is not continuous). In every micro-batch, the provided function will be called in + every micro-batch with (i) the output rows as a DataFrame and (ii) the batch identifier. + The batchId can be used deduplicate and transactionally write the output + (that is, the provided Dataset) to external systems. The output DataFrame is guaranteed + to exactly same for the same batchId (assuming all operations are deterministic in the + query). + + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Notes + ----- + This API is evolving. + This function behaves differently in Spark Connect mode. See examples. + In Connect, the provided function doesn't have access to variables defined outside of it. + + Examples + -------- + >>> import time + >>> df = spark.readStream.format("rate").load() + >>> my_value = -1 + >>> def func(batch_df, batch_id): + ... global my_value + ... my_value = 100 + ... batch_df.collect() + ... + >>> q = df.writeStream.foreachBatch(func).start() + >>> time.sleep(3) + >>> q.stop() + >>> # if in Spark Connect, my_value = -1, else my_value = 100 + """ + from py4j.java_gateway import java_import + from pyspark.java_gateway import ensure_callback_server_started + + gw = self._spark._sc._gateway + assert gw is not None + java_import(gw.jvm, "org.apache.spark.sql.execution.streaming.sources.*") + + wrapped_func = ForeachBatchFunction(self._spark, func) + gw.jvm.PythonForeachBatchHelper.callForeachBatch(self._jwrite, wrapped_func) + ensure_callback_server_started(gw) + return self + + def start( + self, + path: Optional[str] = None, + format: Optional[str] = None, + outputMode: Optional[str] = None, + partitionBy: Optional[Union[str, List[str]]] = None, + queryName: Optional[str] = None, + **options: "OptionalPrimitiveType", + ) -> "StreamingQuery": + """Streams the contents of the :class:`DataFrame` to a data source. + + The data source is specified by the ``format`` and a set of ``options``. + If ``format`` is not specified, the default data source configured by + ``spark.sql.sources.default`` will be used. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + path : str, optional + the path in a Hadoop supported file system + format : str, optional + the format used to save + outputMode : str, optional + specifies how data of a streaming DataFrame/Dataset is written to a + streaming sink. + + * `append`: Only the new rows in the streaming DataFrame/Dataset will be written to the + sink + * `complete`: All the rows in the streaming DataFrame/Dataset will be written to the + sink every time these are some updates + * `update`: only the rows that were updated in the streaming DataFrame/Dataset will be + written to the sink every time there are some updates. If the query doesn't contain + aggregations, it will be equivalent to `append` mode. + partitionBy : str or list, optional + names of partitioning columns + queryName : str, optional + unique name for the query + **options : dict + All other string options. You may want to provide a `checkpointLocation` + for most streams, however it is not required for a `memory` stream. + + Notes + ----- + This API is evolving. + + Examples + -------- + >>> df = spark.readStream.format("rate").load() + + Basic example. + + >>> q = df.writeStream.format('memory').queryName('this_query').start() + >>> q.isActive + True + >>> q.name + 'this_query' + >>> q.stop() + >>> q.isActive + False + + Example with using other parameters with a trigger. + + >>> q = df.writeStream.trigger(processingTime='5 seconds').start( + ... queryName='that_query', outputMode="append", format='memory') + >>> q.name + 'that_query' + >>> q.isActive + True + >>> q.stop() + """ + self.options(**options) + if outputMode is not None: + self.outputMode(outputMode) + if partitionBy is not None: + self.partitionBy(partitionBy) + if format is not None: + self.format(format) + if queryName is not None: + self.queryName(queryName) + if path is None: + return self._sq(self._jwrite.start()) + else: + return self._sq(self._jwrite.start(path)) + + def toTable( + self, + tableName: str, + format: Optional[str] = None, + outputMode: Optional[str] = None, + partitionBy: Optional[Union[str, List[str]]] = None, + queryName: Optional[str] = None, + **options: "OptionalPrimitiveType", + ) -> "StreamingQuery": + """ + Starts the execution of the streaming query, which will continually output results to the + given table as new data arrives. + + The returned :class:`StreamingQuery` object can be used to interact with the stream. + + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.5.0 + Supports Spark Connect. + + Parameters + ---------- + tableName : str + string, for the name of the table. + format : str, optional + the format used to save. + outputMode : str, optional + specifies how data of a streaming DataFrame/Dataset is written to a + streaming sink. + + * `append`: Only the new rows in the streaming DataFrame/Dataset will be written to the + sink + * `complete`: All the rows in the streaming DataFrame/Dataset will be written to the + sink every time these are some updates + * `update`: only the rows that were updated in the streaming DataFrame/Dataset will be + written to the sink every time there are some updates. If the query doesn't contain + aggregations, it will be equivalent to `append` mode. + partitionBy : str or list, optional + names of partitioning columns + queryName : str, optional + unique name for the query + **options : dict + All other string options. You may want to provide a `checkpointLocation`. + + Notes + ----- + This API is evolving. + + For v1 table, partitioning columns provided by `partitionBy` will be respected no matter + the table exists or not. A new table will be created if the table not exists. + + For v2 table, `partitionBy` will be ignored if the table already exists. `partitionBy` will + be respected only if the v2 table does not exist. Besides, the v2 table created by this API + lacks some functionalities (e.g., customized properties, options, and serde info). If you + need them, please create the v2 table manually before the execution to avoid creating a + table with incomplete information. + + Examples + -------- + Save a data stream to a table. + + >>> import tempfile + >>> import time + >>> _ = spark.sql("DROP TABLE IF EXISTS my_table2") + >>> with tempfile.TemporaryDirectory(prefix="toTable") as d: + ... # Create a table with Rate source. + ... q = spark.readStream.format("rate").option( + ... "rowsPerSecond", 10).load().writeStream.toTable( + ... "my_table2", + ... queryName='that_query', + ... outputMode="append", + ... format='parquet', + ... checkpointLocation=d) + ... time.sleep(3) + ... q.stop() + ... spark.read.table("my_table2").show() + ... _ = spark.sql("DROP TABLE my_table2") + +...---------+-----+ + |...timestamp|value| + +...---------+-----+ + ... + """ + self.options(**options) + if outputMode is not None: + self.outputMode(outputMode) + if partitionBy is not None: + self.partitionBy(partitionBy) + if format is not None: + self.format(format) + if queryName is not None: + self.queryName(queryName) + return self._sq(self._jwrite.toTable(tableName)) + + +def _test() -> None: + import doctest + import os + from pyspark.sql import SparkSession + import pyspark.sql.streaming.readwriter - # Check if the data should be processed - should_process = True - if open_exists: - should_process = f.open(partition_id, int_epoch_id) + os.chdir(os.environ["SPARK_HOME"]) - error = None + globs = pyspark.sql.streaming.readwriter.__dict__.copy() + globs["spark"] = ( + SparkSession.builder.master("local[4]") + .appName("sql.streaming.readwriter tests") + .getOrCreate() + ) - try: - if should_process: - for x in iterator: - f.process(x) - except Exception as ex: - error = ex - finally: - if close_exists: - f.close(error) - if error: - raise error + failure_count, test_count = doctest.testmod( + pyspark.sql.streaming.readwriter, + globs=globs, + optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF, + ) + globs["spark"].stop() - return iter([]) + if failure_count: + sys.exit(-1) - return func_with_open_process_close -__all__ = ["DataStreamReader", "DataStreamWriter", "Trigger"] +if __name__ == "__main__": + _test() diff --git a/python/pyspark/sql/streaming/readwriter.pyi b/python/pyspark/sql/streaming/readwriter.pyi deleted file mode 100644 index d618a2cc..00000000 --- a/python/pyspark/sql/streaming/readwriter.pyi +++ /dev/null @@ -1,1700 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Generated by scripts/gen_type_stubs.py from python/pyspark/sql/streaming/readwriter.py of Apache Spark v4.3.0: -# do not edit. Type checkers read this stub instead of the module of the drop-in. - -import re -from collections.abc import Iterator -from typing import overload, Any, Callable, List, Optional, Union -from pyspark.sql.readwriter import OptionUtils, to_str -from pyspark.sql.streaming.query import StreamingQuery -from pyspark.sql.types import Row, StructType -from pyspark.errors import PySparkTypeError, PySparkValueError, PySparkAttributeError as PySparkAttributeError, PySparkRuntimeError as PySparkRuntimeError -from py4j.java_gateway import JavaObject -from pyspark.sql.session import SparkSession -from pyspark.sql._typing import SupportsProcess, OptionalPrimitiveType -from pyspark.sql.dataframe import DataFrame - -__all__ = ['DataStreamReader', 'DataStreamWriter', 'Trigger'] - -class DataStreamReader(OptionUtils): - """Interface used to load a streaming :class:`DataFrame ` from external - storage systems (e.g. file systems, key-value stores, etc). - Use :attr:`SparkSession.readStream ` to access this. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> spark.readStream - <...streaming.readwriter.DataStreamReader object ...> - - The example below uses Rate source that generates rows continuously. - After that, we operate a modulo by 3, and then writes the stream out to the console. - The streaming query stops in 3 seconds. - - >>> import time - >>> df = spark.readStream.format("rate").load() - >>> df = df.selectExpr("value % 3 as v") - >>> q = df.writeStream.format("console").start() - >>> time.sleep(3) - >>> q.stop() - """ - def __init__(self, spark: 'SparkSession') -> None: - self._jreader = spark._jsparkSession.readStream() - self._spark = spark - def _df(self, jdf: 'JavaObject') -> 'DataFrame': ... - def format(self, source: str) -> 'DataStreamReader': - """Specifies the input data source format. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - source : str - name of the data source, e.g. 'json', 'parquet'. - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> spark.readStream.format("text") - <...streaming.readwriter.DataStreamReader object ...> - - This API allows to configure other sources to read. The example below writes a small text - file, and reads it back via Text source. - - >>> import tempfile - >>> import time - >>> with tempfile.TemporaryDirectory(prefix="format") as d: - ... # Write a temporary text file to read it. - ... spark.createDataFrame( - ... [("hello",), ("this",)]).write.mode("overwrite").format("text").save(d) - ... - ... # Start a streaming query to read the text file. - ... q = spark.readStream.format("text").load(d).writeStream.format("console").start() - ... time.sleep(3) - ... q.stop() - """ - self._jreader = self._jreader.format(source) - return self - def schema(self, schema: Union[StructType, str]) -> 'DataStreamReader': - """Specifies the input schema. - - Some data sources (e.g. JSON) can infer the input schema automatically from data. - By specifying the schema here, the underlying data source can skip the schema - inference step, and thus speed up data loading. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - schema : :class:`pyspark.sql.types.StructType` or str - a :class:`pyspark.sql.types.StructType` object or a DDL-formatted string - (For example ``col0 INT, col1 DOUBLE``). - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> from pyspark.sql.types import StructField, StructType, StringType - >>> spark.readStream.schema(StructType([StructField("data", StringType(), True)])) - <...streaming.readwriter.DataStreamReader object ...> - >>> spark.readStream.schema("col0 INT, col1 DOUBLE") - <...streaming.readwriter.DataStreamReader object ...> - - The example below specifies a different schema to CSV file. - - >>> import tempfile - >>> import time - >>> with tempfile.TemporaryDirectory(prefix="schema") as d: - ... # Start a streaming query to read the CSV file. - ... spark.readStream.schema("col0 INT, col1 STRING").format("csv").load(d).printSchema() - root - |-- col0: integer (nullable = true) - |-- col1: string (nullable = true) - """ - from pyspark.sql import SparkSession - - spark = SparkSession._getActiveSessionOrCreate() - if isinstance(schema, StructType): - jschema = spark._jsparkSession.parseDataType(schema.json()) - self._jreader = self._jreader.schema(jschema) - elif isinstance(schema, str): - self._jreader = self._jreader.schema(schema) - else: - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "arg_name": "schema", - "expected_type": "str or struct type", - "arg_type": type(schema).__name__, - }, - ) - return self - def option(self, key: str, value: 'OptionalPrimitiveType') -> 'DataStreamReader': - """Adds an input option for the underlying data source. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> spark.readStream.option("x", 1) - <...streaming.readwriter.DataStreamReader object ...> - - The example below specifies 'rowsPerSecond' option to Rate source in order to generate - 10 rows every second. - - >>> import time - >>> q = spark.readStream.format( - ... "rate").option("rowsPerSecond", 10).load().writeStream.format("console").start() - >>> time.sleep(3) - >>> q.stop() - """ - if value is None: - return self - self._jreader = self._jreader.option(key, to_str(value)) - return self - def options(self, **options: 'OptionalPrimitiveType') -> 'DataStreamReader': - """Adds input options for the underlying data source. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> spark.readStream.options(x="1", y=2) - <...streaming.readwriter.DataStreamReader object ...> - - Specify options in a dictionary. - - >>> spark.readStream.options(**{"k1": "v1", "k2": "v2"}) - <...streaming.readwriter.DataStreamReader object ...> - - The example below specifies 'rowsPerSecond' and 'numPartitions' options to - Rate source in order to generate 10 rows with 10 partitions every second. - - >>> import time - >>> q = spark.readStream.format("rate").options( - ... rowsPerSecond=10, numPartitions=10 - ... ).load().writeStream.format("console").start() - >>> time.sleep(3) - >>> q.stop() - """ - for k, v in options.items(): - if v is not None: - self._jreader = self._jreader.option(k, to_str(v)) - return self - def name(self, source_name: str) -> 'DataStreamReader': - """Specifies a name for the streaming source. - - This name is used to identify the source in checkpoint metadata and enables - stable checkpoint locations for source evolution. - - .. versionadded:: 4.2.0 - - Parameters - ---------- - source_name : str - the name to assign to this streaming source. Must contain only ASCII letters, - digits, and underscores. - - Returns - ------- - :class:`DataStreamReader` - - Notes - ----- - This API is experimental. - - Examples - -------- - >>> spark.readStream.format("rate").name("my_source") # doctest: +SKIP - <...streaming.readwriter.DataStreamReader object ...> - """ - if not isinstance(source_name, str): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "arg_name": "source_name", - "expected_type": "str", - "arg_type": type(source_name).__name__, - }, - ) - - # Validate that source_name contains only ASCII letters, digits, and underscores - if not re.match(r"^[a-zA-Z0-9_]+$", source_name): - raise PySparkValueError( - errorClass="INVALID_STREAMING_SOURCE_NAME", - messageParameters={"source_name": source_name}, - ) - - self._jreader = self._jreader.name(source_name) - return self - def load(self, path: Optional[str]=None, format: Optional[str]=None, schema: Optional[Union[StructType, str]]=None, **options: 'OptionalPrimitiveType') -> 'DataFrame': - """Loads a data stream from a data source and returns it as a - :class:`DataFrame `. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - path : str, optional - optional string for file-system backed data sources. - format : str, optional - optional string for format of the data source. Default to 'parquet'. - schema : :class:`pyspark.sql.types.StructType` or str, optional - optional :class:`pyspark.sql.types.StructType` for the input schema - or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). - **options : dict - all other string options - - Notes - ----- - This API is evolving. - - Examples - -------- - Load a data stream from a temporary JSON file. - - >>> import tempfile - >>> import time - >>> with tempfile.TemporaryDirectory(prefix="load") as d: - ... # Write a temporary JSON file to read it. - ... spark.createDataFrame( - ... [(100, "Hyukjin Kwon"),], ["age", "name"] - ... ).write.mode("overwrite").format("json").save(d) - ... - ... # Start a streaming query to read the JSON file. - ... q = spark.readStream.schema( - ... "age INT, name STRING" - ... ).format("json").load(d).writeStream.format("console").start() - ... time.sleep(3) - ... q.stop() - """ - def json(self, path: str, schema: Optional[Union[StructType, str]]=None, primitivesAsString: Optional[Union[bool, str]]=None, prefersDecimal: Optional[Union[bool, str]]=None, allowComments: Optional[Union[bool, str]]=None, allowUnquotedFieldNames: Optional[Union[bool, str]]=None, allowSingleQuotes: Optional[Union[bool, str]]=None, allowNumericLeadingZero: Optional[Union[bool, str]]=None, allowBackslashEscapingAnyCharacter: Optional[Union[bool, str]]=None, mode: Optional[str]=None, columnNameOfCorruptRecord: Optional[str]=None, dateFormat: Optional[str]=None, timestampFormat: Optional[str]=None, multiLine: Optional[Union[bool, str]]=None, allowUnquotedControlChars: Optional[Union[bool, str]]=None, lineSep: Optional[str]=None, locale: Optional[str]=None, dropFieldIfAllNull: Optional[Union[bool, str]]=None, encoding: Optional[str]=None, pathGlobFilter: Optional[Union[bool, str]]=None, recursiveFileLookup: Optional[Union[bool, str]]=None, allowNonNumericNumbers: Optional[Union[bool, str]]=None, useUnsafeRow: Optional[Union[bool, str]]=None) -> 'DataFrame': - """Loads a JSON file stream and returns the results as a :class:`DataFrame`. - - `JSON Lines `_ (newline-delimited JSON) is supported by default. - For JSON (one record per file), set the ``multiLine`` parameter to ``true``. - - If the ``schema`` parameter is not specified, this function goes - through the input once to determine the input schema. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - path : str - string represents path to the JSON dataset, - or RDD of Strings storing JSON objects. - schema : :class:`pyspark.sql.types.StructType` or str, optional - an optional :class:`pyspark.sql.types.StructType` for the input schema - or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - in the version you use. - - .. # noqa - - Notes - ----- - This API is evolving. - - Examples - -------- - Load a data stream from a temporary JSON file. - - >>> import tempfile - >>> import time - >>> with tempfile.TemporaryDirectory(prefix="json") as d: - ... # Write a temporary JSON file to read it. - ... spark.createDataFrame( - ... [(100, "Hyukjin Kwon"),], ["age", "name"] - ... ).write.mode("overwrite").format("json").save(d) - ... - ... # Start a streaming query to read the JSON file. - ... q = spark.readStream.schema( - ... "age INT, name STRING" - ... ).json(d).writeStream.format("console").start() - ... time.sleep(3) - ... q.stop() - """ - def orc(self, path: str, mergeSchema: Optional[bool]=None, pathGlobFilter: Optional[Union[bool, str]]=None, recursiveFileLookup: Optional[Union[bool, str]]=None) -> 'DataFrame': - """Loads a ORC file stream, returning the result as a :class:`DataFrame`. - - .. versionadded:: 2.3.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - in the version you use. - - .. # noqa - - Examples - -------- - Load a data stream from a temporary ORC file. - - >>> import tempfile - >>> import time - >>> with tempfile.TemporaryDirectory(prefix="orc") as d: - ... # Write a temporary ORC file to read it. - ... spark.range(10).write.mode("overwrite").format("orc").save(d) - ... - ... # Start a streaming query to read the ORC file. - ... q = spark.readStream.schema("id LONG").orc(d).writeStream.format("console").start() - ... time.sleep(3) - ... q.stop() - """ - def parquet(self, path: str, mergeSchema: Optional[bool]=None, pathGlobFilter: Optional[Union[bool, str]]=None, recursiveFileLookup: Optional[Union[bool, str]]=None, datetimeRebaseMode: Optional[Union[bool, str]]=None, int96RebaseMode: Optional[Union[bool, str]]=None) -> 'DataFrame': - """Loads a Parquet file stream, returning the result as a :class:`DataFrame`. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - path : str - the path in any Hadoop supported file system - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_. - in the version you use. - - .. # noqa - - Examples - -------- - Load a data stream from a temporary Parquet file. - - >>> import tempfile - >>> import time - >>> with tempfile.TemporaryDirectory(prefix="parquet") as d: - ... # Write a temporary Parquet file to read it. - ... spark.range(10).write.mode("overwrite").format("parquet").save(d) - ... - ... # Start a streaming query to read the Parquet file. - ... q = spark.readStream.schema( - ... "id LONG").parquet(d).writeStream.format("console").start() - ... time.sleep(3) - ... q.stop() - """ - def text(self, path: str, wholetext: bool=False, lineSep: Optional[str]=None, pathGlobFilter: Optional[Union[bool, str]]=None, recursiveFileLookup: Optional[Union[bool, str]]=None) -> 'DataFrame': - """Loads a text file stream and returns a :class:`DataFrame` whose schema starts with a - string column named "value", and followed by partitioned columns if there - are any. - The text files must be encoded as UTF-8. - - By default, each line in the text file is a new row in the resulting DataFrame. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - path : str - string for input path. - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - in the version you use. - - .. # noqa - - Notes - ----- - This API is evolving. - - Examples - -------- - Load a data stream from a temporary text file. - - >>> import tempfile - >>> import time - >>> with tempfile.TemporaryDirectory(prefix="text") as d: - ... # Write a temporary text file to read it. - ... spark.createDataFrame( - ... [("hello",), ("this",)]).write.mode("overwrite").format("text").save(d) - ... - ... # Start a streaming query to read the text file. - ... q = spark.readStream.text(d).writeStream.format("console").start() - ... time.sleep(3) - ... q.stop() - """ - def csv(self, path: str, schema: Optional[Union[StructType, str]]=None, sep: Optional[str]=None, encoding: Optional[str]=None, quote: Optional[str]=None, escape: Optional[str]=None, comment: Optional[str]=None, header: Optional[Union[bool, str]]=None, inferSchema: Optional[Union[bool, str]]=None, ignoreLeadingWhiteSpace: Optional[Union[bool, str]]=None, ignoreTrailingWhiteSpace: Optional[Union[bool, str]]=None, nullValue: Optional[str]=None, nanValue: Optional[str]=None, positiveInf: Optional[str]=None, negativeInf: Optional[str]=None, dateFormat: Optional[str]=None, timestampFormat: Optional[str]=None, maxColumns: Optional[Union[int, str]]=None, maxCharsPerColumn: Optional[Union[int, str]]=None, maxMalformedLogPerPartition: Optional[Union[int, str]]=None, mode: Optional[str]=None, columnNameOfCorruptRecord: Optional[str]=None, multiLine: Optional[Union[bool, str]]=None, charToEscapeQuoteEscaping: Optional[Union[bool, str]]=None, enforceSchema: Optional[Union[bool, str]]=None, emptyValue: Optional[str]=None, locale: Optional[str]=None, lineSep: Optional[str]=None, pathGlobFilter: Optional[Union[bool, str]]=None, recursiveFileLookup: Optional[Union[bool, str]]=None, unescapedQuoteHandling: Optional[str]=None) -> 'DataFrame': - """Loads a CSV file stream and returns the result as a :class:`DataFrame`. - - This function will go through the input once to determine the input schema if - ``inferSchema`` is enabled. To avoid going through the entire data once, disable - ``inferSchema`` option or specify the schema explicitly using ``schema``. - - Parameters - ---------- - path : str - string for input path. - schema : :class:`pyspark.sql.types.StructType` or str, optional - an optional :class:`pyspark.sql.types.StructType` for the input schema - or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - in the version you use. - - .. # noqa - - Notes - ----- - This API is evolving. - - Examples - -------- - Load a data stream from a temporary CSV file. - - >>> import tempfile - >>> import time - >>> with tempfile.TemporaryDirectory(prefix="csv") as d: - ... # Write a temporary text file to read it. - ... spark.createDataFrame([(1, "2"),]).write.mode("overwrite").format("csv").save(d) - ... - ... # Start a streaming query to read the CSV file. - ... q = spark.readStream.schema( - ... "col0 INT, col1 STRING" - ... ).format("csv").load(d).writeStream.format("console").start() - ... time.sleep(3) - ... q.stop() - """ - def xml(self, path: str, rowTag: Optional[str]=None, schema: Optional[Union[StructType, str]]=None, excludeAttribute: Optional[Union[bool, str]]=None, attributePrefix: Optional[str]=None, valueTag: Optional[str]=None, ignoreSurroundingSpaces: Optional[Union[bool, str]]=None, rowValidationXSDPath: Optional[str]=None, ignoreNamespace: Optional[Union[bool, str]]=None, wildcardColName: Optional[str]=None, encoding: Optional[str]=None, inferSchema: Optional[Union[bool, str]]=None, nullValue: Optional[str]=None, dateFormat: Optional[str]=None, timestampFormat: Optional[str]=None, mode: Optional[str]=None, columnNameOfCorruptRecord: Optional[str]=None, multiLine: Optional[Union[bool, str]]=None, samplingRatio: Optional[Union[float, str]]=None, locale: Optional[str]=None) -> 'DataFrame': - """Loads a XML file stream and returns the result as a :class:`DataFrame`. - - If the ``schema`` parameter is not specified, this function goes - through the input once to determine the input schema. - - Parameters - ---------- - path : str - string for input path. - schema : :class:`pyspark.sql.types.StructType` or str, optional - an optional :class:`pyspark.sql.types.StructType` for the input schema - or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). - - .. versionadded:: 4.0.0 - - Other Parameters - ---------------- - Extra options - For the extra options, refer to - `Data Source Option `_ - in the version you use. - - .. # noqa - - Notes - ----- - This API is evolving. - - Examples - -------- - Write a DataFrame into a XML file and read it back. - - >>> import tempfile - >>> import time - >>> with tempfile.TemporaryDirectory(prefix="xml") as d: - ... # Write a DataFrame into a XML file - ... spark.createDataFrame( - ... [{"age": 100, "name": "Hyukjin Kwon"}] - ... ).write.mode("overwrite").option("rowTag", "person").xml(d) - ... - ... # Start a streaming query to read the XML file. - ... q = spark.readStream.schema( - ... "age INT, name STRING" - ... ).xml(d, rowTag="person").writeStream.format("console").start() - ... time.sleep(3) - ... q.stop() - """ - def table(self, tableName: str) -> 'DataFrame': - """Define a Streaming DataFrame on a Table. The DataSource corresponding to the table should - support streaming mode. - - .. versionadded:: 3.1.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - tableName : str - string, for the name of the table. - - Returns - ------- - :class:`DataFrame` - - Notes - ----- - This API is evolving. - - Examples - -------- - Load a data stream from a table. - - >>> import tempfile - >>> import time - >>> _ = spark.sql("DROP TABLE IF EXISTS my_table") - >>> with tempfile.TemporaryDirectory(prefix="table") as d: - ... # Create a table with Rate source. - ... q1 = spark.readStream.format("rate").load().writeStream.toTable( - ... "my_table", checkpointLocation=d) - ... - ... # Read the table back and print out in the console. - ... q2 = spark.readStream.table("my_table").writeStream.format("console").start() - ... time.sleep(3) - ... q1.stop() - ... q2.stop() - ... _ = spark.sql("DROP TABLE my_table") - """ - def changes(self, tableName: str) -> 'DataFrame': - """Returns the row-level changes (Change Data Capture) from the specified table - as a streaming :class:`DataFrame`. Currently this API is only supported for - Data Source V2 tables whose catalog implements ``TableCatalog.loadChangelog()``. - - Use :meth:`option` to specify the starting version/timestamp and processing options. - - .. versionadded:: 4.2.0 - - Parameters - ---------- - tableName : str - string, name of the table. - - Returns - ------- - :class:`DataFrame` - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> spark.readStream.option("startingVersion", "10").changes( - ... "my_table" - ... ) # doctest: +SKIP - """ - -class DataStreamWriter: - """Interface used to write a streaming :class:`DataFrame ` to external - storage systems (e.g. file systems, key-value stores, etc). - Use :attr:`DataFrame.writeStream ` - to access this. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Notes - ----- - This API is evolving. - - Examples - -------- - The example below uses Rate source that generates rows continuously. - After that, we operate a modulo by 3, and then writes the stream out to the console. - The streaming query stops in 3 seconds. - - >>> import time - >>> df = spark.readStream.format("rate").load() - >>> df = df.selectExpr("value % 3 as v") - >>> q = df.writeStream.format("console").start() - >>> time.sleep(3) - >>> q.stop() - """ - def __init__(self, df: 'DataFrame') -> None: - self._df = df - self._spark = df.sparkSession - self._jwrite = df._jdf.writeStream() - def _sq(self, jsq: 'JavaObject') -> StreamingQuery: ... - def outputMode(self, outputMode: str) -> 'DataStreamWriter': - """Specifies how data of a streaming DataFrame/Dataset is written to a streaming sink. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Options include: - - * `append`: Only the new rows in the streaming DataFrame/Dataset will be written to - the sink - * `complete`: All the rows in the streaming DataFrame/Dataset will be written to the sink - every time these are some updates - * `update`: only the rows that were updated in the streaming DataFrame/Dataset will be - written to the sink every time there are some updates. If the query doesn't contain - aggregations, it will be equivalent to `append` mode. - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> df = spark.readStream.format("rate").load() - >>> df.writeStream.outputMode('append') - <...streaming.readwriter.DataStreamWriter object ...> - - The example below uses Complete mode that the entire aggregated counts are printed out. - - >>> import time - >>> df = spark.readStream.format("rate").option("rowsPerSecond", 10).load() - >>> df = df.groupby().count() - >>> q = df.writeStream.outputMode("complete").format("console").start() - >>> time.sleep(3) - >>> q.stop() - """ - if not outputMode or not isinstance(outputMode, str) or len(outputMode.strip()) == 0: - raise PySparkValueError( - errorClass="VALUE_NOT_NON_EMPTY_STR", - messageParameters={"arg_name": "outputMode", "arg_value": str(outputMode)}, - ) - self._jwrite = self._jwrite.outputMode(outputMode) - return self - def format(self, source: str) -> 'DataStreamWriter': - """Specifies the underlying output data source. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - source : str - string, name of the data source, which for now can be 'parquet'. - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> df = spark.readStream.format("rate").load() - >>> df.writeStream.format("text") - <...streaming.readwriter.DataStreamWriter object ...> - - This API allows to configure the source to write. The example below writes a CSV - file from Rate source in a streaming manner. - - >>> import tempfile - >>> import time - >>> with tempfile.TemporaryDirectory(prefix="format1") as d: - ... with tempfile.TemporaryDirectory(prefix="format2") as cp: - ... df = spark.readStream.format("rate").load() - ... q = df.writeStream.format("csv").option("checkpointLocation", cp).start(d) - ... time.sleep(5) - ... q.stop() - ... spark.read.schema("timestamp TIMESTAMP, value STRING").csv(d).show() - +...---------+-----+ - |...timestamp|value| - +...---------+-----+ - ... - """ - self._jwrite = self._jwrite.format(source) - return self - def option(self, key: str, value: 'OptionalPrimitiveType') -> 'DataStreamWriter': - """Adds an output option for the underlying data source. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> df = spark.readStream.format("rate").load() - >>> df.writeStream.option("x", 1) - <...streaming.readwriter.DataStreamWriter object ...> - - The example below specifies 'numRows' option to Console source in order to print - 3 rows for every batch. - - >>> import time - >>> q = spark.readStream.format( - ... "rate").option("rowsPerSecond", 10).load().writeStream.format( - ... "console").option("numRows", 3).start() - >>> time.sleep(3) - >>> q.stop() - """ - if value is None: - return self - self._jwrite = self._jwrite.option(key, to_str(value)) - return self - def options(self, **options: 'OptionalPrimitiveType') -> 'DataStreamWriter': - """Adds output options for the underlying data source. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> df = spark.readStream.format("rate").load() - >>> df.writeStream.option("x", 1) - <...streaming.readwriter.DataStreamWriter object ...> - - Specify options in a dictionary. - - >>> df.writeStream.options(**{"k1": "v1", "k2": "v2"}) - <...streaming.readwriter.DataStreamWriter object ...> - - The example below specifies 'numRows' and 'truncate' options to Console source in order - to print 3 rows for every batch without truncating the results. - - >>> import time - >>> q = spark.readStream.format( - ... "rate").option("rowsPerSecond", 10).load().writeStream.format( - ... "console").options(numRows=3, truncate=False).start() - >>> time.sleep(3) - >>> q.stop() - """ - for k, v in options.items(): - if v is not None: - self._jwrite = self._jwrite.option(k, to_str(v)) - return self - @overload - def partitionBy(self, *cols: str) -> 'DataStreamWriter': - """Partitions the output by the given columns on the file system. - - If specified, the output is laid out on the file system similar - to Hive's partitioning scheme. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - cols : str or list - name of columns - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> df = spark.readStream.format("rate").load() - >>> df.writeStream.partitionBy("value") - <...streaming.readwriter.DataStreamWriter object ...> - - Partition-by timestamp column from Rate source. - - >>> import tempfile - >>> import time - >>> with tempfile.TemporaryDirectory(prefix="partitionBy1") as d: - ... with tempfile.TemporaryDirectory(prefix="partitionBy2") as cp: - ... df = spark.readStream.format("rate").option("rowsPerSecond", 10).load() - ... q = df.writeStream.partitionBy( - ... "timestamp").format("parquet").option("checkpointLocation", cp).start(d) - ... time.sleep(5) - ... q.stop() - ... spark.read.schema(df.schema).parquet(d).show() - +...---------+-----+ - |...timestamp|value| - +...---------+-----+ - ... - """ - @overload - def partitionBy(self, __cols: List[str]) -> 'DataStreamWriter': - """Partitions the output by the given columns on the file system. - - If specified, the output is laid out on the file system similar - to Hive's partitioning scheme. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - cols : str or list - name of columns - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> df = spark.readStream.format("rate").load() - >>> df.writeStream.partitionBy("value") - <...streaming.readwriter.DataStreamWriter object ...> - - Partition-by timestamp column from Rate source. - - >>> import tempfile - >>> import time - >>> with tempfile.TemporaryDirectory(prefix="partitionBy1") as d: - ... with tempfile.TemporaryDirectory(prefix="partitionBy2") as cp: - ... df = spark.readStream.format("rate").option("rowsPerSecond", 10).load() - ... q = df.writeStream.partitionBy( - ... "timestamp").format("parquet").option("checkpointLocation", cp).start(d) - ... time.sleep(5) - ... q.stop() - ... spark.read.schema(df.schema).parquet(d).show() - +...---------+-----+ - |...timestamp|value| - +...---------+-----+ - ... - """ - @overload - def clusterBy(self, *cols: str) -> 'DataStreamWriter': - """Clusters the output by the given columns. - - If specified, the output is laid out such that records with similar values on the clustering - column(s) are grouped together in the same file. - - Clustering improves query efficiency by allowing queries with predicates on the clustering - columns to skip unnecessary data. Unlike partitioning, clustering can be used on very high - cardinality columns. - - .. versionadded:: 4.0.0 - - Parameters - ---------- - cols : str or list - name of columns - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> df = spark.readStream.format("rate").load() - >>> df.writeStream.clusterBy("value") - <...streaming.readwriter.DataStreamWriter object ...> - - Cluster-by timestamp column from Rate source. - - >>> import tempfile - >>> import time - >>> with tempfile.TemporaryDirectory(prefix="partitionBy1") as d: - ... with tempfile.TemporaryDirectory(prefix="partitionBy2") as cp: - ... df = spark.readStream.format("rate").option("rowsPerSecond", 10).load() - ... q = df.writeStream.clusterBy( - ... "timestamp").format("parquet").option("checkpointLocation", cp).start(d) - ... time.sleep(5) - ... q.stop() - ... spark.read.schema(df.schema).parquet(d).show() - +...---------+-----+ - |...timestamp|value| - +...---------+-----+ - ... - """ - @overload - def clusterBy(self, __cols: List[str]) -> 'DataStreamWriter': - """Clusters the output by the given columns. - - If specified, the output is laid out such that records with similar values on the clustering - column(s) are grouped together in the same file. - - Clustering improves query efficiency by allowing queries with predicates on the clustering - columns to skip unnecessary data. Unlike partitioning, clustering can be used on very high - cardinality columns. - - .. versionadded:: 4.0.0 - - Parameters - ---------- - cols : str or list - name of columns - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> df = spark.readStream.format("rate").load() - >>> df.writeStream.clusterBy("value") - <...streaming.readwriter.DataStreamWriter object ...> - - Cluster-by timestamp column from Rate source. - - >>> import tempfile - >>> import time - >>> with tempfile.TemporaryDirectory(prefix="partitionBy1") as d: - ... with tempfile.TemporaryDirectory(prefix="partitionBy2") as cp: - ... df = spark.readStream.format("rate").option("rowsPerSecond", 10).load() - ... q = df.writeStream.clusterBy( - ... "timestamp").format("parquet").option("checkpointLocation", cp).start(d) - ... time.sleep(5) - ... q.stop() - ... spark.read.schema(df.schema).parquet(d).show() - +...---------+-----+ - |...timestamp|value| - +...---------+-----+ - ... - """ - def queryName(self, queryName: str) -> 'DataStreamWriter': - """Specifies the name of the :class:`StreamingQuery` that can be started with - :func:`start`. This name must be unique among all the currently active queries - in the associated SparkSession. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - queryName : str - unique name for the query - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> import time - >>> df = spark.readStream.format("rate").load() - >>> q = df.writeStream.queryName("streaming_query").format("console").start() - >>> q.stop() - >>> q.name - 'streaming_query' - """ - if not queryName or not isinstance(queryName, str) or len(queryName.strip()) == 0: - raise PySparkValueError( - errorClass="VALUE_NOT_NON_EMPTY_STR", - messageParameters={"arg_name": "queryName", "arg_value": str(queryName)}, - ) - self._jwrite = self._jwrite.queryName(queryName) - return self - @overload - def trigger(self, *, processingTime: str) -> 'DataStreamWriter': - """Set the trigger for the stream query. If this is not set it will run the query as fast - as possible, which is equivalent to setting the trigger to ``processingTime='0 seconds'``. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - processingTime : str, optional - a processing time interval as a string, e.g. '5 seconds', '1 minute'. - Set a trigger that runs a microbatch query periodically based on the - processing time. Only one trigger can be set. - once : bool, optional - if set to True, set a trigger that processes only one batch of data in a - streaming query then terminates the query. Only one trigger can be set. - continuous : str, optional - a time interval as a string, e.g. '5 seconds', '1 minute'. - Set a trigger that runs a continuous query with a given checkpoint - interval. Only one trigger can be set. - availableNow : bool, optional - if set to True, set a trigger that processes all available data in multiple - batches then terminates the query. Only one trigger can be set. - realTime : str, optional - a batch duration as a string, e.g. '5 seconds', '1 minute'. - Set a trigger that runs a real time mode query with - batch at the specified duration. Only one trigger can be set. - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> df = spark.readStream.format("rate").load() - - Trigger the query for execution every 5 seconds - - >>> df.writeStream.trigger(processingTime='5 seconds') - <...streaming.readwriter.DataStreamWriter object ...> - - Trigger the query for execution every 5 seconds - - >>> df.writeStream.trigger(continuous='5 seconds') - <...streaming.readwriter.DataStreamWriter object ...> - - Trigger the query for reading all available data with multiple batches - - >>> df.writeStream.trigger(availableNow=True) - <...streaming.readwriter.DataStreamWriter object ...> - - Trigger the query for real time mode execution every 5 seconds. - - >>> df.writeStream.trigger(realTime='5 seconds') - <...streaming.readwriter.DataStreamWriter object ...> - """ - @overload - def trigger(self, *, once: bool) -> 'DataStreamWriter': - """Set the trigger for the stream query. If this is not set it will run the query as fast - as possible, which is equivalent to setting the trigger to ``processingTime='0 seconds'``. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - processingTime : str, optional - a processing time interval as a string, e.g. '5 seconds', '1 minute'. - Set a trigger that runs a microbatch query periodically based on the - processing time. Only one trigger can be set. - once : bool, optional - if set to True, set a trigger that processes only one batch of data in a - streaming query then terminates the query. Only one trigger can be set. - continuous : str, optional - a time interval as a string, e.g. '5 seconds', '1 minute'. - Set a trigger that runs a continuous query with a given checkpoint - interval. Only one trigger can be set. - availableNow : bool, optional - if set to True, set a trigger that processes all available data in multiple - batches then terminates the query. Only one trigger can be set. - realTime : str, optional - a batch duration as a string, e.g. '5 seconds', '1 minute'. - Set a trigger that runs a real time mode query with - batch at the specified duration. Only one trigger can be set. - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> df = spark.readStream.format("rate").load() - - Trigger the query for execution every 5 seconds - - >>> df.writeStream.trigger(processingTime='5 seconds') - <...streaming.readwriter.DataStreamWriter object ...> - - Trigger the query for execution every 5 seconds - - >>> df.writeStream.trigger(continuous='5 seconds') - <...streaming.readwriter.DataStreamWriter object ...> - - Trigger the query for reading all available data with multiple batches - - >>> df.writeStream.trigger(availableNow=True) - <...streaming.readwriter.DataStreamWriter object ...> - - Trigger the query for real time mode execution every 5 seconds. - - >>> df.writeStream.trigger(realTime='5 seconds') - <...streaming.readwriter.DataStreamWriter object ...> - """ - @overload - def trigger(self, *, continuous: str) -> 'DataStreamWriter': - """Set the trigger for the stream query. If this is not set it will run the query as fast - as possible, which is equivalent to setting the trigger to ``processingTime='0 seconds'``. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - processingTime : str, optional - a processing time interval as a string, e.g. '5 seconds', '1 minute'. - Set a trigger that runs a microbatch query periodically based on the - processing time. Only one trigger can be set. - once : bool, optional - if set to True, set a trigger that processes only one batch of data in a - streaming query then terminates the query. Only one trigger can be set. - continuous : str, optional - a time interval as a string, e.g. '5 seconds', '1 minute'. - Set a trigger that runs a continuous query with a given checkpoint - interval. Only one trigger can be set. - availableNow : bool, optional - if set to True, set a trigger that processes all available data in multiple - batches then terminates the query. Only one trigger can be set. - realTime : str, optional - a batch duration as a string, e.g. '5 seconds', '1 minute'. - Set a trigger that runs a real time mode query with - batch at the specified duration. Only one trigger can be set. - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> df = spark.readStream.format("rate").load() - - Trigger the query for execution every 5 seconds - - >>> df.writeStream.trigger(processingTime='5 seconds') - <...streaming.readwriter.DataStreamWriter object ...> - - Trigger the query for execution every 5 seconds - - >>> df.writeStream.trigger(continuous='5 seconds') - <...streaming.readwriter.DataStreamWriter object ...> - - Trigger the query for reading all available data with multiple batches - - >>> df.writeStream.trigger(availableNow=True) - <...streaming.readwriter.DataStreamWriter object ...> - - Trigger the query for real time mode execution every 5 seconds. - - >>> df.writeStream.trigger(realTime='5 seconds') - <...streaming.readwriter.DataStreamWriter object ...> - """ - @overload - def trigger(self, *, availableNow: bool) -> 'DataStreamWriter': - """Set the trigger for the stream query. If this is not set it will run the query as fast - as possible, which is equivalent to setting the trigger to ``processingTime='0 seconds'``. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - processingTime : str, optional - a processing time interval as a string, e.g. '5 seconds', '1 minute'. - Set a trigger that runs a microbatch query periodically based on the - processing time. Only one trigger can be set. - once : bool, optional - if set to True, set a trigger that processes only one batch of data in a - streaming query then terminates the query. Only one trigger can be set. - continuous : str, optional - a time interval as a string, e.g. '5 seconds', '1 minute'. - Set a trigger that runs a continuous query with a given checkpoint - interval. Only one trigger can be set. - availableNow : bool, optional - if set to True, set a trigger that processes all available data in multiple - batches then terminates the query. Only one trigger can be set. - realTime : str, optional - a batch duration as a string, e.g. '5 seconds', '1 minute'. - Set a trigger that runs a real time mode query with - batch at the specified duration. Only one trigger can be set. - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> df = spark.readStream.format("rate").load() - - Trigger the query for execution every 5 seconds - - >>> df.writeStream.trigger(processingTime='5 seconds') - <...streaming.readwriter.DataStreamWriter object ...> - - Trigger the query for execution every 5 seconds - - >>> df.writeStream.trigger(continuous='5 seconds') - <...streaming.readwriter.DataStreamWriter object ...> - - Trigger the query for reading all available data with multiple batches - - >>> df.writeStream.trigger(availableNow=True) - <...streaming.readwriter.DataStreamWriter object ...> - - Trigger the query for real time mode execution every 5 seconds. - - >>> df.writeStream.trigger(realTime='5 seconds') - <...streaming.readwriter.DataStreamWriter object ...> - """ - @staticmethod - def _construct_foreach_function(f: Union[Callable[[Row], None], 'SupportsProcess']) -> Callable[[Any, Iterator], Iterator]: ... - @overload - def foreach(self, f: Callable[[Row], None]) -> 'DataStreamWriter': - """Sets the output of the streaming query to be processed using the provided writer ``f``. - This is often used to write the output of a streaming query to arbitrary storage systems. - The processing logic can be specified in two ways. - - #. A **function** that takes a row as input. - This is a simple way to express your processing logic. Note that this does - not allow you to deduplicate generated data when failures cause reprocessing of - some input data. That would require you to specify the processing logic in the next - way. - - #. An **object** with a ``process`` method and optional ``open`` and ``close`` methods. - The object can have the following methods. - - * ``open(partition_id, epoch_id)``: *Optional* method that initializes the processing - (for example, open a connection, start a transaction, etc). Additionally, you can - use the `partition_id` and `epoch_id` to deduplicate regenerated data - (discussed later). - - * ``process(row)``: *Non-optional* method that processes each :class:`Row`. - - * ``close(error)``: *Optional* method that finalizes and cleans up (for example, - close connection, commit transaction, etc.) after all rows have been processed. - - The object will be used by Spark in the following way. - - * A single copy of this object is responsible of all the data generated by a - single task in a query. In other words, one instance is responsible for - processing one partition of the data generated in a distributed manner. - - * This object must be serializable because each task will get a fresh - serialized-deserialized copy of the provided object. Hence, it is strongly - recommended that any initialization for writing data (e.g. opening a - connection or starting a transaction) is done after the `open(...)` - method has been called, which signifies that the task is ready to generate data. - - * The lifecycle of the methods are as follows. - - For each partition with ``partition_id``: - - ... For each batch/epoch of streaming data with ``epoch_id``: - - ....... Method ``open(partitionId, epochId)`` is called. - - ....... If ``open(...)`` returns true, for each row in the partition and - batch/epoch, method ``process(row)`` is called. - - ....... Method ``close(errorOrNull)`` is called with error (if any) seen while - processing rows. - - Important points to note: - - * The `partitionId` and `epochId` can be used to deduplicate generated data when - failures cause reprocessing of some input data. This depends on the execution - mode of the query. If the streaming query is being executed in the micro-batch - mode, then every partition represented by a unique tuple (partition_id, epoch_id) - is guaranteed to have the same data. Hence, (partition_id, epoch_id) can be used - to deduplicate and/or transactionally commit data and achieve exactly-once - guarantees. However, if the streaming query is being executed in the continuous - mode, then this guarantee does not hold and therefore should not be used for - deduplication. - - * The ``close()`` method (if exists) will be called if `open()` method exists and - returns successfully (irrespective of the return value), except if the Python - crashes in the middle. - - .. versionadded:: 2.4.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> import time - >>> df = spark.readStream.format("rate").load() - - Print every row using a function - - >>> def print_row(row): - ... print(row) - ... - >>> q = df.writeStream.foreach(print_row).start() - >>> time.sleep(3) - >>> q.stop() - - Print every row using a object with process() method - - >>> class RowPrinter: - ... def open(self, partition_id, epoch_id): - ... print("Opened %d, %d" % (partition_id, epoch_id)) - ... return True - ... - ... def process(self, row): - ... print(row) - ... - ... def close(self, error): - ... print("Closed with error: %s" % str(error)) - ... - >>> q = df.writeStream.foreach(RowPrinter()).start() - >>> time.sleep(3) - >>> q.stop() - """ - @overload - def foreach(self, f: 'SupportsProcess') -> 'DataStreamWriter': - """Sets the output of the streaming query to be processed using the provided writer ``f``. - This is often used to write the output of a streaming query to arbitrary storage systems. - The processing logic can be specified in two ways. - - #. A **function** that takes a row as input. - This is a simple way to express your processing logic. Note that this does - not allow you to deduplicate generated data when failures cause reprocessing of - some input data. That would require you to specify the processing logic in the next - way. - - #. An **object** with a ``process`` method and optional ``open`` and ``close`` methods. - The object can have the following methods. - - * ``open(partition_id, epoch_id)``: *Optional* method that initializes the processing - (for example, open a connection, start a transaction, etc). Additionally, you can - use the `partition_id` and `epoch_id` to deduplicate regenerated data - (discussed later). - - * ``process(row)``: *Non-optional* method that processes each :class:`Row`. - - * ``close(error)``: *Optional* method that finalizes and cleans up (for example, - close connection, commit transaction, etc.) after all rows have been processed. - - The object will be used by Spark in the following way. - - * A single copy of this object is responsible of all the data generated by a - single task in a query. In other words, one instance is responsible for - processing one partition of the data generated in a distributed manner. - - * This object must be serializable because each task will get a fresh - serialized-deserialized copy of the provided object. Hence, it is strongly - recommended that any initialization for writing data (e.g. opening a - connection or starting a transaction) is done after the `open(...)` - method has been called, which signifies that the task is ready to generate data. - - * The lifecycle of the methods are as follows. - - For each partition with ``partition_id``: - - ... For each batch/epoch of streaming data with ``epoch_id``: - - ....... Method ``open(partitionId, epochId)`` is called. - - ....... If ``open(...)`` returns true, for each row in the partition and - batch/epoch, method ``process(row)`` is called. - - ....... Method ``close(errorOrNull)`` is called with error (if any) seen while - processing rows. - - Important points to note: - - * The `partitionId` and `epochId` can be used to deduplicate generated data when - failures cause reprocessing of some input data. This depends on the execution - mode of the query. If the streaming query is being executed in the micro-batch - mode, then every partition represented by a unique tuple (partition_id, epoch_id) - is guaranteed to have the same data. Hence, (partition_id, epoch_id) can be used - to deduplicate and/or transactionally commit data and achieve exactly-once - guarantees. However, if the streaming query is being executed in the continuous - mode, then this guarantee does not hold and therefore should not be used for - deduplication. - - * The ``close()`` method (if exists) will be called if `open()` method exists and - returns successfully (irrespective of the return value), except if the Python - crashes in the middle. - - .. versionadded:: 2.4.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> import time - >>> df = spark.readStream.format("rate").load() - - Print every row using a function - - >>> def print_row(row): - ... print(row) - ... - >>> q = df.writeStream.foreach(print_row).start() - >>> time.sleep(3) - >>> q.stop() - - Print every row using a object with process() method - - >>> class RowPrinter: - ... def open(self, partition_id, epoch_id): - ... print("Opened %d, %d" % (partition_id, epoch_id)) - ... return True - ... - ... def process(self, row): - ... print(row) - ... - ... def close(self, error): - ... print("Closed with error: %s" % str(error)) - ... - >>> q = df.writeStream.foreach(RowPrinter()).start() - >>> time.sleep(3) - >>> q.stop() - """ - def foreachBatch(self, func: Callable[['DataFrame', int], None]) -> 'DataStreamWriter': - """Sets the output of the streaming query to be processed using the provided - function. This is supported only the in the micro-batch execution modes (that is, when the - trigger is not continuous). In every micro-batch, the provided function will be called in - every micro-batch with (i) the output rows as a DataFrame and (ii) the batch identifier. - The batchId can be used deduplicate and transactionally write the output - (that is, the provided Dataset) to external systems. The output DataFrame is guaranteed - to exactly same for the same batchId (assuming all operations are deterministic in the - query). - - .. versionadded:: 2.4.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Notes - ----- - This API is evolving. - This function behaves differently in Spark Connect mode. See examples. - In Connect, the provided function doesn't have access to variables defined outside of it. - - Examples - -------- - >>> import time - >>> df = spark.readStream.format("rate").load() - >>> my_value = -1 - >>> def func(batch_df, batch_id): - ... global my_value - ... my_value = 100 - ... batch_df.collect() - ... - >>> q = df.writeStream.foreachBatch(func).start() - >>> time.sleep(3) - >>> q.stop() - >>> # if in Spark Connect, my_value = -1, else my_value = 100 - """ - def start(self, path: Optional[str]=None, format: Optional[str]=None, outputMode: Optional[str]=None, partitionBy: Optional[Union[str, List[str]]]=None, queryName: Optional[str]=None, **options: 'OptionalPrimitiveType') -> 'StreamingQuery': - """Streams the contents of the :class:`DataFrame` to a data source. - - The data source is specified by the ``format`` and a set of ``options``. - If ``format`` is not specified, the default data source configured by - ``spark.sql.sources.default`` will be used. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - path : str, optional - the path in a Hadoop supported file system - format : str, optional - the format used to save - outputMode : str, optional - specifies how data of a streaming DataFrame/Dataset is written to a - streaming sink. - - * `append`: Only the new rows in the streaming DataFrame/Dataset will be written to the - sink - * `complete`: All the rows in the streaming DataFrame/Dataset will be written to the - sink every time these are some updates - * `update`: only the rows that were updated in the streaming DataFrame/Dataset will be - written to the sink every time there are some updates. If the query doesn't contain - aggregations, it will be equivalent to `append` mode. - partitionBy : str or list, optional - names of partitioning columns - queryName : str, optional - unique name for the query - **options : dict - All other string options. You may want to provide a `checkpointLocation` - for most streams, however it is not required for a `memory` stream. - - Notes - ----- - This API is evolving. - - Examples - -------- - >>> df = spark.readStream.format("rate").load() - - Basic example. - - >>> q = df.writeStream.format('memory').queryName('this_query').start() - >>> q.isActive - True - >>> q.name - 'this_query' - >>> q.stop() - >>> q.isActive - False - - Example with using other parameters with a trigger. - - >>> q = df.writeStream.trigger(processingTime='5 seconds').start( - ... queryName='that_query', outputMode="append", format='memory') - >>> q.name - 'that_query' - >>> q.isActive - True - >>> q.stop() - """ - def toTable(self, tableName: str, format: Optional[str]=None, outputMode: Optional[str]=None, partitionBy: Optional[Union[str, List[str]]]=None, queryName: Optional[str]=None, **options: 'OptionalPrimitiveType') -> 'StreamingQuery': - """Starts the execution of the streaming query, which will continually output results to the - given table as new data arrives. - - The returned :class:`StreamingQuery` object can be used to interact with the stream. - - .. versionadded:: 3.1.0 - - .. versionchanged:: 3.5.0 - Supports Spark Connect. - - Parameters - ---------- - tableName : str - string, for the name of the table. - format : str, optional - the format used to save. - outputMode : str, optional - specifies how data of a streaming DataFrame/Dataset is written to a - streaming sink. - - * `append`: Only the new rows in the streaming DataFrame/Dataset will be written to the - sink - * `complete`: All the rows in the streaming DataFrame/Dataset will be written to the - sink every time these are some updates - * `update`: only the rows that were updated in the streaming DataFrame/Dataset will be - written to the sink every time there are some updates. If the query doesn't contain - aggregations, it will be equivalent to `append` mode. - partitionBy : str or list, optional - names of partitioning columns - queryName : str, optional - unique name for the query - **options : dict - All other string options. You may want to provide a `checkpointLocation`. - - Notes - ----- - This API is evolving. - - For v1 table, partitioning columns provided by `partitionBy` will be respected no matter - the table exists or not. A new table will be created if the table not exists. - - For v2 table, `partitionBy` will be ignored if the table already exists. `partitionBy` will - be respected only if the v2 table does not exist. Besides, the v2 table created by this API - lacks some functionalities (e.g., customized properties, options, and serde info). If you - need them, please create the v2 table manually before the execution to avoid creating a - table with incomplete information. - - Examples - -------- - Save a data stream to a table. - - >>> import tempfile - >>> import time - >>> _ = spark.sql("DROP TABLE IF EXISTS my_table2") - >>> with tempfile.TemporaryDirectory(prefix="toTable") as d: - ... # Create a table with Rate source. - ... q = spark.readStream.format("rate").option( - ... "rowsPerSecond", 10).load().writeStream.toTable( - ... "my_table2", - ... queryName='that_query', - ... outputMode="append", - ... format='parquet', - ... checkpointLocation=d) - ... time.sleep(3) - ... q.stop() - ... spark.read.table("my_table2").show() - ... _ = spark.sql("DROP TABLE my_table2") - +...---------+-----+ - |...timestamp|value| - +...---------+-----+ - ... - """ - -# Names of the drop-in's module that PySpark's does not have. -class Trigger: - @staticmethod - def processingTime(interval: str) -> Trigger: ... - @staticmethod - def once() -> Trigger: ... - @staticmethod - def availableNow() -> Trigger: ... - @staticmethod - def continuous(interval: str) -> Trigger: ... diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py index 9ad10cb8..f535efd3 100644 --- a/python/pyspark/sql/udf.py +++ b/python/pyspark/sql/udf.py @@ -14,45 +14,142 @@ # See the License for the specific language governing permissions and # limitations under the License. # - """ -User-defined function (UDF) support for Spark Connect client. -Mirrors pyspark.sql.connect.udf and pyspark.sql.functions.udf/pandas_udf. +User-defined function related classes and functions """ import functools +import inspect import sys import warnings -from typing import Callable, Optional, Any +from typing import Callable, Any, TYPE_CHECKING, Optional, cast, Union + -from pyspark.errors import PySparkNotImplementedError, PySparkRuntimeError, PySparkTypeError -from pyspark.sql.pandas.types import to_arrow_type -from pyspark.sql.types import DataType, StringType, StructType -# As PySpark's module, of whose namespace it is (see the stubs). -from pyspark.serializers import CloudPickleSerializer # noqa: F401 from pyspark.util import PythonEvalType +from pyspark.sql.column import Column +from pyspark.sql.types import ( + DataType, + StringType, + StructType, + _parse_datatype_string, +) +from pyspark.sql.utils import get_active_spark_context +from pyspark.sql.pandas.types import to_arrow_type +from pyspark.sql.pandas.utils import require_minimum_pandas_version, require_minimum_pyarrow_version +from pyspark.errors import PySparkTypeError, PySparkNotImplementedError, PySparkRuntimeError + +if TYPE_CHECKING: + from py4j.java_gateway import JavaObject + from pyspark.core.context import SparkContext + from pyspark.sql._typing import DataTypeOrString, ColumnOrName, UserDefinedFunctionLike + from pyspark.sql.session import SparkSession __all__ = ["UDFRegistration"] -def _as_concrete_datatype(returnType: Any) -> DataType: - """Return a concrete, picklable pyspark ``DataType``. +def _wrap_function( + sc: "SparkContext", func: Callable[..., Any], returnType: Optional[DataType] = None +) -> "JavaObject": + from pyspark.core.rdd import _prepare_for_python_RDD - A DDL string (e.g. ``"double"``, ``"a int, b string"``) is parsed into its - concrete type with ``DataType.fromDDL``. This matters because the cloudpickled UDF - command must carry a real ``DataType`` (not a DDL string): the pandas/Arrow - worker pairs each result column with its spark type to build the Arrow batch, - so a bare string breaks it ("not enough values to unpack"). Mirrors how the - reference Connect client pickles the *parsed* output type. - """ - if isinstance(returnType, str): - return DataType.fromDDL(returnType) - return returnType + command: Any + if returnType is None: + command = func + else: + command = (func, returnType) + pickled_command, broadcast_vars, env, includes = _prepare_for_python_RDD(sc, command) + assert sc._jvm is not None + return sc._jvm.SimplePythonFunction( + bytearray(pickled_command), + env, + includes, + sc.pythonExec, + sc.pythonVer, + broadcast_vars, + sc._javaAccumulator, + ) + + +def _create_udf( + f: Callable[..., Any], + returnType: "DataTypeOrString", + evalType: int, + name: Optional[str] = None, + deterministic: bool = True, +) -> "UserDefinedFunctionLike": + """Create a regular(non-Arrow-optimized) Python UDF.""" + # Set the name of the UserDefinedFunction object to be the name of function f + udf_obj = UserDefinedFunction( + f, returnType=returnType, name=name, evalType=evalType, deterministic=deterministic + ) + return udf_obj._wrapped() + + +def _create_py_udf( + f: Callable[..., Any], + returnType: "DataTypeOrString", + useArrow: Optional[bool] = None, +) -> "UserDefinedFunctionLike": + """Create a regular/Arrow-optimized Python UDF.""" + # The tables in python/pyspark/sql/tests/udf_type_tests show the results when the type coercion + # in Arrow is needed, that is, when the user-specified return type(SQL Type) of the UDF and the + # actual instance(Python Value(Type)) that the UDF returns are different. + # Arrow and Pickle have different type coercion rules, so a UDF might have a different result + # with/without Arrow optimization. That's the main reason the Arrow optimization for Python + # UDFs is disabled by default. + + is_arrow_enabled = False + if useArrow is None: + from pyspark.sql import SparkSession + + session = SparkSession._instantiatedSession + is_arrow_enabled = ( + False + if session is None + else session.conf.get("spark.sql.execution.pythonUDF.arrow.enabled") == "true" + ) + else: + is_arrow_enabled = useArrow + + if is_arrow_enabled: + try: + require_minimum_pandas_version() + require_minimum_pyarrow_version() + except ImportError: + is_arrow_enabled = False + warnings.warn( + "Arrow optimization failed to enable because PyArrow or Pandas is not installed. " + "Falling back to a non-Arrow-optimized UDF.", + RuntimeWarning, + ) + + eval_type: Optional[int] = None + if useArrow is None: + # If the user doesn't explicitly set useArrow + from pyspark.sql.pandas.typehints import infer_eval_type_for_udf + + try: + # Try to infer the eval type from type hints + eval_type = infer_eval_type_for_udf(f) + except Exception: + warnings.warn("Cannot infer the eval type from type hints. ", UserWarning) + + if eval_type is None: + if is_arrow_enabled: + # Arrow optimized Python UDF + eval_type = PythonEvalType.SQL_ARROW_BATCHED_UDF + else: + # Fallback to Regular Python UDF + eval_type = PythonEvalType.SQL_BATCHED_UDF + + return _create_udf(f, returnType, eval_type) class UserDefinedFunction: """ - User defined function in Python. Mirrors ``pyspark.sql.connect.udf.UserDefinedFunction``. + User defined function in Python + + .. versionadded:: 1.3 Notes ----- @@ -61,24 +158,14 @@ class UserDefinedFunction: to create this instance. """ - # As PySpark Connect's: the class is that of pyspark.sql.connect.udf. A UDF that a function - # captures is pickled by reference to that name, which the Python worker of the server has - # as the Connect class, not as the classic class of pyspark.sql.udf, which needs a - # SparkContext to be called. The name of pyspark.sql.udf stays an alias of this class. - __module__ = "pyspark.sql.connect.udf" - def __init__( self, func: Callable[..., Any], - returnType: Any = StringType(), + returnType: "DataTypeOrString" = StringType(), name: Optional[str] = None, evalType: int = PythonEvalType.SQL_BATCHED_UDF, deterministic: bool = True, ): - # cloudpickle pickles this class by reference only when its module is imported. - import pyspark.sql.connect.udf # noqa: F401 - - # The checks of PySpark Connect's UserDefinedFunction. if not callable(func): raise PySparkTypeError( errorClass="NOT_EXPECTED_TYPE", @@ -111,25 +198,114 @@ def __init__( self.func = func self._returnType = returnType + # Stores UserDefinedPythonFunctions jobj, once initialized self._returnType_placeholder: Optional[DataType] = None + self._judf_placeholder = None self._name = name or ( func.__name__ if hasattr(func, "__name__") else func.__class__.__name__ ) self.evalType = evalType self.deterministic = deterministic + # Extract Python UDF details if transpilation is enabled. + self.transpiled: list = [] + self._transpiled_param_names: list[str] = [] + # The subset of the names above that Python forbids calling by keyword. + # The kwargs rewrite in ``__call__`` must leave these alone rather than + # silently coerce them to positional -- see the note there. + self._positional_only_param_names: frozenset = frozenset() + # Per-option input-type categories ("numeric"/"string" per public param), + # parallel to ``self.transpiled``; the JVM picks the option matching the + # actual column types or falls back to interpreted Python. + self._transpiled_input_categories: list = [] + # When we have a transpiled rewrite, ``__call__`` resolves any + # user-supplied kwargs against this positional parameter list so + # the JVM-side ``_udf_param_N`` substitution sees the inputs in + # the right order. Empty list when transpilation didn't happen. + from pyspark.sql import SparkSession + + session = SparkSession._instantiatedSession + + # A nondeterministic UDF must not be transpiled: replacing it with a plain + # Catalyst expression would let the optimizer fold/reorder/duplicate it, + # discarding the nondeterminism barrier. (asNondeterministic() also clears + # any options set here, for the udf(f).asNondeterministic() ordering.) + # Conf values are compared case-insensitively: `SET conf=True` stores + # the literal "True", which would otherwise silently disable + # transpilation (or mis-trigger the ANSI warning below). + # + # Each conf read is a JVM roundtrip, so keep the default construction + # path cheap: the experimental gate is only read for deterministic + # batched UDFs (the only shape we transpile), and the ANSI conf is only + # read once the gate is known to be on. When ``default`` is given it is + # passed through to ``RuntimeConfig.get`` so construction never depends + # on the JVM having the (experimental) conf registered -- e.g. a newer + # Python client against an older driver. No default is passed for + # ``spark.sql.ansi.enabled``: its registered default is dynamic + # (environment-driven) and must be respected when the key is unset. + def _conf_is_true(key: str, default: Optional[str] = None) -> bool: + if session is None: + return False + if default is None: + value = session.conf.get(key) + else: + value = session.conf.get(key, default) + return value is not None and value.lower() == "true" - @property - def returnType(self) -> DataType: - # As PySpark's, the return type is parsed once, and checked for the eval type each - # time. It is a concrete DataType, which the pickled command carries for the pandas - # and Arrow workers; a DDL string is parsed on the client. - if self._returnType_placeholder is None: - self._returnType_placeholder = _as_concrete_datatype(self._returnType) - - UserDefinedFunction._check_return_type(self._returnType_placeholder, self.evalType) - return self._returnType_placeholder + try: + transpile_enabled = ( + deterministic + and evalType == PythonEvalType.SQL_BATCHED_UDF + and _conf_is_true("spark.sql.experimental.optimizer.transpilePyUDFs", "false") + ) + # Transpilation only attempts to reproduce ANSI-mode Spark SQL + # semantics (no silent integer overflow, divide-by-zero raises, + # etc.). Running it against non-ANSI Spark would balloon the test + # matrix we'd have to maintain to verify Python-vs-SQL equivalence, + # so we gate on ANSI here and warn the user instead of trying to + # transpile in a mode we don't claim to support yet. + if transpile_enabled and not _conf_is_true("spark.sql.ansi.enabled"): + warnings.warn( + "Python UDF transpilation " + "(spark.sql.experimental.optimizer.transpilePyUDFs) is only " + "supported when ANSI mode is enabled " + "(spark.sql.ansi.enabled=true). Skipping transpilation for " + f"{func} -- enable ANSI mode or set transpilePyUDFs=false to " + "silence this warning.", + RuntimeWarning, + ) + transpile_enabled = False + if transpile_enabled and session: + # Import only if needed, also avoid circular import loops. + from pyspark.sql.transpile import _transpile_func + + # ``self.returnType`` parses (and caches) the declared return + # type; the transpiler needs the parsed form to decide whether + # the final Cast to it can resolve at all. The parse is reused + # later by ``_create_judf``, so this adds no extra JVM work. + ( + self.transpiled, + errors, + self._transpiled_param_names, + self._transpiled_input_categories, + positional_only, + ) = _transpile_func(session, func, self.returnType) + self._positional_only_param_names = frozenset(positional_only) + if not self.transpiled: + detail = f": {errors}" if errors else "" + warnings.warn(f"Unable to transpile UDF {func}{detail}") + except Exception as e: + # An inability to transpile must never break a working UDF -- fall + # back to interpreted Python execution and surface the failure as a + # warning so users can opt to investigate without losing their + # query. The conf reads above are included: a session whose JVM + # cannot answer them should degrade to "no transpilation", not + # break UDF definition. + warnings.warn(f"Exception transpiling UDF {func}: {e}") + self.transpiled = [] + self._transpiled_param_names = [] + self._transpiled_input_categories = [] + self._positional_only_param_names = frozenset() - # Verbatim from PySpark (pyspark.sql.udf.UserDefinedFunction). @staticmethod def _check_return_type(returnType: DataType, evalType: int) -> None: if evalType == PythonEvalType.SQL_ARROW_BATCHED_UDF: @@ -311,42 +487,181 @@ def _check_return_type(returnType: DataType, evalType: int) -> None: }, ) - def __call__(self, *args: Any, **kwargs: Any) -> Any: - """ - Call the UDF with Column arguments. - Returns a Column representing the UDF call. - """ - from pyspark import _pyspark - - # A UDF argument is ColumnOrName: as PySpark's, a bare string is a column name, of - # which pyfunc_make_udf makes a ColumnReference, and each keyword argument is a - # NamedArgumentExpression, after the positional ones. - cols = list(args) - named = list(kwargs.items()) - - # As PySpark's PythonUDF, which keeps the function and the return type object: each - # request that builds the plan converts the output type of that object and pickles - # (func, returnType) for the server-side Python worker with CloudPickleSerializer, as - # PySpark's `PythonUDF.to_plan` does. So the function is pickled as it is then, with - # the globals that it reads, and a StructType changed in place after the call is the - # type that is sent, and pickled, as it is then; a function that does not pickle raises - # the PicklingError of `CloudPickleSerializer.dumps` from the request. - return _pyspark.functions.pyfunc_make_udf( + @property + def returnType(self) -> DataType: + # Make sure this is called after SparkContext is initialized. + # ``_parse_datatype_string`` accesses to JVM for parsing a DDL formatted string. + if self._returnType_placeholder is None: + if isinstance(self._returnType, DataType): + self._returnType_placeholder = self._returnType + else: + self._returnType_placeholder = _parse_datatype_string(self._returnType) + + UserDefinedFunction._check_return_type(self._returnType_placeholder, self.evalType) + return self._returnType_placeholder + + @property + def _judf(self) -> "JavaObject": + # It is possible that concurrent access, to newly created UDF, + # will initialize multiple UserDefinedPythonFunctions. + # This is unlikely, doesn't affect correctness, + # and should have a minimal performance impact. + if self._judf_placeholder is None: + self._judf_placeholder = self._create_judf(self.func) + return self._judf_placeholder + + def _create_judf( + self, func: Callable[..., Any], include_transpiled: bool = True + ) -> "JavaObject": + from pyspark.sql import SparkSession + from pyspark.sql.classic.column import _to_java_column_opt + + spark = SparkSession._getActiveSessionOrCreate() + sc = spark.sparkContext + + wrapped_func = _wrap_function(sc, func, self.returnType) + jdt = spark._jsparkSession.parseDataType(self.returnType.json()) + assert sc._jvm is not None + transpiled = self.transpiled if include_transpiled else [] + input_categories = self._transpiled_input_categories if include_transpiled else [] + judf = getattr(sc._jvm, "org.apache.spark.sql.execution.python.UserDefinedPythonFunction")( self._name, - self.returnType, + wrapped_func, + jdt, self.evalType, - self.func, - "%d.%d" % sys.version_info[:2], - *cols, - deterministic=self.deterministic, - named_args=named, + self.deterministic, + map(_to_java_column_opt, transpiled), + input_categories, ) + return judf + + def __call__(self, *args: "ColumnOrName", **kwargs: "ColumnOrName") -> Column: + from pyspark.sql.classic.column import _to_java_column, _to_seq + + sc = get_active_spark_context() + + # Transpilation rewrites the UDF into a Catalyst expression that + # references its inputs positionally via ``_udf_param_N`` (see + # ``UserDefinedPythonFunction.builder.resolveUDFParams``). If the + # caller used kwargs, the JVM-side substitution would otherwise + # splice ``NamedArgumentExpression`` wrappers into the rewritten + # tree (and into nested function calls like ``isnotnull``, which + # rejects named arguments). Resolve kwargs to positional here + # using the parameter list captured at transpilation time so the + # rewritten expression sees plain column refs in declared order. + # + # A positional-only param is never resolved here -- Python itself + # rejects calling one by keyword, and rewriting would paper over that. + # Left unresolved, its kwarg reaches the JVM as a + # ``NamedArgumentExpression``, which drops the transpiled expression and + # falls back to interpreted Python, so the worker's own keyword call + # raises the same ``TypeError`` Python would. + if kwargs and self.transpiled and self._transpiled_param_names: + params = self._transpiled_param_names + ordered: list = list(args) + remaining_kwargs = dict(kwargs) + for pname in params[len(args) :]: + if pname in remaining_kwargs and pname not in self._positional_only_param_names: + ordered.append(remaining_kwargs.pop(pname)) + else: + # Caller didn't supply this param positionally or by + # name -- bail out of the rewrite and let the regular + # JVM-side path raise a user-facing error. + break + else: + if not remaining_kwargs: + args = tuple(ordered) + kwargs = {} + + assert sc._jvm is not None + jcols = [_to_java_column(arg) for arg in args] + [ + sc._jvm.PythonSQLUtils.namedArgumentExpression(key, _to_java_column(value)) + for key, value in kwargs.items() + ] + + profiler_enabled = sc._conf.get("spark.python.profile", "false") == "true" + memory_profiler_enabled = sc._conf.get("spark.python.profile.memory", "false") == "true" + + if profiler_enabled or memory_profiler_enabled: + # Disable profiling Pandas UDFs with iterators as input/output. + if self.evalType in [ + PythonEvalType.SQL_SCALAR_PANDAS_ITER_UDF, + PythonEvalType.SQL_SCALAR_ARROW_ITER_UDF, + PythonEvalType.SQL_MAP_PANDAS_ITER_UDF, + PythonEvalType.SQL_MAP_ARROW_ITER_UDF, + PythonEvalType.SQL_GROUPED_AGG_ARROW_ITER_UDF, + PythonEvalType.SQL_GROUPED_AGG_PANDAS_ITER_UDF, + ]: + warnings.warn( + "Profiling UDFs with iterators input/output is not supported.", + UserWarning, + ) + judf = self._judf + return Column(judf.apply(_to_seq(sc, jcols))) + + # Disallow enabling two profilers at the same time. + if profiler_enabled and memory_profiler_enabled: + # When both profilers are enabled, they interfere with each other, + # that makes the result profile misleading. + raise PySparkRuntimeError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={ + "arg_list": "'spark.python.profile' and " + "'spark.python.profile.memory' configuration" + }, + ) + elif profiler_enabled: + f = self.func + profiler = sc.profiler_collector.new_udf_profiler(sc) + + @functools.wraps(f) + def func(*args: Any, **kwargs: Any) -> Any: + assert profiler is not None + return profiler.profile(f, *args, **kwargs) + + func.__signature__ = inspect.signature(f) # type: ignore[attr-defined] + # Profiling requires the Python function to actually execute, + # and the transpiled path never runs it (it also produces a + # TranspiledPythonUDF, which has no resultId for the profiler + # to key on). Build this call's judf without transpiled options. + judf = self._create_judf(func, include_transpiled=False) + jUDFExpr = judf.builderWithColumns(_to_seq(sc, jcols)) + jPythonUDF = judf.fromUDFExpr(jUDFExpr) + id = jUDFExpr.resultId().id() + sc.profiler_collector.add_profiler(id, profiler) + else: # memory_profiler_enabled + f = self.func + memory_profiler = sc.profiler_collector.new_memory_profiler(sc) + sub_lines, start_line = inspect.getsourcelines(f.__code__) + + @functools.wraps(f) + def func(*args: Any, **kwargs: Any) -> Any: + assert memory_profiler is not None + return memory_profiler.profile( + sub_lines, # type: ignore[arg-type] + start_line, + f, + *args, + **kwargs, + ) + + func.__signature__ = inspect.signature(f) # type: ignore[attr-defined] + # See the profiler branch above: no transpiled options while + # profiling, since only the interpreted path runs the function. + judf = self._create_judf(func, include_transpiled=False) + jUDFExpr = judf.builderWithColumns(_to_seq(sc, jcols)) + jPythonUDF = judf.fromUDFExpr(jUDFExpr) + id = jUDFExpr.resultId().id() + sc.profiler_collector.add_profiler(id, memory_profiler) + else: + judf = self._judf + jPythonUDF = judf.apply(_to_seq(sc, jcols)) + return Column(jPythonUDF) - # Verbatim from PySpark Connect (pyspark.sql.connect.udf.UserDefinedFunction). # This function is for improving the online help system in the interactive interpreter. # For example, the built-in help / pydoc.help. It wraps the UDF with the docstring and # argument annotation. (See: SPARK-19161) - def _wrapped(self) -> Any: + def _wrapped(self) -> "UserDefinedFunctionLike": """ Wrap this udf with a function and attach docstring from func """ @@ -361,7 +676,7 @@ def _wrapped(self) -> Any: ) @functools.wraps(self.func, assigned=assignments) - def wrapper(*args: Any, **kwargs: Any) -> Any: + def wrapper(*args: "ColumnOrName", **kwargs: "ColumnOrName") -> Column: return self(*args, **kwargs) wrapper.__name__ = self._name @@ -385,189 +700,149 @@ def asNondeterministic(self) -> "UserDefinedFunction": """ Updates UserDefinedFunction to nondeterministic. - .. versionadded:: 3.4.0 + .. versionadded:: 2.3 """ + # Here, we explicitly clean the cache to create a JVM UDF instance + # with 'deterministic' updated. See SPARK-23233. + self._judf_placeholder = None self.deterministic = False + # A transpiled rewrite replaces the (now nondeterministic) Python UDF + # with a plain Catalyst expression, which the optimizer is free to + # fold, reorder, or duplicate -- discarding the nondeterminism barrier + # the caller just asked for. Drop any transpiled options so a + # nondeterministic UDF always runs as interpreted Python. + self.transpiled = [] + self._transpiled_param_names = [] + self._transpiled_input_categories = [] + self._positional_only_param_names = frozenset() return self -def udf( - f: Optional[Any] = None, - returnType: Any = StringType(), - *, - useArrow: Optional[bool] = None, -) -> Any: - """ - Create a user-defined function (UDF). - - Parameters - ---------- - f : callable, optional - The Python function to wrap as a UDF. - returnType : DataType, optional - The return type of the UDF. Defaults to StringType(). - useArrow : bool, optional - Whether to use Arrow optimization. Defaults to None (auto-detect). - - Returns - ------- - function or callable - If f is provided, returns the function wrapper of the UDF, as PySpark does. If f is - None or a return type, returns a decorator. - - Examples - -------- - >>> from pyspark.sql import functions as F - >>> from pyspark.sql.types import IntegerType - >>> u = F.udf(lambda x: x + 1, IntegerType()) - >>> result = spark.range(3).select(u(F.col('id')).alias('inc_id')) - """ - # Verbatim from PySpark Connect (pyspark.sql.connect.functions.builtin.udf). - if f is None or isinstance(f, (str, DataType)): - # If DataType has been passed as a positional argument - # for decorator use it as a returnType - return_type = f or returnType - return functools.partial( - _create_py_udf, - returnType=return_type, - useArrow=useArrow, - ) - else: - return _create_py_udf(f=f, returnType=returnType, useArrow=useArrow) - - -# Verbatim from PySpark Connect (pyspark.sql.connect.udf). -def _create_py_udf( - f: Callable[..., Any], - returnType: Any, - useArrow: Optional[bool] = None, -) -> Any: - from pyspark.sql.pandas.utils import ( - require_minimum_pandas_version, - require_minimum_pyarrow_version, - ) - - is_arrow_enabled = False - if useArrow is None: - is_arrow_enabled = False - try: - from pyspark.sql import SparkSession - - session = SparkSession.active() - is_arrow_enabled = ( - str(session.conf.get("spark.sql.execution.pythonUDF.arrow.enabled")).lower() - == "true" - ) - except PySparkRuntimeError as e: - if e.getCondition() == "NO_ACTIVE_OR_DEFAULT_SESSION": - pass # Just uses the default if no session found. - else: - raise e - else: - is_arrow_enabled = useArrow - - if is_arrow_enabled: - try: - require_minimum_pandas_version() - require_minimum_pyarrow_version() - except ImportError: - is_arrow_enabled = False - warnings.warn( - "Arrow optimization failed to enable because PyArrow or Pandas is not installed. " - "Falling back to a non-Arrow-optimized UDF.", - RuntimeWarning, - ) - - eval_type: Optional[int] = None - if useArrow is None: - # If the user doesn't explicitly set useArrow - from pyspark.sql.pandas.typehints import infer_eval_type_for_udf - - try: - # Try to infer the eval type from type hints - eval_type = infer_eval_type_for_udf(f) - except Exception: - warnings.warn("Cannot infer the eval type from type hints. ", UserWarning) - - if eval_type is None: - if is_arrow_enabled: - # Arrow optimized Python UDF - eval_type = PythonEvalType.SQL_ARROW_BATCHED_UDF - else: - # Fallback to Regular Python UDF - eval_type = PythonEvalType.SQL_BATCHED_UDF - - return _create_udf(f, returnType, eval_type) - - -# Verbatim from PySpark Connect (pyspark.sql.connect.udf); pyspark.sql.pandas.functions -# creates pandas and Arrow UDFs with it. -def _create_udf( - f: Callable[..., Any], - returnType: Any, - evalType: int, - name: Optional[str] = None, - deterministic: bool = True, -) -> Any: - # Set the name of the UserDefinedFunction object to be the name of function f - udf_obj = UserDefinedFunction( - f, returnType=returnType, name=name, evalType=evalType, deterministic=deterministic - ) - return udf_obj._wrapped() - - class UDFRegistration: """ - Wrapper for user-defined function registration (spark.udf.register). - """ + Wrapper for user-defined function registration. This instance can be accessed by + :attr:`spark.udf` or :attr:`sqlContext.udf`. - # As PySpark Connect's `spark.udf`, of pyspark.sql.connect.udf, whose name this class has too. - __module__ = "pyspark.sql.connect.udf" + .. versionadded:: 1.3.1 + """ - def __init__(self, sparkSession: Any): + def __init__(self, sparkSession: "SparkSession"): self.sparkSession = sparkSession def register( self, name: str, - f: Callable[..., Any], - returnType: Optional[DataType] = None, - ) -> Any: - """ - Register a Python UDF with the given name. + f: Union[Callable[..., Any], "UserDefinedFunctionLike"], + returnType: Optional["DataTypeOrString"] = None, + ) -> "UserDefinedFunctionLike": + """Register a Python function (including lambda function) or a user-defined function + as a SQL function. + + .. versionadded:: 1.3.1 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - name : str - Name to register the UDF with. - f : callable - The Python function to register. - returnType : DataType, optional - The return type of the UDF. Defaults to StringType(). + name : str, + name of the user-defined function in SQL statements. + f : function, :meth:`pyspark.sql.functions.udf` or :meth:`pyspark.sql.functions.pandas_udf` + a Python function, or a user-defined function. The user-defined function can + be either row-at-a-time or vectorized. See :meth:`pyspark.sql.functions.udf` and + :meth:`pyspark.sql.functions.pandas_udf`. + returnType : :class:`pyspark.sql.types.DataType` or str, optional + the return type of the registered user-defined function. The value can + be either a :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. + `returnType` can be optionally specified when `f` is a Python function but not + when `f` is a user-defined function. Please see the examples below. Returns ------- function - The registered UDF that can be called with columns: ``f`` if it is a UDF, - or the function wrapper of the UDF of ``f``, as PySpark returns. + a user-defined function + + Notes + ----- + To register a nondeterministic Python function, users need to first build + a nondeterministic user-defined function for the Python function and then register it + as a SQL function. Examples -------- - >>> from pyspark.sql.types import IntegerType - >>> spark.udf.register("inc_id", lambda x: x + 1, IntegerType()) - >>> result = spark.sql("SELECT inc_id(id) AS incremented FROM range(3)") + 1. When `f` is a Python function: + + `returnType` defaults to string type and can be optionally specified. The produced + object must match the specified type. In this case, this API works as if + `register(name, f, returnType=StringType())`. + + >>> strlen = spark.udf.register("stringLengthString", lambda x: len(x)) + >>> spark.sql("SELECT stringLengthString('test')").collect() + [Row(stringLengthString(test)='4')] + + >>> spark.sql("SELECT 'foo' AS text").select(strlen("text")).collect() + [Row(stringLengthString(text)='3')] + + >>> from pyspark.sql.types import IntegerType + >>> _ = spark.udf.register("stringLengthInt", lambda x: len(x), IntegerType()) + >>> spark.sql("SELECT stringLengthInt('test')").collect() + [Row(stringLengthInt(test)=4)] + + >>> from pyspark.sql.types import IntegerType + >>> _ = spark.udf.register("stringLengthInt", lambda x: len(x), IntegerType()) + >>> spark.sql("SELECT stringLengthInt('test')").collect() + [Row(stringLengthInt(test)=4)] + + 2. When `f` is a user-defined function (from Spark 2.3.0): + + Spark uses the return type of the given user-defined function as the return type of + the registered user-defined function. `returnType` should not be specified. + In this case, this API works as if `register(name, f)`. + + >>> from pyspark.sql.types import IntegerType + >>> from pyspark.sql.functions import udf + >>> slen = udf(lambda s: len(s), IntegerType()) + >>> _ = spark.udf.register("slen", slen) + >>> spark.sql("SELECT slen('test')").collect() + [Row(slen(test)=4)] + + >>> import random + >>> from pyspark.sql.functions import udf + >>> from pyspark.sql.types import IntegerType + >>> random_udf = udf(lambda: random.randint(0, 100), IntegerType()).asNondeterministic() + >>> new_random_udf = spark.udf.register("random_udf", random_udf) + >>> spark.sql("SELECT random_udf()").collect() # doctest: +SKIP + [Row(random_udf()=82)] + + >>> import pandas as pd + >>> from pyspark.sql.functions import pandas_udf + >>> @pandas_udf("integer") + ... def add_one(s: pd.Series) -> pd.Series: + ... return s + 1 + ... + >>> _ = spark.udf.register("add_one", add_one) + >>> spark.sql("SELECT add_one(id) FROM range(3)").collect() + [Row(add_one(id)=1), Row(add_one(id)=2), Row(add_one(id)=3)] + + >>> @pandas_udf("integer") + ... def sum_udf(v: pd.Series) -> int: + ... return v.sum() + ... + >>> _ = spark.udf.register("sum_udf", sum_udf) + >>> q = "SELECT sum_udf(v1) FROM VALUES (3, 0), (2, 0), (1, 1) tbl(v1, v2) GROUP BY v2" + >>> spark.sql(q).sort("sum_udf(v1)").collect() + [Row(sum_udf(v1)=1), Row(sum_udf(v1)=5)] """ - # As PySpark Connect's: a UDF (from udf() or pandas_udf()) is registered with its - # eval type, and a plain Python function as a SQL_BATCHED_UDF of `name`. The - # registration is a RegisterFunction command carrying the cloudpickled PythonUDF - # with no bound arguments, as the reference client.register_udf sends, so `name` - # resolves in SQL. + + # This is to check whether the input function is from a user-defined function or + # Python function. if hasattr(f, "asNondeterministic"): if returnType is not None: raise PySparkTypeError( errorClass="CANNOT_SPECIFY_RETURN_TYPE_FOR_UDF", messageParameters={"arg_name": "f", "return_type": str(returnType)}, ) + f = cast("UserDefinedFunctionLike", f) if f.evalType not in [ PythonEvalType.SQL_BATCHED_UDF, PythonEvalType.SQL_ARROW_BATCHED_UDF, @@ -590,45 +865,127 @@ def register( "SQL_GROUPED_AGG_PANDAS_ITER_UDF or SQL_GROUPED_AGG_ARROW_ITER_UDF" }, ) - self.sparkSession._client.register_udf( - f.func, f.returnType, name, f.evalType, f.deterministic + source_udf = _create_udf( + f.func, + returnType=f.returnType, + name=name, + evalType=f.evalType, + deterministic=f.deterministic, ) - return f + register_udf = source_udf._unwrapped # type: ignore[attr-defined] + return_udf = register_udf else: if returnType is None: returnType = StringType() - py_udf = _create_udf( + return_udf = _create_udf( f, returnType=returnType, evalType=PythonEvalType.SQL_BATCHED_UDF, name=name ) - - # As PySpark's: the registration takes the return type as given, not the one that - # the wrapper parsed, so a DDL string is parsed again for the command. - self.sparkSession._client.register_udf(py_udf.func, returnType, name) - return py_udf + register_udf = return_udf._unwrapped # type: ignore[attr-defined] + self.sparkSession._jsparkSession.udf().registerPython(name, register_udf._judf) + return return_udf def registerJavaFunction( self, name: str, javaClassName: str, - returnType: Optional[DataType] = None, + returnType: Optional["DataTypeOrString"] = None, ) -> None: - """Register a Java UDF by fully-qualified class name so it is callable by - ``name`` in SQL. Mirrors ``UDFRegistration.registerJavaFunction``. + """Register a Java user-defined function as a SQL function. + + In addition to a name and the function itself, the return type can be optionally specified. + When the return type is not specified we would infer it via reflection. + + .. versionadded:: 2.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- name : str - Name to register the function with. + name of the user-defined function javaClassName : str - Fully-qualified name of the Java class implementing the UDF. - returnType : DataType or str, optional - Return type: a DataType, or a DDL string, which the server parses, as PySpark - sends it. Defaults to the class's own type. + fully qualified name of java class + returnType : :class:`pyspark.sql.types.DataType` or str, optional + the return type of the registered Java function. The value can be either + a :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. + + Examples + -------- + >>> from pyspark.sql.types import IntegerType + >>> spark.udf.registerJavaFunction( + ... "javaStringLength", "test.org.apache.spark.sql.JavaStringLength", IntegerType()) + ... # doctest: +SKIP + >>> spark.sql("SELECT javaStringLength('test')").collect() # doctest: +SKIP + [Row(javaStringLength(test)=4)] + + >>> spark.udf.registerJavaFunction( + ... "javaStringLength2", "test.org.apache.spark.sql.JavaStringLength") + ... # doctest: +SKIP + >>> spark.sql("SELECT javaStringLength2('test')").collect() # doctest: +SKIP + [Row(javaStringLength2(test)=4)] + + >>> spark.udf.registerJavaFunction( + ... "javaStringLength3", "test.org.apache.spark.sql.JavaStringLength", "integer") + ... # doctest: +SKIP + >>> spark.sql("SELECT javaStringLength3('test')").collect() # doctest: +SKIP + [Row(javaStringLength3(test)=4)] """ - self.sparkSession._client.register_java(name, javaClassName, returnType) + + jdt = None + if returnType is not None: + if not isinstance(returnType, DataType): + returnType = _parse_datatype_string(returnType) + jdt = self.sparkSession._jsparkSession.parseDataType(returnType.json()) + self.sparkSession._jsparkSession.udf().registerJava(name, javaClassName, jdt) def registerJavaUDAF(self, name: str, javaClassName: str) -> None: - """Register a Java user-defined aggregate function (UDAF) by class name. - Mirrors ``UDFRegistration.registerJavaUDAF``. + """Register a Java user-defined aggregate function as a SQL function. + + .. versionadded:: 2.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + name : str + name of the user-defined aggregate function + javaClassName : str + fully qualified name of java class + + Examples + -------- + >>> spark.udf.registerJavaUDAF("javaUDAF", "test.org.apache.spark.sql.MyDoubleAvg") + ... # doctest: +SKIP + >>> df = spark.createDataFrame([(1, "a"),(2, "b"), (3, "a")],["id", "name"]) + >>> df.createOrReplaceTempView("df") + >>> q = "SELECT name, javaUDAF(id) as avg from df group by name order by name desc" + >>> spark.sql(q).collect() # doctest: +SKIP + [Row(name='b', avg=102.0), Row(name='a', avg=102.0)] """ - self.sparkSession._client.register_java(name, javaClassName, aggregate=True) + + self.sparkSession._jsparkSession.udf().registerJavaUDAF(name, javaClassName) + + +def _test() -> None: + import doctest + from pyspark.sql import SparkSession + import pyspark.sql.udf + from pyspark.testing.utils import have_pandas, have_pyarrow + + globs = pyspark.sql.udf.__dict__.copy() + + if not have_pandas or not have_pyarrow: + del pyspark.sql.udf.UDFRegistration.register.__doc__ + + spark = SparkSession.builder.master("local[4]").appName("sql.udf tests").getOrCreate() + globs["spark"] = spark + failure_count, test_count = doctest.testmod( + pyspark.sql.udf, globs=globs, optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE + ) + spark.stop() + if failure_count: + sys.exit(-1) + + +if __name__ == "__main__": + _test() diff --git a/python/pyspark/sql/udf.pyi b/python/pyspark/sql/udf.pyi deleted file mode 100644 index a69d3ca9..00000000 --- a/python/pyspark/sql/udf.pyi +++ /dev/null @@ -1,405 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Generated by scripts/gen_type_stubs.py from python/pyspark/sql/udf.py of Apache Spark v4.3.0: -# do not edit. Type checkers read this stub instead of the module of the drop-in. - -"""User-defined function related classes and functions""" - -import warnings -from typing import Callable, Any, Optional, Union -from pyspark.util import PythonEvalType as PythonEvalType -from pyspark.sql.column import Column -from pyspark.sql.types import DataType as DataType, StringType as StringType, StructType as StructType, _parse_datatype_string -from pyspark.sql.pandas.types import to_arrow_type as to_arrow_type -from pyspark.errors import PySparkTypeError as PySparkTypeError, PySparkNotImplementedError as PySparkNotImplementedError, PySparkRuntimeError as PySparkRuntimeError -from py4j.java_gateway import JavaObject -from pyspark.sql._typing import DataTypeOrString, ColumnOrName, UserDefinedFunctionLike -from pyspark.sql.session import SparkSession - -__all__ = ['UDFRegistration'] - -def _create_udf(f: Callable[..., Any], returnType: 'DataTypeOrString', evalType: int, name: Optional[str]=None, deterministic: bool=True) -> 'UserDefinedFunctionLike': - """Create a regular(non-Arrow-optimized) Python UDF.""" -def _create_py_udf(f: Callable[..., Any], returnType: 'DataTypeOrString', useArrow: Optional[bool]=None) -> 'UserDefinedFunctionLike': - """Create a regular/Arrow-optimized Python UDF.""" - -class UserDefinedFunction: - """User defined function in Python - - .. versionadded:: 1.3 - - Notes - ----- - The constructor of this class is not supposed to be directly called. - Use :meth:`pyspark.sql.functions.udf` or :meth:`pyspark.sql.functions.pandas_udf` - to create this instance. - """ - def __init__(self, func: Callable[..., Any], returnType: 'DataTypeOrString'=StringType(), name: Optional[str]=None, evalType: int=PythonEvalType.SQL_BATCHED_UDF, deterministic: bool=True): - if not callable(func): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "callable", - "arg_name": "func", - "arg_type": type(func).__name__, - }, - ) - - if not isinstance(returnType, (DataType, str)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "DataType or str", - "arg_name": "returnType", - "arg_type": type(returnType).__name__, - }, - ) - - if not isinstance(evalType, int): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "int", - "arg_name": "evalType", - "arg_type": type(evalType).__name__, - }, - ) - - self.func = func - self._returnType = returnType - # Stores UserDefinedPythonFunctions jobj, once initialized - self._returnType_placeholder: Optional[DataType] = None - self._judf_placeholder = None - self._name = name or ( - func.__name__ if hasattr(func, "__name__") else func.__class__.__name__ - ) - self.evalType = evalType - self.deterministic = deterministic - # Extract Python UDF details if transpilation is enabled. - self.transpiled: list = [] - self._transpiled_param_names: list[str] = [] - # The subset of the names above that Python forbids calling by keyword. - # The kwargs rewrite in ``__call__`` must leave these alone rather than - # silently coerce them to positional -- see the note there. - self._positional_only_param_names: frozenset = frozenset() - # Per-option input-type categories ("numeric"/"string" per public param), - # parallel to ``self.transpiled``; the JVM picks the option matching the - # actual column types or falls back to interpreted Python. - self._transpiled_input_categories: list = [] - # When we have a transpiled rewrite, ``__call__`` resolves any - # user-supplied kwargs against this positional parameter list so - # the JVM-side ``_udf_param_N`` substitution sees the inputs in - # the right order. Empty list when transpilation didn't happen. - from pyspark.sql import SparkSession - - session = SparkSession._instantiatedSession - - # A nondeterministic UDF must not be transpiled: replacing it with a plain - # Catalyst expression would let the optimizer fold/reorder/duplicate it, - # discarding the nondeterminism barrier. (asNondeterministic() also clears - # any options set here, for the udf(f).asNondeterministic() ordering.) - # Conf values are compared case-insensitively: `SET conf=True` stores - # the literal "True", which would otherwise silently disable - # transpilation (or mis-trigger the ANSI warning below). - # - # Each conf read is a JVM roundtrip, so keep the default construction - # path cheap: the experimental gate is only read for deterministic - # batched UDFs (the only shape we transpile), and the ANSI conf is only - # read once the gate is known to be on. When ``default`` is given it is - # passed through to ``RuntimeConfig.get`` so construction never depends - # on the JVM having the (experimental) conf registered -- e.g. a newer - # Python client against an older driver. No default is passed for - # ``spark.sql.ansi.enabled``: its registered default is dynamic - # (environment-driven) and must be respected when the key is unset. - def _conf_is_true(key: str, default: Optional[str] = None) -> bool: - if session is None: - return False - if default is None: - value = session.conf.get(key) - else: - value = session.conf.get(key, default) - return value is not None and value.lower() == "true" - - try: - transpile_enabled = ( - deterministic - and evalType == PythonEvalType.SQL_BATCHED_UDF - and _conf_is_true("spark.sql.experimental.optimizer.transpilePyUDFs", "false") - ) - # Transpilation only attempts to reproduce ANSI-mode Spark SQL - # semantics (no silent integer overflow, divide-by-zero raises, - # etc.). Running it against non-ANSI Spark would balloon the test - # matrix we'd have to maintain to verify Python-vs-SQL equivalence, - # so we gate on ANSI here and warn the user instead of trying to - # transpile in a mode we don't claim to support yet. - if transpile_enabled and not _conf_is_true("spark.sql.ansi.enabled"): - warnings.warn( - "Python UDF transpilation " - "(spark.sql.experimental.optimizer.transpilePyUDFs) is only " - "supported when ANSI mode is enabled " - "(spark.sql.ansi.enabled=true). Skipping transpilation for " - f"{func} -- enable ANSI mode or set transpilePyUDFs=false to " - "silence this warning.", - RuntimeWarning, - ) - transpile_enabled = False - if transpile_enabled and session: - # Import only if needed, also avoid circular import loops. - from pyspark.sql.transpile import _transpile_func - - # ``self.returnType`` parses (and caches) the declared return - # type; the transpiler needs the parsed form to decide whether - # the final Cast to it can resolve at all. The parse is reused - # later by ``_create_judf``, so this adds no extra JVM work. - ( - self.transpiled, - errors, - self._transpiled_param_names, - self._transpiled_input_categories, - positional_only, - ) = _transpile_func(session, func, self.returnType) - self._positional_only_param_names = frozenset(positional_only) - if not self.transpiled: - detail = f": {errors}" if errors else "" - warnings.warn(f"Unable to transpile UDF {func}{detail}") - except Exception as e: - # An inability to transpile must never break a working UDF -- fall - # back to interpreted Python execution and surface the failure as a - # warning so users can opt to investigate without losing their - # query. The conf reads above are included: a session whose JVM - # cannot answer them should degrade to "no transpilation", not - # break UDF definition. - warnings.warn(f"Exception transpiling UDF {func}: {e}") - self.transpiled = [] - self._transpiled_param_names = [] - self._transpiled_input_categories = [] - self._positional_only_param_names = frozenset() - @staticmethod - def _check_return_type(returnType: DataType, evalType: int) -> None: ... - @property - def returnType(self) -> DataType: - if self._returnType_placeholder is None: - if isinstance(self._returnType, DataType): - self._returnType_placeholder = self._returnType - else: - self._returnType_placeholder = _parse_datatype_string(self._returnType) - - UserDefinedFunction._check_return_type(self._returnType_placeholder, self.evalType) - return self._returnType_placeholder - @property - def _judf(self) -> 'JavaObject': - if self._judf_placeholder is None: - self._judf_placeholder = self._create_judf(self.func) - return self._judf_placeholder - def _create_judf(self, func: Callable[..., Any], include_transpiled: bool=True) -> 'JavaObject': ... - def __call__(self, *args: 'ColumnOrName', **kwargs: 'ColumnOrName') -> Column: ... - def _wrapped(self) -> 'UserDefinedFunctionLike': - """Wrap this udf with a function and attach docstring from func""" - def asNondeterministic(self) -> 'UserDefinedFunction': - """Updates UserDefinedFunction to nondeterministic. - - .. versionadded:: 2.3 - """ - self._judf_placeholder = None - self.deterministic = False - # A transpiled rewrite replaces the (now nondeterministic) Python UDF - # with a plain Catalyst expression, which the optimizer is free to - # fold, reorder, or duplicate -- discarding the nondeterminism barrier - # the caller just asked for. Drop any transpiled options so a - # nondeterministic UDF always runs as interpreted Python. - self.transpiled = [] - self._transpiled_param_names = [] - self._transpiled_input_categories = [] - self._positional_only_param_names = frozenset() - return self - -class UDFRegistration: - """Wrapper for user-defined function registration. This instance can be accessed by - :attr:`spark.udf` or :attr:`sqlContext.udf`. - - .. versionadded:: 1.3.1 - """ - def __init__(self, sparkSession: 'SparkSession'): - self.sparkSession = sparkSession - def register(self, name: str, f: Union[Callable[..., Any], 'UserDefinedFunctionLike'], returnType: Optional['DataTypeOrString']=None) -> 'UserDefinedFunctionLike': - """Register a Python function (including lambda function) or a user-defined function - as a SQL function. - - .. versionadded:: 1.3.1 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - name : str, - name of the user-defined function in SQL statements. - f : function, :meth:`pyspark.sql.functions.udf` or :meth:`pyspark.sql.functions.pandas_udf` - a Python function, or a user-defined function. The user-defined function can - be either row-at-a-time or vectorized. See :meth:`pyspark.sql.functions.udf` and - :meth:`pyspark.sql.functions.pandas_udf`. - returnType : :class:`pyspark.sql.types.DataType` or str, optional - the return type of the registered user-defined function. The value can - be either a :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. - `returnType` can be optionally specified when `f` is a Python function but not - when `f` is a user-defined function. Please see the examples below. - - Returns - ------- - function - a user-defined function - - Notes - ----- - To register a nondeterministic Python function, users need to first build - a nondeterministic user-defined function for the Python function and then register it - as a SQL function. - - Examples - -------- - 1. When `f` is a Python function: - - `returnType` defaults to string type and can be optionally specified. The produced - object must match the specified type. In this case, this API works as if - `register(name, f, returnType=StringType())`. - - >>> strlen = spark.udf.register("stringLengthString", lambda x: len(x)) - >>> spark.sql("SELECT stringLengthString('test')").collect() - [Row(stringLengthString(test)='4')] - - >>> spark.sql("SELECT 'foo' AS text").select(strlen("text")).collect() - [Row(stringLengthString(text)='3')] - - >>> from pyspark.sql.types import IntegerType - >>> _ = spark.udf.register("stringLengthInt", lambda x: len(x), IntegerType()) - >>> spark.sql("SELECT stringLengthInt('test')").collect() - [Row(stringLengthInt(test)=4)] - - >>> from pyspark.sql.types import IntegerType - >>> _ = spark.udf.register("stringLengthInt", lambda x: len(x), IntegerType()) - >>> spark.sql("SELECT stringLengthInt('test')").collect() - [Row(stringLengthInt(test)=4)] - - 2. When `f` is a user-defined function (from Spark 2.3.0): - - Spark uses the return type of the given user-defined function as the return type of - the registered user-defined function. `returnType` should not be specified. - In this case, this API works as if `register(name, f)`. - - >>> from pyspark.sql.types import IntegerType - >>> from pyspark.sql.functions import udf - >>> slen = udf(lambda s: len(s), IntegerType()) - >>> _ = spark.udf.register("slen", slen) - >>> spark.sql("SELECT slen('test')").collect() - [Row(slen(test)=4)] - - >>> import random - >>> from pyspark.sql.functions import udf - >>> from pyspark.sql.types import IntegerType - >>> random_udf = udf(lambda: random.randint(0, 100), IntegerType()).asNondeterministic() - >>> new_random_udf = spark.udf.register("random_udf", random_udf) - >>> spark.sql("SELECT random_udf()").collect() # doctest: +SKIP - [Row(random_udf()=82)] - - >>> import pandas as pd - >>> from pyspark.sql.functions import pandas_udf - >>> @pandas_udf("integer") - ... def add_one(s: pd.Series) -> pd.Series: - ... return s + 1 - ... - >>> _ = spark.udf.register("add_one", add_one) - >>> spark.sql("SELECT add_one(id) FROM range(3)").collect() - [Row(add_one(id)=1), Row(add_one(id)=2), Row(add_one(id)=3)] - - >>> @pandas_udf("integer") - ... def sum_udf(v: pd.Series) -> int: - ... return v.sum() - ... - >>> _ = spark.udf.register("sum_udf", sum_udf) - >>> q = "SELECT sum_udf(v1) FROM VALUES (3, 0), (2, 0), (1, 1) tbl(v1, v2) GROUP BY v2" - >>> spark.sql(q).sort("sum_udf(v1)").collect() - [Row(sum_udf(v1)=1), Row(sum_udf(v1)=5)] - """ - def registerJavaFunction(self, name: str, javaClassName: str, returnType: Optional['DataTypeOrString']=None) -> None: - """Register a Java user-defined function as a SQL function. - - In addition to a name and the function itself, the return type can be optionally specified. - When the return type is not specified we would infer it via reflection. - - .. versionadded:: 2.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - name : str - name of the user-defined function - javaClassName : str - fully qualified name of java class - returnType : :class:`pyspark.sql.types.DataType` or str, optional - the return type of the registered Java function. The value can be either - a :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string. - - Examples - -------- - >>> from pyspark.sql.types import IntegerType - >>> spark.udf.registerJavaFunction( - ... "javaStringLength", "test.org.apache.spark.sql.JavaStringLength", IntegerType()) - ... # doctest: +SKIP - >>> spark.sql("SELECT javaStringLength('test')").collect() # doctest: +SKIP - [Row(javaStringLength(test)=4)] - - >>> spark.udf.registerJavaFunction( - ... "javaStringLength2", "test.org.apache.spark.sql.JavaStringLength") - ... # doctest: +SKIP - >>> spark.sql("SELECT javaStringLength2('test')").collect() # doctest: +SKIP - [Row(javaStringLength2(test)=4)] - - >>> spark.udf.registerJavaFunction( - ... "javaStringLength3", "test.org.apache.spark.sql.JavaStringLength", "integer") - ... # doctest: +SKIP - >>> spark.sql("SELECT javaStringLength3('test')").collect() # doctest: +SKIP - [Row(javaStringLength3(test)=4)] - """ - def registerJavaUDAF(self, name: str, javaClassName: str) -> None: - """Register a Java user-defined aggregate function as a SQL function. - - .. versionadded:: 2.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - name : str - name of the user-defined aggregate function - javaClassName : str - fully qualified name of java class - - Examples - -------- - >>> spark.udf.registerJavaUDAF("javaUDAF", "test.org.apache.spark.sql.MyDoubleAvg") - ... # doctest: +SKIP - >>> df = spark.createDataFrame([(1, "a"),(2, "b"), (3, "a")],["id", "name"]) - >>> df.createOrReplaceTempView("df") - >>> q = "SELECT name, javaUDAF(id) as avg from df group by name order by name desc" - >>> spark.sql(q).collect() # doctest: +SKIP - [Row(name='b', avg=102.0), Row(name='a', avg=102.0)] - """ - -# Names of the drop-in's module that PySpark's does not have. -from pyspark.serializers import CloudPickleSerializer as CloudPickleSerializer -from pyspark.sql.functions.builtin import udf as udf diff --git a/python/pyspark/sql/udtf.py b/python/pyspark/sql/udtf.py index 4aeac456..74400c3d 100644 --- a/python/pyspark/sql/udtf.py +++ b/python/pyspark/sql/udtf.py @@ -14,24 +14,33 @@ # See the License for the specific language governing permissions and # limitations under the License. # - """ -User-defined table function (UDTF) support for the Spark Connect client. -Mirrors pyspark.sql.connect.udtf and pyspark.sql.functions.udtf. +User-defined table function related classes and functions """ -from pyspark._pyspark import SkipRestOfInputTableException - -import functools -import inspect import pickle -import sys from dataclasses import dataclass, field -from typing import Any, Optional, Sequence, Type, Union - -from pyspark.errors import PySparkAttributeError, PySparkPicklingError, PySparkTypeError -from pyspark.serializers import CloudPickleSerializer -from pyspark.sql.types import DataType, StructType +import inspect +import sys +import warnings +from typing import Any, Type, TYPE_CHECKING, Optional, Sequence, Union + +from pyspark.errors import ( + PySparkAttributeError, + PySparkPicklingError, + PySparkTypeError, + PySparkImportError, +) +from pyspark.util import PythonEvalType +from pyspark.sql.pandas.utils import require_minimum_pandas_version, require_minimum_pyarrow_version +from pyspark.sql.types import DataType, StructType, _parse_datatype_string +from pyspark.sql.udf import _wrap_function + +if TYPE_CHECKING: + from py4j.java_gateway import JavaObject + from pyspark.sql._typing import TVFArgumentOrName + from pyspark.sql.dataframe import DataFrame + from pyspark.sql.session import SparkSession __all__ = [ "AnalyzeArgument", @@ -43,15 +52,7 @@ "UDTFRegistration", ] -# Eval types for Python UDTFs (mirrors pyspark.util.PythonEvalType). -SQL_TABLE_UDF = 300 -SQL_ARROW_TABLE_UDF = 301 -SQL_ARROW_UDTF = 302 - -# Verbatim from PySpark (pyspark.sql.udtf): the frozen dataclasses of the analyze method of a -# UDTF, which a handler that the client pickles may capture, and which the Python worker of -# the server unpickles as PySpark's of the same names. @dataclass(frozen=True) class AnalyzeArgument: """ @@ -139,12 +140,35 @@ class SelectedColumn: alias: str = "" +# Note: this class is a "dataclass" for purposes of convenience, but it is not marked "frozen" +# because the intention is that users may create subclasses of it for purposes of returning custom +# information from the "analyze" method. @dataclass class AnalyzeResult: - """The result of a Python UDTF's ``analyze`` static method. + """ + The return of Python UDTF's analyze static method. - As in PySpark, this is a mutable dataclass so a handler can subclass it to - return custom analysis state, which is passed to the handler's constructor. + Parameters + ---------- + schema: :class:`StructType` + The schema that the Python UDTF will return. + withSinglePartition: bool + If true, the UDTF is specifying for Catalyst to repartition all rows of the input TABLE + argument to one collection for consumption by exactly one instance of the corresponding + UDTF class. + partitionBy: sequence of :class:`PartitioningColumn` + If non-empty, this is a sequence of expressions that the UDTF is specifying for Catalyst to + partition the input TABLE argument by. In this case, calls to the UDTF may not include any + explicit PARTITION BY clause, in which case Catalyst will return an error. This option is + mutually exclusive with 'withSinglePartition'. + orderBy: sequence of :class:`OrderingColumn` + If non-empty, this is a sequence of expressions that the UDTF is specifying for Catalyst to + sort the input TABLE argument by. Note that the 'partitionBy' list must also be non-empty + in this case. + select: sequence of :class:`SelectedColumn` + If non-empty, this is a sequence of expressions that the UDTF is specifying for Catalyst to + evaluate against the columns in the input TABLE argument. The UDTF then receives one input + attribute for each name in the list, in the order they are listed. """ schema: StructType @@ -154,8 +178,117 @@ class AnalyzeResult: select: Sequence[SelectedColumn] = field(default_factory=tuple) -# Verbatim from PySpark (pyspark.sql.udtf): validates the handler class of a UDTF. -def _validate_udtf_handler(cls: Any, returnType: Optional[Any]) -> None: +class SkipRestOfInputTableException(Exception): + """ + This represents an exception that the 'eval' method may raise to indicate that it is done + consuming rows from the current partition of the input table. Then the UDTF's 'terminate' + method runs (if any). + """ + + pass + + +def _create_udtf( + cls: Type, + returnType: Optional[Union[StructType, str]], + name: Optional[str] = None, + evalType: int = PythonEvalType.SQL_TABLE_UDF, + deterministic: bool = False, +) -> "UserDefinedTableFunction": + """Create a Python UDTF with the given eval type.""" + udtf_obj = UserDefinedTableFunction( + cls, returnType=returnType, name=name, evalType=evalType, deterministic=deterministic + ) + + return udtf_obj + + +def _create_py_udtf( + cls: Type, + returnType: Optional[Union[StructType, str]], + name: Optional[str] = None, + deterministic: bool = False, + useArrow: Optional[bool] = None, +) -> "UserDefinedTableFunction": + """Create a regular or an Arrow-optimized Python UDTF.""" + # Determine whether to create Arrow-optimized UDTFs. + if useArrow is not None: + arrow_enabled = useArrow + else: + from pyspark.sql import SparkSession + + session = SparkSession._instantiatedSession + arrow_enabled = False + if session is not None: + value = session.conf.get("spark.sql.execution.pythonUDTF.arrow.enabled") + if isinstance(value, str) and value.lower() == "true": + arrow_enabled = True + + eval_type: int = PythonEvalType.SQL_TABLE_UDF + + if arrow_enabled: + # Return the regular UDTF if the required dependencies are not satisfied. + try: + require_minimum_pandas_version() + require_minimum_pyarrow_version() + eval_type = PythonEvalType.SQL_ARROW_TABLE_UDF + except ImportError as e: + warnings.warn( + f"Arrow optimization for Python UDTFs cannot be enabled: {str(e)}. " + f"Falling back to using regular Python UDTFs.", + UserWarning, + ) + + return _create_udtf( + cls=cls, + returnType=returnType, + name=name, + evalType=eval_type, + deterministic=deterministic, + ) + + +def _create_pyarrow_udtf( + cls: Type, + returnType: Optional[Union[StructType, str]], + name: Optional[str] = None, + deterministic: bool = False, +) -> "UserDefinedTableFunction": + """Create a PyArrow-native Python UDTF.""" + # Validate PyArrow dependencies + try: + require_minimum_pyarrow_version() + except ImportError as e: + raise PySparkImportError(f"PyArrow UDTF requires pyarrow dependencies: {str(e)}") from e + + # Validate the handler class with PyArrow-specific checks + _validate_arrow_udtf_handler(cls, returnType) + + return _create_udtf( + cls=cls, + returnType=returnType, + name=name, + evalType=PythonEvalType.SQL_ARROW_UDTF, + deterministic=deterministic, + ) + + +def _validate_arrow_udtf_handler(cls: Any, returnType: Optional[Union[StructType, str]]) -> None: + """Validate the handler class of a PyArrow UDTF.""" + # First run standard UDTF validation + _validate_udtf_handler(cls, returnType) + + # Block analyze method usage in arrow UDTFs + # TODO(SPARK-53286): Support analyze method for Arrow UDTFs to enable dynamic return types + has_analyze = hasattr(cls, "analyze") + if has_analyze: + raise PySparkAttributeError( + errorClass="INVALID_ARROW_UDTF_WITH_ANALYZE", + messageParameters={"name": cls.__name__}, + ) + + +def _validate_udtf_handler(cls: Any, returnType: Optional[Union[StructType, str]]) -> None: """Validate the handler class of a UDTF.""" if not isinstance(cls, type): @@ -183,240 +316,164 @@ def _validate_udtf_handler(cls: Any, returnType: Optional[Any]) -> None: ) -# Verbatim from PySpark Connect (pyspark.sql.connect.udtf): validates the handler class of a -# PyArrow UDTF. -def _validate_arrow_udtf_handler(cls: Any, returnType: Optional[Any]) -> None: - """Validate the handler class of a PyArrow UDTF.""" - # First run standard UDTF validation - _validate_udtf_handler(cls, returnType) - - # Block analyze method usage in arrow UDTFs - # TODO(SPARK-53286): Support analyze method for Arrow UDTFs to enable dynamic return types - has_analyze = hasattr(cls, "analyze") - if has_analyze: - raise PySparkAttributeError( - errorClass="INVALID_ARROW_UDTF_WITH_ANALYZE", - messageParameters={"name": cls.__name__}, - ) - - -def _python_ver() -> str: - """The Python version of the client, ``major.minor``, as PySpark's ``get_python_ver``, which - the plan of a UDTF carries.""" - return f"{sys.version_info.major}.{sys.version_info.minor}" - - -def _pickled_handler(func: Type, name: Optional[str] = None) -> bytes: - """The command of a UDTF of the handler class ``func``, as PySpark Connect's - ``PythonUDTF.to_plan`` pickles it each time it builds a plan that holds the UDTF, and its - registration when it registers it: ``UDTF_SERIALIZATION_ERROR`` of the name of the class, of - a handler that does not pickle, which is ``name``, the name that the ``PythonUDTF`` of a call - read at the call, or else the name of the class now. Connect sends the schema separately, in - ``PythonUDTF.return_type``, so the command is only the handler class, unlike that of a scalar - UDF.""" - try: - return CloudPickleSerializer().dumps(func) - except pickle.PicklingError: - raise PySparkPicklingError( - errorClass="UDTF_SERIALIZATION_ERROR", - messageParameters={ - "name": func.__name__ if name is None else name, - "message": "Please check the stack trace and " - "make sure the function is serializable.", - }, - ) - - -def _udtf_command(func: Type, name: str, returnType: Optional[Any]) -> bytes: - """The ``PythonUDTF`` message of the return type and the command of a call of a UDTF, as - PySpark Connect's ``PythonUDTF.to_plan`` builds them each time it builds a plan that holds the - call, of what the ``PythonUDTF`` of the call keeps: the handler class ``func``, its ``name`` - then and the return type object ``returnType``. It converts the return type first, of the - object as it is now, with ``pyspark_types_to_proto_types``: so a ``StructType`` changed in - place after the call is the type that the request sends, and the error of a type that does not - convert, such as ``UNSUPPORTED_OPERATION`` of a value that is not a DataType, is raised before - the handler is pickled. Then it pickles the handler (see ``_pickled_handler``). The request - sends this type and this command, with the eval type and the Python version of the call.""" - import pyspark.sql.connect.proto as pb2 - from pyspark.sql.connect.types import pyspark_types_to_proto_types - - udtf = pb2.PythonUDTF() - if returnType is not None: - udtf.return_type.CopyFrom(pyspark_types_to_proto_types(returnType)) - udtf.command = _pickled_handler(func, name) - return udtf.SerializeToString() +class UserDefinedTableFunction: + """ + User-defined table function in Python + .. versionadded:: 3.5.0 -class UserDefinedTableFunction: - """A user-defined table function: a Python handler class that yields rows. + Notes + ----- + The constructor of this class is not supposed to be directly called. + Use :meth:`pyspark.sql.functions.udtf` to create this instance. - Calling it with Column arguments returns a DataFrame — the - ``CommonInlineUserDefinedTableFunction`` relation. Mirrors - ``pyspark.sql.connect.udtf.UserDefinedTableFunction``. + This API is evolving. """ - # As PySpark Connect's: the class is that of pyspark.sql.connect.udtf. A UDTF that a function - # captures is pickled by reference to that name, which the Python worker of the server has - # as the Connect class, not as the classic class of PySpark, which needs a SparkContext. - __module__ = "pyspark.sql.connect.udtf" - def __init__( self, func: Type, returnType: Optional[Union[StructType, str]], name: Optional[str] = None, - evalType: int = SQL_TABLE_UDF, + evalType: int = PythonEvalType.SQL_TABLE_UDF, deterministic: bool = False, - ) -> None: - # cloudpickle pickles this class by reference only when its module is imported. - import pyspark.sql.connect.udtf # noqa: F401 - - # As PySpark's UserDefinedTableFunction, for udtf(), arrow_udtf() and register(), whose - # returnType has no default, and which sets its attributes in this order, as vars() shows - # them. A returnType that is neither a DataType nor a str is kept as it is, as PySpark - # converts it when it builds a plan of a call. + ): _validate_udtf_handler(func, returnType) - self.func = func - # As PySpark Connect's: a DDL string is a type that the server parses. - if isinstance(returnType, str): - from pyspark.sql.connect.types import UnparsedDataType - returnType = UnparsedDataType(returnType) - self.returnType = returnType + self.func = func + self._returnType = returnType + self._returnType_placeholder: Optional[StructType] = None + self._inputTypes_placeholder = None + self._judtf_placeholder = None self._name = name or func.__name__ self.evalType = evalType self.deterministic = deterministic - def __call__(self, *args: Any, **kwargs: Any) -> Any: - from pyspark import _pyspark - from pyspark.sql import SparkSession - from pyspark.sql.connect.types import UnparsedDataType - - # As PySpark's: an argument is ColumnOrName, so a bare string is a column name, of - # which pyfunc_make_udtf makes a ColumnReference, and each keyword argument is a - # NamedArgumentExpression, after the positional ones. As its `PythonUDTF`, made at the - # call, the plan keeps the handler class, its name and the return type object as they - # are now, a str as an UnparsedDataType, so a returnType or a func set after the call is - # not that of the call. As its `PythonUDTF.to_plan`, each request converts that type, as - # the object is then, and pickles the handler class when it builds the plan of the - # DataFrame, after the arguments: `_udtf_command` gives both, so the call passes no type. - cols = list(args) - named = list(kwargs.items()) - returnType = self.returnType - if isinstance(returnType, str): - returnType = UnparsedDataType(returnType) - return _pyspark.functions.pyfunc_make_udtf( - SparkSession.active(), - self._name, - None, - self.evalType, - functools.partial(_udtf_command, self.func, self.func.__name__, returnType), - _python_ver(), - *cols, - deterministic=self.deterministic, - named_args=named, - ) - - def asDeterministic(self) -> "UserDefinedTableFunction": - """Mark this UDTF as deterministic, and return it. Mirrors - ``UserDefinedTableFunction.asDeterministic``.""" - self.deterministic = True - return self - - -def udtf( - cls: Optional[Type] = None, - *, - returnType: Optional[Any] = None, - useArrow: Optional[bool] = None, -) -> Any: - """Create a user-defined table function (UDTF). - - Use as ``@udtf(returnType=...)`` on a handler class, or call - ``udtf(MyClass, returnType=...)`` directly. As PySpark's ``_create_py_udtf``, the UDTF is - Arrow-optimized when ``useArrow`` is True, or is not given and - ``spark.sql.execution.pythonUDTF.arrow.enabled`` of the active session is true. - """ - - def _udtf_decorator(handler: Type) -> UserDefinedTableFunction: - if useArrow is not None: - arrow_enabled = useArrow - else: - from pyspark.errors import PySparkRuntimeError - from pyspark.sql import SparkSession - - arrow_enabled = False - try: - session = SparkSession.active() - arrow_enabled = ( - str(session.conf.get("spark.sql.execution.pythonUDTF.arrow.enabled")).lower() - == "true" - ) - except PySparkRuntimeError as e: - if e.getCondition() == "NO_ACTIVE_OR_DEFAULT_SESSION": - pass # Just uses the default if no session found. - else: - raise e - eval_type = SQL_TABLE_UDF - if arrow_enabled: - try: - from pyspark.sql.pandas.utils import ( - require_minimum_pandas_version, - require_minimum_pyarrow_version, - ) - - require_minimum_pandas_version() - require_minimum_pyarrow_version() - eval_type = SQL_ARROW_TABLE_UDF - except ImportError as e: - import warnings - - warnings.warn( - f"Arrow optimization for Python UDTFs cannot be enabled: {str(e)}. " - f"Falling back to using regular Python UDTFs.", - UserWarning, + @property + def returnType(self) -> Optional[StructType]: + if self._returnType is None: + return None + # `_parse_datatype_string` accesses to JVM for parsing a DDL formatted string. + # This makes sure this is called after SparkContext is initialized. + if self._returnType_placeholder is None: + if isinstance(self._returnType, str): + parsed = _parse_datatype_string(self._returnType) + else: + parsed = self._returnType + if not isinstance(parsed, StructType): + raise PySparkTypeError( + errorClass="UDTF_RETURN_TYPE_MISMATCH", + messageParameters={ + "name": self._name, + "return_type": f"{parsed}", + }, ) - return UserDefinedTableFunction(handler, returnType=returnType, evalType=eval_type) + self._returnType_placeholder = parsed + return self._returnType_placeholder - if cls is not None: - return _udtf_decorator(cls) - return _udtf_decorator + @property + def _judtf(self) -> "JavaObject": + if self._judtf_placeholder is None: + self._judtf_placeholder = self._create_judtf(self.func) + return self._judtf_placeholder + def _create_judtf(self, func: Type) -> "JavaObject": + from pyspark.sql import SparkSession -def arrow_udtf( - cls: Optional[Type] = None, - *, - returnType: Optional[Any] = None, -) -> Any: - """Create an Arrow-based user-defined table function (UDTF). - - Mirrors ``pyspark.sql.functions.arrow_udtf``: a PyArrow-native UDTF (``SQL_ARROW_UDTF``), - whose ``eval`` takes ``pyarrow`` arrays and returns ``pyarrow`` tables. - """ - def _arrow_udtf_decorator(handler: Type) -> UserDefinedTableFunction: - from pyspark.sql.pandas.utils import require_minimum_pyarrow_version + spark = SparkSession._getActiveSessionOrCreate() + sc = spark.sparkContext + + try: + wrapped_func = _wrap_function(sc, func) + except pickle.PicklingError as e: + if "CONTEXT_ONLY_VALID_ON_DRIVER" in str(e): + raise PySparkPicklingError( + errorClass="UDTF_SERIALIZATION_ERROR", + messageParameters={ + "name": self._name, + "message": "it appears that you are attempting to reference SparkSession " + "inside a UDTF. SparkSession can only be used on the driver, " + "not in code that runs on workers. Please remove the reference " + "and try again.", + }, + ) from None + raise PySparkPicklingError( + errorClass="UDTF_SERIALIZATION_ERROR", + messageParameters={ + "name": self._name, + "message": "Please check the stack trace and make sure the " + "function is serializable.", + }, + ) - require_minimum_pyarrow_version() - _validate_arrow_udtf_handler(handler, returnType) - return UserDefinedTableFunction(handler, returnType=returnType, evalType=SQL_ARROW_UDTF) + assert sc._jvm is not None + if self.returnType is None: + judtf = getattr( + sc._jvm, "org.apache.spark.sql.execution.python.UserDefinedPythonTableFunction" + )(self._name, wrapped_func, self.evalType, self.deterministic) + else: + jdt = spark._jsparkSession.parseDataType(self.returnType.json()) + judtf = getattr( + sc._jvm, "org.apache.spark.sql.execution.python.UserDefinedPythonTableFunction" + )(self._name, wrapped_func, jdt, self.evalType, self.deterministic) + return judtf + + def __call__(self, *args: "TVFArgumentOrName", **kwargs: "TVFArgumentOrName") -> "DataFrame": + from pyspark.sql.classic.column import _to_java_column, _to_seq + + from pyspark.sql import DataFrame, SparkSession + from pyspark.sql.table_arg import TableArg + + spark = SparkSession._getActiveSessionOrCreate() + sc = spark.sparkContext + + assert sc._jvm is not None + # Process positional arguments + jargs = [] + for arg in args: + if isinstance(arg, TableArg): + # If the argument is a TableArg, get the Java TableArg object + jargs.append(arg._j_table_arg) # type: ignore[attr-defined] + else: + # Otherwise, convert it to a Java column + jargs.append(_to_java_column(arg)) # type: ignore[arg-type] + + # Process keyword arguments + jkwargs = [] + for key, value in kwargs.items(): + if isinstance(value, TableArg): + # If the value is a TableArg, get the Java TableArg object + j_arg = value._j_table_arg # type: ignore[attr-defined] + else: + # Otherwise, convert it to a Java column + j_arg = _to_java_column(value) # type: ignore[arg-type] + # Create a named argument expression + j_named_arg = sc._jvm.PythonSQLUtils.namedArgumentExpression(key, j_arg) + jkwargs.append(j_named_arg) + + judtf = self._judtf + jPythonUDTF = judtf.apply(spark._jsparkSession, _to_seq(sc, jargs + jkwargs)) + return DataFrame(jPythonUDTF, spark) - if cls is not None: - return _arrow_udtf_decorator(cls) - return _arrow_udtf_decorator + def asDeterministic(self) -> "UserDefinedTableFunction": + """ + Updates UserDefinedTableFunction to deterministic. + """ + # Explicitly clean the cache to create a JVM UDTF instance. + self._judtf_placeholder = None + self.deterministic = True + return self class UDTFRegistration: """ - Wrapper for user-defined table function registration, as ``spark.udtf``. Mirrors - ``pyspark.sql.connect.udtf.UDTFRegistration``. + Wrapper for user-defined table function registration. This instance can be accessed by + :attr:`spark.udtf` or :attr:`sqlContext.udtf`. .. versionadded:: 3.5.0 """ - # As PySpark Connect's `spark.udtf`, of pyspark.sql.connect.udtf, whose name this class has too. - __module__ = "pyspark.sql.connect.udtf" - - def __init__(self, sparkSession: Any): + def __init__(self, sparkSession: "SparkSession"): self.sparkSession = sparkSession def register( @@ -424,8 +481,47 @@ def register( name: str, f: "UserDefinedTableFunction", ) -> "UserDefinedTableFunction": - """Register the UDTF ``f`` as ``name`` on the server, so that SQL calls it, and return - ``f``. As PySpark's, ``f`` is a UDTF of ``udtf()`` or ``arrow_udtf()``.""" + """Register a Python user-defined table function as a SQL table function. + + .. versionadded:: 3.5.0 + + Parameters + ---------- + name : str + The name of the user-defined table function in SQL statements. + f : function or :meth:`pyspark.sql.functions.udtf` + The user-defined table function. + + Returns + ------- + function + The registered user-defined table function. + + Notes + ----- + Spark uses the return type of the given user-defined table function as the return + type of the registered user-defined function. + + To register a nondeterministic Python table function, users need to first build + a nondeterministic user-defined table function and then register it as a SQL function. + + Examples + -------- + >>> from pyspark.sql.functions import udtf + >>> @udtf(returnType="c1: int, c2: int") + ... class PlusOne: + ... def eval(self, x: int): + ... yield x, x + 1 + ... + >>> _ = spark.udtf.register(name="plus_one", f=PlusOne) + >>> spark.sql("SELECT * FROM plus_one(1)").collect() + [Row(c1=1, c2=2)] + + Use it with lateral join + + >>> spark.sql("SELECT * FROM VALUES (0, 1), (1, 2) t(x, y), LATERAL plus_one(x)").collect() + [Row(x=0, y=1, c1=0, c2=1), Row(x=1, y=2, c1=1, c2=2)] + """ if not isinstance(f, UserDefinedTableFunction): raise PySparkTypeError( errorClass="CANNOT_REGISTER_UDTF", @@ -434,24 +530,45 @@ def register( }, ) - if f.evalType not in [SQL_TABLE_UDF, SQL_ARROW_TABLE_UDF, SQL_ARROW_UDTF]: + if f.evalType not in [ + PythonEvalType.SQL_TABLE_UDF, + PythonEvalType.SQL_ARROW_TABLE_UDF, + PythonEvalType.SQL_ARROW_UDTF, + ]: raise PySparkTypeError( errorClass="INVALID_UDTF_EVAL_TYPE", messageParameters={ "name": name, - "eval_type": "SQL_TABLE_UDF, SQL_ARROW_TABLE_UDF, SQL_ARROW_UDTF", + "eval_type": "SQL_TABLE_UDF, SQL_ARROW_TABLE_UDF", }, ) - # As PySpark Connect's `register_udtf`, which takes the Python version of the client, - # which a UDTF does not keep, and sets the name and converts the return type before it - # pickles the handler class: `_registerPythonUdtf` calls the pickling after them. - self.sparkSession._registerPythonUdtf( - name, - f.returnType, - f.evalType, - functools.partial(_pickled_handler, f.func), - _python_ver(), - f.deterministic, + register_udtf = _create_udtf( + cls=f.func, + returnType=f.returnType, + name=name, + evalType=f.evalType, + deterministic=f.deterministic, ) - return f + self.sparkSession._jsparkSession.udtf().registerPython(name, register_udtf._judtf) + return register_udtf + + +def _test() -> None: + import doctest + from pyspark.sql import SparkSession + import pyspark.sql.udf + + globs = pyspark.sql.udtf.__dict__.copy() + spark = SparkSession.builder.master("local[4]").appName("sql.udtf tests").getOrCreate() + globs["spark"] = spark + failure_count, test_count = doctest.testmod( + pyspark.sql.udtf, globs=globs, optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE + ) + spark.stop() + if failure_count: + sys.exit(-1) + + +if __name__ == "__main__": + _test() diff --git a/python/pyspark/sql/udtf.pyi b/python/pyspark/sql/udtf.pyi deleted file mode 100644 index 932fa636..00000000 --- a/python/pyspark/sql/udtf.pyi +++ /dev/null @@ -1,266 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Generated by scripts/gen_type_stubs.py from python/pyspark/sql/udtf.py of Apache Spark v4.3.0: -# do not edit. Type checkers read this stub instead of the module of the drop-in. - -"""User-defined table function related classes and functions""" - -from dataclasses import dataclass -from typing import Any, Type, Optional, Sequence, Union -from pyspark.errors import PySparkAttributeError as PySparkAttributeError, PySparkPicklingError as PySparkPicklingError, PySparkTypeError as PySparkTypeError -from pyspark.util import PythonEvalType -from pyspark.sql.types import DataType as DataType, StructType as StructType, _parse_datatype_string -from py4j.java_gateway import JavaObject -from pyspark.sql._typing import TVFArgumentOrName -from pyspark.sql.dataframe import DataFrame -from pyspark.sql.session import SparkSession - -__all__ = ['AnalyzeArgument', 'AnalyzeResult', 'PartitioningColumn', 'OrderingColumn', 'SelectedColumn', 'SkipRestOfInputTableException', 'UDTFRegistration'] - -@dataclass(frozen=True) -class AnalyzeArgument: - """The argument for Python UDTF's analyze static method. - - Parameters - ---------- - dataType : :class:`DataType` - The argument's data type - value : any, optional - The calculated value if the argument is foldable; otherwise None - isTable : bool - If True, the argument is a table argument. - isConstantExpression : bool - If True, the argument is a constant-foldable scalar expression. Then the 'value' field - contains None if the argument is a NULL literal, or a non-None value if the argument is a - non-NULL literal. In this way, we can distinguish between a literal NULL argument and other - types of arguments such as complex expression trees or table arguments where the 'value' - field is always None. - """ - dataType: DataType - value: Optional[Any] - isTable: bool - isConstantExpression: bool - -@dataclass(frozen=True) -class PartitioningColumn: - """Represents an expression that the UDTF is specifying for Catalyst to partition the input table - by. This can be either the name of a single column from the input table (such as "columnA"), or - a SQL expression based on the column names of the input table (such as "columnA + columnB"). - - Parameters - ---------- - name : str - The contents of the partitioning column name or expression represented as a SQL string. - """ - name: str - -@dataclass(frozen=True) -class OrderingColumn: - """Represents an expression that the UDTF is specifying for Catalyst to order the input partition - by. This can be either the name of a single column from the input table (such as "columnA"), - or a SQL expression based on the column names of the input table (such as "columnA + columnB"). - - Parameters - ---------- - name : str - The contents of the ordering column name or expression represented as a SQL string. - ascending : bool, default True - This is if this expression specifies an ascending sorting order. - overrideNullsFirst : str, optional - If this is None, use the default behavior to sort NULL values first when sorting in - ascending order, or last when sorting in descending order. Otherwise, if this is - True or False, we override the default behavior accordingly. - """ - name: str - ascending: bool = ... - overrideNullsFirst: Optional[bool] = ... - -@dataclass(frozen=True) -class SelectedColumn: - """Represents an expression that the UDTF is specifying for Catalyst to evaluate against the - columns in the input TABLE argument. The UDTF then receives one input column for each expression - in the list, in the order they are listed. - - Parameters - ---------- - name : str - The contents of the selected column name or expression represented as a SQL string. - alias : str, default '' - If non-empty, this is the alias for the column or expression as visible from the UDTF's - 'eval' method. This is required if the expression is not a simple column reference. - """ - name: str - alias: str = ... - -@dataclass -class AnalyzeResult: - """The return of Python UDTF's analyze static method. - - Parameters - ---------- - schema: :class:`StructType` - The schema that the Python UDTF will return. - withSinglePartition: bool - If true, the UDTF is specifying for Catalyst to repartition all rows of the input TABLE - argument to one collection for consumption by exactly one instance of the corresponding - UDTF class. - partitionBy: sequence of :class:`PartitioningColumn` - If non-empty, this is a sequence of expressions that the UDTF is specifying for Catalyst to - partition the input TABLE argument by. In this case, calls to the UDTF may not include any - explicit PARTITION BY clause, in which case Catalyst will return an error. This option is - mutually exclusive with 'withSinglePartition'. - orderBy: sequence of :class:`OrderingColumn` - If non-empty, this is a sequence of expressions that the UDTF is specifying for Catalyst to - sort the input TABLE argument by. Note that the 'partitionBy' list must also be non-empty - in this case. - select: sequence of :class:`SelectedColumn` - If non-empty, this is a sequence of expressions that the UDTF is specifying for Catalyst to - evaluate against the columns in the input TABLE argument. The UDTF then receives one input - attribute for each name in the list, in the order they are listed. - """ - schema: StructType - withSinglePartition: bool = ... - partitionBy: Sequence[PartitioningColumn] = ... - orderBy: Sequence[OrderingColumn] = ... - select: Sequence[SelectedColumn] = ... - -class SkipRestOfInputTableException(Exception): - """This represents an exception that the 'eval' method may raise to indicate that it is done - consuming rows from the current partition of the input table. Then the UDTF's 'terminate' - method runs (if any). - """ - -def _validate_arrow_udtf_handler(cls: Any, returnType: Optional[Union[StructType, str]]) -> None: - """Validate the handler class of a PyArrow UDTF.""" -def _validate_udtf_handler(cls: Any, returnType: Optional[Union[StructType, str]]) -> None: - """Validate the handler class of a UDTF.""" - -class UserDefinedTableFunction: - """User-defined table function in Python - - .. versionadded:: 3.5.0 - - Notes - ----- - The constructor of this class is not supposed to be directly called. - Use :meth:`pyspark.sql.functions.udtf` to create this instance. - - This API is evolving. - """ - def __init__(self, func: Type, returnType: Optional[Union[StructType, str]], name: Optional[str]=None, evalType: int=PythonEvalType.SQL_TABLE_UDF, deterministic: bool=False): - _validate_udtf_handler(func, returnType) - - self.func = func - self._returnType = returnType - self._returnType_placeholder: Optional[StructType] = None - self._inputTypes_placeholder = None - self._judtf_placeholder = None - self._name = name or func.__name__ - self.evalType = evalType - self.deterministic = deterministic - @property - def returnType(self) -> Optional[StructType]: - if self._returnType is None: - return None - # `_parse_datatype_string` accesses to JVM for parsing a DDL formatted string. - # This makes sure this is called after SparkContext is initialized. - if self._returnType_placeholder is None: - if isinstance(self._returnType, str): - parsed = _parse_datatype_string(self._returnType) - else: - parsed = self._returnType - if not isinstance(parsed, StructType): - raise PySparkTypeError( - errorClass="UDTF_RETURN_TYPE_MISMATCH", - messageParameters={ - "name": self._name, - "return_type": f"{parsed}", - }, - ) - self._returnType_placeholder = parsed - return self._returnType_placeholder - @property - def _judtf(self) -> 'JavaObject': - if self._judtf_placeholder is None: - self._judtf_placeholder = self._create_judtf(self.func) - return self._judtf_placeholder - def _create_judtf(self, func: Type) -> 'JavaObject': ... - def __call__(self, *args: 'TVFArgumentOrName', **kwargs: 'TVFArgumentOrName') -> 'DataFrame': ... - def asDeterministic(self) -> 'UserDefinedTableFunction': - """Updates UserDefinedTableFunction to deterministic.""" - self._judtf_placeholder = None - self.deterministic = True - return self - -class UDTFRegistration: - """Wrapper for user-defined table function registration. This instance can be accessed by - :attr:`spark.udtf` or :attr:`sqlContext.udtf`. - - .. versionadded:: 3.5.0 - """ - def __init__(self, sparkSession: 'SparkSession'): - self.sparkSession = sparkSession - def register(self, name: str, f: 'UserDefinedTableFunction') -> 'UserDefinedTableFunction': - """Register a Python user-defined table function as a SQL table function. - - .. versionadded:: 3.5.0 - - Parameters - ---------- - name : str - The name of the user-defined table function in SQL statements. - f : function or :meth:`pyspark.sql.functions.udtf` - The user-defined table function. - - Returns - ------- - function - The registered user-defined table function. - - Notes - ----- - Spark uses the return type of the given user-defined table function as the return - type of the registered user-defined function. - - To register a nondeterministic Python table function, users need to first build - a nondeterministic user-defined table function and then register it as a SQL function. - - Examples - -------- - >>> from pyspark.sql.functions import udtf - >>> @udtf(returnType="c1: int, c2: int") - ... class PlusOne: - ... def eval(self, x: int): - ... yield x, x + 1 - ... - >>> _ = spark.udtf.register(name="plus_one", f=PlusOne) - >>> spark.sql("SELECT * FROM plus_one(1)").collect() - [Row(c1=1, c2=2)] - - Use it with lateral join - - >>> spark.sql("SELECT * FROM VALUES (0, 1), (1, 2) t(x, y), LATERAL plus_one(x)").collect() - [Row(x=0, y=1, c1=0, c2=1), Row(x=1, y=2, c1=1, c2=2)] - """ - -# Names of the drop-in's module that PySpark's does not have. -from pyspark.serializers import CloudPickleSerializer as CloudPickleSerializer -from pyspark.sql.functions.builtin import arrow_udtf as arrow_udtf -from pyspark.sql.functions.builtin import udtf as udtf -SQL_TABLE_UDF: int -SQL_ARROW_TABLE_UDF: int -SQL_ARROW_UDTF: int diff --git a/python/pyspark/sql/utils.py b/python/pyspark/sql/utils.py index 9f46f6ca..6c054f50 100644 --- a/python/pyspark/sql/utils.py +++ b/python/pyspark/sql/utils.py @@ -44,7 +44,10 @@ if TYPE_CHECKING: from py4j.java_gateway import JavaObject, JVMView + from pyspark.core.context import SparkContext from pyspark.pandas._typing import IndexOpsLike, SeriesOrIndex + from pyspark.sql.dataframe import DataFrame + from pyspark.sql.session import SparkSession FuncT = TypeVar("FuncT", bound=Callable[..., Any]) @@ -63,6 +66,51 @@ def to_str(value: Any) -> Optional[str]: return str(value) +# PySpark's, of Spark Classic, which its classic pyspark.sql.streaming.readwriter imports. +class ForeachBatchFunction: + """ + This is the Python implementation of Java interface 'ForeachBatchFunction'. This wraps + the user-defined 'foreachBatch' function such that it can be called from the JVM when + the query is active. + """ + + def __init__(self, session: "SparkSession", func: Callable[["DataFrame", int], None]): + self.func = func + self.session = session + + def call(self, jdf: "JavaObject", batch_id: int) -> None: + from pyspark.sql.dataframe import DataFrame + from pyspark.sql.session import SparkSession + + try: + session_jdf = jdf.sparkSession() + # assuming that spark context is still the same between JVM and PySpark + wrapped_session_jdf = SparkSession(self.session.sparkContext, session_jdf) + self.func(DataFrame(jdf, wrapped_session_jdf), batch_id) + except Exception as e: + self.error = e + raise e + + class Java: + implements = ["org.apache.spark.sql.execution.streaming.sources.PythonForeachBatchFunction"] + + +# PySpark's, of Spark Classic, which its classic pyspark.sql.udf imports: a Connect-only client has +# no SparkContext, so `from pyspark import SparkContext` raises its ImportError. +def get_active_spark_context() -> "SparkContext": + """Raise RuntimeError if SparkContext is not initialized, + otherwise, returns the active SparkContext.""" + from pyspark import SparkContext # type: ignore[attr-defined] + + sc = SparkContext._active_spark_context + if sc is None or sc._jvm is None: + raise PySparkRuntimeError( + errorClass="SESSION_OR_CONTEXT_NOT_EXISTS", + messageParameters={}, + ) + return sc + + def to_scala_map(jvm: "JVMView", dic: dict) -> "JavaObject": """ Convert a dict into a Scala Map. diff --git a/python/tests/streaming_listener_cases.py b/python/tests/streaming_listener_cases.py index 867f9f48..788796cc 100644 --- a/python/tests/streaming_listener_cases.py +++ b/python/tests/streaming_listener_cases.py @@ -303,7 +303,7 @@ def onQueryIdle(self, event): def test_streaming_duplicate_async_listeners_share_one_decoded_event( self, mock_connect, monkeypatch, ): - from pyspark.sql.streaming import query as query_module + from pyspark.sql.connect.streaming import query as query_module spark, service = mock_connect service.listener_bus_enabled = True diff --git a/python/tests/test_dropin_class_modules_offline.py b/python/tests/test_dropin_class_modules_offline.py index cec7aa1c..490ddf42 100644 --- a/python/tests/test_dropin_class_modules_offline.py +++ b/python/tests/test_dropin_class_modules_offline.py @@ -34,6 +34,7 @@ "na": "pyspark.sql.connect.dataframe.DataFrameNaFunctions", "stat": "pyspark.sql.connect.dataframe.DataFrameStatFunctions", "groupBy": "pyspark.sql.connect.group.GroupedData", + "cogroup": "pyspark.sql.connect.group.PandasCogroupedOps", "column": "pyspark.sql.connect.column.Column", "window": "pyspark.sql.connect.window.WindowSpec", "asTable": "pyspark.sql.connect.table_arg.TableArg", @@ -66,6 +67,7 @@ def _objects(spark): "na": df.na, "stat": df.stat, "groupBy": df.groupBy(), + "cogroup": df.groupBy().cogroup(df.groupBy()), "column": F.col("id"), "window": Window.partitionBy("id"), "asTable": df.asTable(), @@ -135,6 +137,130 @@ def test_the_classic_conf_tvf_merge_and_observation_classes_are_pysparks(objects assert Observation.__module__ == "pyspark.sql.observation" +# The classic class of each object of a session, of its module of PySpark's, which pyspark.sql or +# pyspark.sql.streaming names too, as PySpark's do. +_STREAMING = "pyspark.sql.streaming" +CLASSIC = { + "catalog": ("pyspark.sql.catalog", "Catalog", "pyspark.sql"), + "read": ("pyspark.sql.readwriter", "DataFrameReader", "pyspark.sql"), + "write": ("pyspark.sql.readwriter", "DataFrameWriter", "pyspark.sql"), + "writeTo": ("pyspark.sql.readwriter", "DataFrameWriterV2", "pyspark.sql"), + "readStream": (f"{_STREAMING}.readwriter", "DataStreamReader", _STREAMING), + "writeStream": (f"{_STREAMING}.readwriter", "DataStreamWriter", _STREAMING), + "streams": (f"{_STREAMING}.query", "StreamingQueryManager", _STREAMING), + "udf": ("pyspark.sql.udf", "UDFRegistration", "pyspark.sql"), + "udtf": ("pyspark.sql.udtf", "UDTFRegistration", "pyspark.sql"), + "groupBy": ("pyspark.sql.group", "GroupedData", "pyspark.sql"), + "cogroup": ("pyspark.sql.pandas.group_ops", "PandasCogroupedOps", "pyspark.sql"), +} + + +@pytest.mark.parametrize("name", list(CLASSIC)) +def test_the_classic_classes_of_the_objects_of_a_session_are_pysparks(objects, name): + # As PySpark's: the classic modules are PySpark's, of Apache Spark as they are, whose classes + # are of the JVM of Spark Classic, and the objects of a session are of PySpark Connect's + # classes, which are not their subclasses. + import importlib + + module, qualname, package = CLASSIC[name] + classic = getattr(importlib.import_module(module), qualname) + assert f"{classic.__module__}.{classic.__qualname__}" == f"{module}.{qualname}" + assert getattr(importlib.import_module(package), qualname) is classic + assert not isinstance(objects[name], classic) + assert type(objects[name]).__module__.startswith("pyspark.sql.connect.") + assert pickle.loads(pickle.dumps(classic)) is classic + + +def test_the_classic_modules_of_the_objects_of_a_session_are_pysparks(): + # Of PySpark's own names of these modules, which the Connect classes do not have: the option + # helpers of its readers and writers, and the decorators and the mixin of its grouped data. + from pyspark.sql import group, readwriter + from pyspark.sql.pandas import group_ops + from pyspark.sql.streaming import query + from pyspark.sql.streaming import readwriter as streaming_readwriter + + assert [readwriter.to_str(v) for v in (True, False, None, 1, "a")] == [ + "true", + "false", + None, + "1", + "a", + ] + for cls in (readwriter.DataFrameReader, readwriter.DataFrameWriter): + assert cls.__mro__ == (cls, readwriter.OptionUtils, object) + assert streaming_readwriter.DataStreamReader.__mro__[1] is readwriter.OptionUtils + assert group.GroupedData.__mro__ == ( + group.GroupedData, + group_ops.PandasGroupedOpsMixin, + object, + ) + assert callable(group.dfapi) and callable(group.df_varargs_api) + assert query.StreamingCheckpointManager.__module__ == "pyspark.sql.streaming.query" + + +def test_the_udfs_and_udtfs_are_of_pyspark_connects_classes(): + # As PySpark's on Spark Connect: udf, pandas_udf, udtf and arrow_udtf make the UDFs and the + # UDTFs of PySpark Connect's classes, of pyspark.sql.connect.udf and pyspark.sql.connect.udtf, + # which are not instances of the classic classes of pyspark.sql.udf and pyspark.sql.udtf, + # which pyspark.sql.functions names, as PySpark's does. + import pandas as pd + import pyarrow as pa + import pyspark.sql.udf as classic_udf + import pyspark.sql.udtf as classic_udtf + from pyspark.sql.connect import udf as connect_udf + from pyspark.sql.connect import udtf as connect_udtf + + class Handler: + def eval(self, a): + yield (a,) + + class ArrowHandler: + def eval(self, a: pa.RecordBatch): + yield pa.table({"a": [1]}) + + for table_function in ( + F.udtf(Handler, returnType="a int"), + F.arrow_udtf(ArrowHandler, returnType="a int"), + ): + assert type(table_function) is connect_udtf.UserDefinedTableFunction + assert not isinstance(table_function, classic_udtf.UserDefinedTableFunction) + + def plus_one(s: pd.Series) -> pd.Series: + return s + 1 + + for function in (F.udf(lambda x: x), F.pandas_udf(plus_one, "long")): + assert type(function._unwrapped) is connect_udf.UserDefinedFunction + assert not isinstance(function._unwrapped, classic_udf.UserDefinedFunction) + + assert F.UserDefinedFunction is classic_udf.UserDefinedFunction + assert F.UserDefinedTableFunction is classic_udtf.UserDefinedTableFunction + for cls, module in [ + (connect_udf.UserDefinedFunction, "pyspark.sql.connect.udf"), + (connect_udtf.UserDefinedTableFunction, "pyspark.sql.connect.udtf"), + ]: + assert cls.__module__ == module + assert pickle.loads(pickle.dumps(cls)) is cls + + +def test_the_native_trigger_and_listener_bus_are_of_the_connect_modules(objects): + # The drop-in's Trigger of a stream writer, of its module of PySpark Connect, where it pickles + # by reference, and the listener bus of a session's streams, of PySpark Connect's module and + # name of the class of its bus. + from pyspark.sql.connect.streaming import readwriter + + trigger = readwriter.Trigger + assert (trigger.__module__, trigger.__qualname__) == ( + "pyspark.sql.connect.streaming.readwriter", + "Trigger", + ) + assert pickle.loads(pickle.dumps(trigger)) is trigger + bus = type(objects["streams"]._sqlb) + assert (bus.__module__, bus.__qualname__) == ( + "pyspark.sql.connect.streaming.query", + "StreamingQueryListenerBus", + ) + + def test_the_connect_window_is_a_class_of_its_own_as_pysparks(): # As PySpark Connect's Window, a subclass of pyspark.sql.window.Window, whose static methods # make the WindowSpecs of PySpark Connect. @@ -154,6 +280,9 @@ def test_the_connect_window_is_a_class_of_its_own_as_pysparks(): def test_the_streaming_query_class_is_pysparks(): + # As PySpark's: a query is of PySpark Connect's class, and the StreamingQuery of + # pyspark.sql.streaming is PySpark's classic class, which is not its base. + import pyspark.sql.streaming.query as classic from pyspark.sql.connect.streaming.query import StreamingQuery from pyspark.sql.streaming import StreamingQuery as ClassicName @@ -161,7 +290,9 @@ def test_the_streaming_query_class_is_pysparks(): "pyspark.sql.connect.streaming.query", "StreamingQuery", ) - assert ClassicName is StreamingQuery + assert ClassicName is classic.StreamingQuery + assert classic.StreamingQuery.__module__ == "pyspark.sql.streaming.query" + assert not issubclass(StreamingQuery, ClassicName) assert pickle.loads(pickle.dumps(StreamingQuery)) is StreamingQuery diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py index 703a3c4c..d1756791 100644 --- a/python/tests/test_dropin_e2e.py +++ b/python/tests/test_dropin_e2e.py @@ -6150,6 +6150,102 @@ def test_create_dataframe_converts_numpy_scalars_as_pyspark(session): assert [tuple(r) for r in df.collect()] == [(1, 0.5, True, 1)] +def test_the_objects_of_a_session_are_not_of_the_classic_classes_as_pysparks(session): + # As official PySpark 4.3.0: the classic modules are PySpark's, of the JVM of Spark Classic, + # whose classes the objects of a session of Spark Connect are not instances of; they are of + # PySpark Connect's classes. Of the na and stat functions, a WindowSpec, a table argument, a + # DataFrame and a Column, the classic names are True in both clients. + import pyspark.sql.catalog + import pyspark.sql.connect.catalog + import pyspark.sql.connect.group + import pyspark.sql.connect.readwriter + import pyspark.sql.connect.streaming.query + import pyspark.sql.connect.streaming.readwriter + import pyspark.sql.connect.udf + import pyspark.sql.connect.udtf + import pyspark.sql.group + import pyspark.sql.pandas.group_ops + import pyspark.sql.readwriter + import pyspark.sql.streaming.query + import pyspark.sql.streaming.readwriter + import pyspark.sql.udf + import pyspark.sql.udtf + from pyspark.sql import ( + Column, + DataFrame, + DataFrameNaFunctions, + DataFrameStatFunctions, + Window, + WindowSpec, + ) + from pyspark.sql import functions as F + from pyspark.sql.table_arg import TableArg + + @F.udtf(returnType="a int") + class Echo: + def eval(self, a: int): + yield (a,) + + df = session.range(3) + query = session.readStream.format("rate").load().writeStream.format("noop").start() + try: + connect, classic = pyspark.sql.connect, pyspark.sql + for obj, connect_class, classic_class in [ + (session.catalog, connect.catalog.Catalog, classic.catalog.Catalog), + (session.read, connect.readwriter.DataFrameReader, classic.readwriter.DataFrameReader), + (df.write, connect.readwriter.DataFrameWriter, classic.readwriter.DataFrameWriter), + ( + df.writeTo("t"), + connect.readwriter.DataFrameWriterV2, + classic.readwriter.DataFrameWriterV2, + ), + ( + session.readStream, + connect.streaming.readwriter.DataStreamReader, + classic.streaming.readwriter.DataStreamReader, + ), + ( + session.readStream.format("rate").load().writeStream, + connect.streaming.readwriter.DataStreamWriter, + classic.streaming.readwriter.DataStreamWriter, + ), + ( + session.streams, + connect.streaming.query.StreamingQueryManager, + classic.streaming.query.StreamingQueryManager, + ), + (query, connect.streaming.query.StreamingQuery, classic.streaming.query.StreamingQuery), + (session.udf, connect.udf.UDFRegistration, classic.udf.UDFRegistration), + (session.udtf, connect.udtf.UDTFRegistration, classic.udtf.UDTFRegistration), + (df.groupBy("id"), connect.group.GroupedData, classic.group.GroupedData), + ( + df.groupBy("id").cogroup(df.groupBy("id")), + connect.group.PandasCogroupedOps, + classic.pandas.group_ops.PandasCogroupedOps, + ), + (Echo, connect.udtf.UserDefinedTableFunction, classic.udtf.UserDefinedTableFunction), + ( + F.udf(lambda x: x)._unwrapped, + connect.udf.UserDefinedFunction, + classic.udf.UserDefinedFunction, + ), + ]: + assert isinstance(obj, connect_class), type(obj) + assert not isinstance(obj, classic_class), type(obj) + finally: + query.stop() + for obj, classic_class in [ + (df.na, DataFrameNaFunctions), + (df.stat, DataFrameStatFunctions), + (Window.partitionBy("id"), WindowSpec), + (df.asTable(), TableArg), + (df, DataFrame), + (F.col("id"), Column), + ]: + assert isinstance(obj, classic_class), type(obj) + assert [r.a for r in Echo(F.lit(7)).collect()] == [7] + + def test_catalogs_return_the_named_tuples_of_pyspark(session): # As PySpark's: the methods of the catalog return the NamedTuples of pyspark.sql.catalog, # and registerFunction registers a UDF with spark.udf.register, with its FutureWarning. diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 4d5cf660..c45166aa 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -3221,7 +3221,7 @@ def test_streaming_query_manager_wrapper_has_listener_api(): # spark.streams returns the Python StreamingQueryManager wrapper, which exposes the # listener-bus API (addListener/removeListener/close). Verify the class contract # offline (no server needed to check the wrapper class). - from pyspark.sql.streaming.query import StreamingQueryManager + from pyspark.sql.connect.streaming.query import StreamingQueryManager for m in ["addListener", "removeListener", "close"]: assert hasattr(StreamingQueryManager, m), f"StreamingQueryManager missing {m}" @@ -3255,7 +3255,7 @@ def test_parquet_sets_only_its_options_as_pyspark(mock_connect): def test_udf_construction_and_call(): from pyspark.sql import functions as F - from pyspark.sql.udf import UserDefinedFunction + from pyspark.sql.connect.udf import UserDefinedFunction # As PySpark's: (func, returnType, name, evalType, deterministic). u = UserDefinedFunction(lambda x: x + 1, T.IntegerType(), "plus1", 100) assert u._name == "plus1" and u._wrapped().__name__ == "plus1" @@ -3462,7 +3462,7 @@ def test_udfs_are_created_as_pyspark(mock_connect): import pandas as pd import pyarrow as pa from pyspark.sql import functions as F - from pyspark.sql.udf import UserDefinedFunction + from pyspark.sql.connect.udf import UserDefinedFunction spark, service = mock_connect arrow_conf = "spark.sql.execution.pythonUDF.arrow.enabled" @@ -6639,15 +6639,15 @@ def __str__(self): def test_udf_registration_has_java_methods(): # The UDFRegistration class (what spark.udf returns) exposes the Java methods. - from pyspark.sql.udf import UDFRegistration + from pyspark.sql.connect.udf import UDFRegistration for m in ("register", "registerJavaFunction", "registerJavaUDAF"): assert hasattr(UDFRegistration, m), f"UDFRegistration missing {m}" def test_udf_udtf_determinism_and_extras(mock_connect): from pyspark.sql import functions as F - from pyspark.sql.udf import UserDefinedFunction, udf - from pyspark.sql.udtf import udtf + from pyspark.sql.connect.udf import UserDefinedFunction, udf + from pyspark.sql.connect.udtf import udtf # As PySpark's, asNondeterministic() of the function wrapper of a UDF marks the UDF # nondeterministic, in place, and returns a wrapper of it. @@ -6675,7 +6675,8 @@ def eval(self, n): def test_catalog_result_classes(): # As PySpark's: the catalog metadata result classes are the NamedTuples of # pyspark.sql.catalog, and the Table `database` property derives from a single-element - # namespace. The Catalog of a session is PySpark Connect's, of pyspark.sql.connect.catalog. + # namespace. The Catalog of a session is PySpark Connect's, of pyspark.sql.connect.catalog, + # and that of pyspark.sql.catalog is PySpark's classic class. from pyspark.sql.catalog import ( Catalog, CatalogMetadata, @@ -6685,7 +6686,11 @@ def test_catalog_result_classes(): Table, TablePartition, ) - assert Catalog.__module__ == "pyspark.sql.connect.catalog" + from pyspark.sql.connect.catalog import Catalog as ConnectCatalog + + assert Catalog.__module__ == "pyspark.sql.catalog" + assert ConnectCatalog.__module__ == "pyspark.sql.connect.catalog" + assert ConnectCatalog is not Catalog for cls in (CatalogMetadata, Database, Table, Column, Function, TablePartition): assert cls.__module__ == "pyspark.sql.catalog", f"{cls.__name__} module = {cls.__module__}" assert issubclass(cls, tuple) @@ -6758,7 +6763,8 @@ def test_udtf_analyze_result_subclass_preserves_custom_state(): from pyspark.serializers import CloudPickleSerializer from pyspark.sql.connect.udtf import AnalyzeResult as ConnectAnalyzeResult - from pyspark.sql.udtf import AnalyzeResult, udtf + from pyspark.sql.connect.udtf import udtf + from pyspark.sql.udtf import AnalyzeResult assert ConnectAnalyzeResult is AnalyzeResult @@ -6789,7 +6795,7 @@ def analyze(argument): def eval(self, argument): yield (self.buffer,) - from pyspark.sql.udtf import _pickled_handler + from pyspark.sql.connect.udtf import _pickled_handler function = udtf(Handler, useArrow=False) # The handler class as it is sent, pickled, as PySpark Connect's PythonUDTF.command. @@ -7207,7 +7213,7 @@ def test_partition_transforms_are_those_of_pyspark_and_the_others_deprecated(): def test_udtf_construction(): - from pyspark.sql.udtf import UserDefinedTableFunction, udtf + from pyspark.sql.connect.udtf import UserDefinedTableFunction, udtf @udtf(returnType="a int") class Echo: @@ -7235,7 +7241,7 @@ def test_udtf_register_sends_the_udtf_as_pyspark(mock_connect): from pyspark.errors import PySparkTypeError from pyspark.serializers import CloudPickleSerializer from pyspark.sql import functions as F - from pyspark.sql.udtf import UDTFRegistration, UserDefinedTableFunction + from pyspark.sql.connect.udtf import UDTFRegistration, UserDefinedTableFunction spark, service = mock_connect @@ -7281,8 +7287,8 @@ def test_udf_and_udtf_pickle_as_pyspark_connect_classes(): from pyspark.serializers import CloudPickleSerializer from pyspark.sql import functions as F + from pyspark.sql.connect.udf import UserDefinedFunction from pyspark.sql.types import IntegerType - from pyspark.sql.udf import UserDefinedFunction def class_references(obj): # The strings of the pickle, which name the classes that it refers to by reference. @@ -7394,7 +7400,7 @@ def test_udtf_return_type_connect_validation_and_command(): subprocess.run([sys.executable, "-c", """ from pyspark.sql import SparkSession, DataFrame, functions as F, types as T from pyspark.sql.connect.types import UnparsedDataType -from pyspark.sql.udtf import _pickled_handler +from pyspark.sql.connect.udtf import _pickled_handler from pyspark.serializers import CloudPickleSerializer # No server listens: with useArrow, udtf() does not read the Arrow config of the session. @@ -7604,7 +7610,7 @@ def test_udf_defaults_and_registration(): # register_udf wires to the session's _registerPythonUdf (so the name resolves in SQL), and # returns the UDF. from pyspark.sql.connect.client import SparkConnectClient - from pyspark.sql.udf import UDFRegistration + from pyspark.sql.connect.udf import UDFRegistration class _StubSession: def __init__(self): @@ -9161,7 +9167,7 @@ def test_udf_with_defaults(): def test_udtf_variants(): """Test UDTF builder methods.""" - from pyspark.sql.udtf import UserDefinedTableFunction + from pyspark.sql.connect.udtf import UserDefinedTableFunction # Create a simple UDTF for testing; not Arrow-optimized whatever the active session. @__import__("pyspark.sql.functions", fromlist=["udtf"]).udtf( @@ -14689,7 +14695,8 @@ def test_the_streaming_package_is_pysparks(): # As PySpark's, pyspark.sql.streaming is PySpark's file, vendored as it is, which has no # __all__: a star import brings in its classes and its loaded modules, `state` among them, as # the drop-in's pyspark.sql.pandas.group_ops imports it as PySpark's does. Trigger, a name of - # the drop-in's readwriter, and the read limits of its datasource module are not its names. + # the drop-in's connect readwriter, and the read limits of its datasource module are not its + # names. import importlib import subprocess import sys @@ -14726,15 +14733,16 @@ def test_the_streaming_package_is_pysparks(): importlib.import_module(f"pyspark.sql.streaming.{module}"), name), name assert streaming.StreamingQueryException is StreamingQueryException for name, module in [ - ("Trigger", "readwriter"), ("ReadLimit", "datasource"), ("ReadAllAvailable", "datasource"), - ("ReadMinRows", "datasource"), ("ReadMaxRows", "datasource"), - ("ReadMaxFiles", "datasource"), ("ReadMaxBytes", "datasource"), - ("SupportsTriggerAvailableNow", "datasource"), + ("Trigger", "connect.streaming.readwriter"), ("ReadLimit", "streaming.datasource"), + ("ReadAllAvailable", "streaming.datasource"), ("ReadMinRows", "streaming.datasource"), + ("ReadMaxRows", "streaming.datasource"), ("ReadMaxFiles", "streaming.datasource"), + ("ReadMaxBytes", "streaming.datasource"), + ("SupportsTriggerAvailableNow", "streaming.datasource"), ]: assert not hasattr(streaming, name), name with pytest.raises(ImportError, match=f"cannot import name '{name}' from 'pyspark.sql.streaming'"): exec(f"from pyspark.sql.streaming import {name}", {}) - assert hasattr(importlib.import_module(f"pyspark.sql.streaming.{module}"), name), name + assert hasattr(importlib.import_module(f"pyspark.sql.{module}"), name), name def test_module_namespaces_of_the_official_doctests_as_pyspark(): @@ -16506,9 +16514,9 @@ def test_nanosecond_timestamps_are_not_collected_as_pyspark( def test_udf_and_pandas_udf_decorator_forms(mock_connect): # @udf(returnType) / @pandas_udf(returnType) return a decorator that wraps a function. + from pyspark.sql.connect.udf import UserDefinedFunction from pyspark.sql.functions import pandas_udf, udf from pyspark.sql.types import IntegerType - from pyspark.sql.udf import UserDefinedFunction # As PySpark's, the function wrapper of the UDF. @udf(IntegerType()) @@ -24842,7 +24850,7 @@ def test_numpy_longdouble_array_matches_dtype_and_element_checks_as_pyspark(): ]) def test_user_defined_function_constructor_checks_its_arguments_as_pyspark(args, kwargs, message): from pyspark.errors import PySparkTypeError - from pyspark.sql.udf import UserDefinedFunction + from pyspark.sql.connect.udf import UserDefinedFunction with pytest.raises(PySparkTypeError, match=re.escape(message)) as error: UserDefinedFunction(*args, **kwargs) diff --git a/python/tests/test_dropin_udf_args_offline.py b/python/tests/test_dropin_udf_args_offline.py index 742bad84..0b787b21 100644 --- a/python/tests/test_dropin_udf_args_offline.py +++ b/python/tests/test_dropin_udf_args_offline.py @@ -118,7 +118,7 @@ def test_udf_names_and_keywords_that_utf8_cannot_encode_fail_when_the_plan_is_bu def test_udtf_has_no_name_until_it_is_set(udf_session): - from pyspark.sql.udtf import UserDefinedTableFunction, udtf + from pyspark.sql.connect.udtf import UserDefinedTableFunction, udtf @udtf(returnType="x int") class Named: @@ -142,7 +142,7 @@ def eval(self, a): def test_udtf_name_setter_and_pickling_are_unchanged(udf_session): import cloudpickle - from pyspark.sql.udtf import udtf + from pyspark.sql.connect.udtf import udtf @udtf(returnType="x int") class Pickled: @@ -156,10 +156,15 @@ def eval(self, a): assert restored.name == "other" and restored._name == "Pickled" -def test_classic_names_are_the_connect_classes(udf_session): - # Deliberate, see docs/compatibility.md: the classic names are the drop-in's own classes. - from pyspark.sql.udf import UserDefinedFunction, udf - from pyspark.sql.udtf import UserDefinedTableFunction, udtf +def test_the_classic_names_are_pysparks_classic_classes(udf_session): + # As PySpark's, see docs/compatibility.md: the UDFs and the UDTFs of udf() and udtf() are of + # PySpark Connect's classes, and the classic names of pyspark.sql.udf and pyspark.sql.udtf + # are PySpark's classic classes, which are not their bases. + import pyspark.sql.udf as classic_udf + import pyspark.sql.udtf as classic_udtf + from pyspark.sql.connect.udf import UserDefinedFunction + from pyspark.sql.connect.udtf import UserDefinedTableFunction + from pyspark.sql.functions import udf, udtf @udtf(returnType="x int") class Classic: @@ -167,5 +172,63 @@ def eval(self, a): yield (a,) assert type(Classic) is UserDefinedTableFunction + assert not isinstance(Classic, classic_udtf.UserDefinedTableFunction) # A UDF of udf() is its function wrapper, which keeps the UserDefinedFunction it wraps. assert type(udf(lambda x: x)._unwrapped) is UserDefinedFunction + assert not isinstance(udf(lambda x: x)._unwrapped, classic_udf.UserDefinedFunction) + + +def test_udf_and_udtf_raise_an_error_of_the_session_other_than_no_session_as_pyspark( + udf_session, monkeypatch +): + # As PySpark Connect's udf() and udtf() without useArrow, which read the Arrow configuration + # of the active session: without a session they make a UDF and a UDTF that are not + # Arrow-optimized, and another error of the session is raised. + from pyspark.errors import PySparkRuntimeError + from pyspark.sql import SparkSession + from pyspark.sql.functions import udf, udtf + + class Echo: + def eval(self, a): + yield (a,) + + def raising(condition): + def active(): + raise PySparkRuntimeError(errorClass=condition, messageParameters={}) + + return staticmethod(active) + + monkeypatch.setattr(SparkSession, "active", raising("NO_ACTIVE_OR_DEFAULT_SESSION")) + assert udf(lambda x: x).evalType == 100 # SQL_BATCHED_UDF + assert udtf(Echo, returnType="a int").evalType == 300 # SQL_TABLE_UDF + monkeypatch.setattr(SparkSession, "active", raising("SESSION_OR_CONTEXT_NOT_EXISTS")) + for make in (lambda: udf(lambda x: x), lambda: udtf(Echo, returnType="a int")): + with pytest.raises(PySparkRuntimeError) as error: + make() + assert error.value.getCondition() == "SESSION_OR_CONTEXT_NOT_EXISTS" + + +def test_a_udtf_of_no_return_type_sends_no_return_type_as_pyspark(udf_session): + # As PySpark Connect's PythonUDTF.to_plan: the PythonUDTF of a UDTF of no return type, whose + # analyze gives the schema, has no return_type, and the command of the handler. + from pyspark.serializers import CloudPickleSerializer + from pyspark.sql import types as T + from pyspark.sql.connect.proto import base_pb2 + from pyspark.sql.functions import lit, udtf + from pyspark.sql.udtf import AnalyzeResult + + _, service = udf_session + + class Dynamic: + @staticmethod + def analyze(a): + return AnalyzeResult(T.StructType().add("a", "int")) + + def eval(self, a): + yield (a,) + + udtf(Dynamic, useArrow=False)(lit(1)).explain() + request = base_pb2.AnalyzePlanRequest.FromString(service.analyze_requests[-1]) + python_udtf = request.explain.plan.root.common_inline_user_defined_table_function.python_udtf + assert not python_udtf.HasField("return_type") + assert CloudPickleSerializer().loads(python_udtf.command).__name__ == "Dynamic" diff --git a/python/tests/test_type_stubs_offline.py b/python/tests/test_type_stubs_offline.py index 06f46b6c..07facb78 100644 --- a/python/tests/test_type_stubs_offline.py +++ b/python/tests/test_type_stubs_offline.py @@ -56,7 +56,6 @@ "_make_type_verifier", "_drop_metadata", ], - "pyspark.sql.udf": ["_create_udf", "_create_py_udf"], "pyspark.util": ["_parse_memory"], } @@ -71,17 +70,10 @@ TYPE_ONLY = { "pyspark.sql.dataframe.DataFrame.rdd": _CLASSIC_ONLY, "pyspark.sql.connect.dataframe.DataFrame.rdd": _CLASSIC_ONLY, - "pyspark.sql.group.dfapi": "the decorator of PySpark's GroupedData, which types its methods", - "pyspark.sql.group.df_varargs_api": "the decorator of PySpark's GroupedData, as dfapi", - "pyspark.sql.pandas.group_ops.PandasGroupedOpsMixin": ( - "the base of PySpark's GroupedData, of its pandas and Arrow methods, which the drop-in's " - "native GroupedData has itself" + "pyspark.sql.connect.readwriter.OptionUtils": ( + "the base of PySpark Connect's readers and writers, of private members only; the " + "drop-in's readers and writers are native" ), - "pyspark.sql.readwriter.OptionUtils": ( - "the base of PySpark's readers and writers, of private members only; the drop-in's " - "readers and writers are native" - ), - "pyspark.sql.connect.readwriter.OptionUtils": "as pyspark.sql.readwriter.OptionUtils", "pyspark.sql.types.T": "a type variable of PySpark's annotations of the module", "pyspark.sql.types.U": "a type variable of PySpark's annotations of the module", "pyspark.sql.connect.streaming.query.StreamingQueryListenerBus": ( @@ -98,7 +90,6 @@ "class, which type checkers see as with pyspark-client, does not have" ) UNDECLARED_MEMBERS = { - "pyspark.sql.streaming.query.StreamingQueryManager.close": _CONNECT_CLASS, "pyspark.sql.table_arg.TableArg.to_plan": _CONNECT_CLASS, } @@ -113,7 +104,7 @@ "pyspark.sql.types.DecimalType.precision": "int", "pyspark.sql.types.GeometryType.srid": "int", "pyspark.storagelevel.StorageLevel.useDisk": "bool", - "pyspark.sql.udf.UserDefinedFunction.func": "def (*Any, **Any) -> Any", + "pyspark.sql.connect.udf.UserDefinedFunction.func": "def (*Any, **Any) -> Any", "pyspark.sql.connect.session.SparkSession.release_session_on_close": "bool", } @@ -128,7 +119,7 @@ "pyspark.sql.types.DecimalType.precision": "int", "pyspark.sql.types.GeometryType.srid": "int", "pyspark.storagelevel.StorageLevel.useDisk": "bool", - "pyspark.sql.udf.UserDefinedFunction.func": "(...) -> Any", + "pyspark.sql.connect.udf.UserDefinedFunction.func": "(...) -> Any", "pyspark.sql.connect.session.SparkSession.release_session_on_close": "bool", } diff --git a/scripts/check_vendored_upstream.sh b/scripts/check_vendored_upstream.sh index 1ea57429..298588d9 100644 --- a/scripts/check_vendored_upstream.sh +++ b/scripts/check_vendored_upstream.sh @@ -151,6 +151,18 @@ VENDORED_FILES=( # Connect returns those of pyspark.sql.connect.context, and their constructors read the # SparkContext of a JVM, which a session of Spark Connect does not have. "sql/context.py" + # PySpark's classic modules of the objects of a session (its Catalog, readers and writers, UDF + # and UDTF classes and registrations, GroupedData, PandasCogroupedOps and the streaming query + # and readwriter), as in PySpark: the objects of a session of Spark Connect are of the classes + # of pyspark.sql.connect, the drop-in's, which are not subclasses of these, as in PySpark. + "sql/catalog.py" + "sql/readwriter.py" + "sql/udf.py" + "sql/udtf.py" + "sql/group.py" + "sql/pandas/group_ops.py" + "sql/streaming/readwriter.py" + "sql/streaming/query.py" # PEP 561: the package is typed, and the type aliases of PySpark's stubs. "py.typed" diff --git a/scripts/gen_type_stubs.py b/scripts/gen_type_stubs.py index 10b4adaf..a249c976 100644 --- a/scripts/gen_type_stubs.py +++ b/scripts/gen_type_stubs.py @@ -46,8 +46,8 @@ any other condition, such as ``is_remote_only()``, both branches are kept, as a type checker of PySpark's source sees both. * The decorators that type checkers apply are kept (``overload``, ``property``, - ``classmethod``, ``functools.cache``, ``dataclass``, ...), and those of the module itself - (``dfapi`` of ``pyspark.sql.group``). PySpark's call-site and dispatch decorators are typed + ``classmethod``, ``functools.cache``, ``dataclass``, ...), and those that the module defines + itself. PySpark's call-site and dispatch decorators are typed ``(f: FuncT) -> FuncT``, which ``IDENTITY_DECORATORS`` lists and which the generator checks in PySpark's source: they change no type and are dropped. Any other decorator is an error. * The implementation of an overloaded function is dropped, as stubs declare only overloads. @@ -73,10 +73,11 @@ lines of a signature that ast.unparse prints on one line merged. ``EXTRAS`` adds the names that the drop-in's module has and PySpark's does not, such as the -native ``Trigger`` of ``pyspark.sql.streaming.readwriter``. python/tests/test_type_stubs_offline.py -checks that every public name of each module of the drop-in, and every public member of each class -of a stub, is in its stub, that mypy and pyright type every public instance attribute of the -stubbed classes, and runs mypy and pyright on a user's application. +native ``Trigger`` of ``pyspark.sql.connect.streaming.readwriter``. +python/tests/test_type_stubs_offline.py checks that every public name of each module of the +drop-in, and every public member of each class of a stub, is in its stub, that mypy and pyright +type every public instance attribute of the stubbed classes, and runs mypy and pyright on a user's +application. The modules of pyspark.ml.connect of the drop-in's native ML classes (``ML_CONNECT_MODULES``) and their package are not PySpark's modules of their import paths, which are its ML API on PyTorch: @@ -134,21 +135,17 @@ # the drop-in whose objects a user imports. The stub is at the path of PySpark's module, with # .pyi for .py (pyspark/sql/functions/__init__.pyi for the package pyspark.sql.functions). STUB_MODULES = ( - # The modules of the native classes, which re-export them from pyspark._pyspark. Not - # pyspark.sql.conf, merge, observation and tvf, whose classic classes are PySpark's annotated - # modules, vendored as they are (scripts/check_vendored_upstream.sh), which a stub would shadow. + # The modules of the native classes, which re-export them from pyspark._pyspark. Not the + # modules of the classic classes of PySpark, such as pyspark.sql.catalog, readwriter, udf, + # udtf, group, conf, merge, observation and tvf, pyspark.sql.pandas.group_ops and the streaming + # query and readwriter, PySpark's annotated modules, vendored as they are + # (scripts/check_vendored_upstream.sh), which a stub would shadow. "pyspark.storagelevel", - "pyspark.sql.catalog", "pyspark.sql.column", "pyspark.sql.dataframe", - "pyspark.sql.group", - "pyspark.sql.pandas.group_ops", - "pyspark.sql.readwriter", "pyspark.sql.table_arg", "pyspark.sql.types", "pyspark.sql.window", - "pyspark.sql.streaming.query", - "pyspark.sql.streaming.readwriter", # The functions: those that functions.py copies into its globals() from # functions_generated.py, and those of Avro and Protobuf, unannotated calls of the Rust core. "pyspark.sql.functions", @@ -160,11 +157,8 @@ # pyspark/sql/connect/functions/_copies.py), which copy them into their globals(). "pyspark.sql.connect.avro.functions", "pyspark.sql.connect.protobuf.functions", - # PySpark's modules with their Rust-backed parts swapped in: the native PythonEvalType and - # SkipRestOfInputTableException, and user-defined functions that annotate fewer parameters. + # PySpark's module with its Rust-backed part swapped in: the native PythonEvalType. "pyspark.util", - "pyspark.sql.udf", - "pyspark.sql.udtf", # The modules of PySpark Connect, which re-export the drop-in's classes and functions. "pyspark.sql.connect.catalog", "pyspark.sql.connect.column", @@ -278,11 +272,12 @@ class Extra: ), # The drop-in defines udf, udtf and arrow_udtf of pyspark.sql.functions in these modules, # and their evaluation types. - "pyspark.sql.udf": Extra( + "pyspark.sql.connect.udf": Extra( "from pyspark.serializers import CloudPickleSerializer as CloudPickleSerializer\n" "from pyspark.sql.functions.builtin import udf as udf\n" ), - "pyspark.sql.udtf": Extra( + "pyspark.sql.connect.udtf": Extra( + "from pyspark.errors import PySparkPicklingError as PySparkPicklingError\n" "from pyspark.serializers import CloudPickleSerializer as CloudPickleSerializer\n" "from pyspark.sql.functions.builtin import arrow_udtf as arrow_udtf\n" "from pyspark.sql.functions.builtin import udtf as udtf\n" @@ -290,17 +285,12 @@ class Extra: "SQL_ARROW_TABLE_UDF: int\n" "SQL_ARROW_UDTF: int\n" ), - # The drop-in's module dispatches the events of the listeners itself. - "pyspark.sql.streaming.query": Extra( - "from pyspark.sql.streaming.listener import QueryIdleEvent as QueryIdleEvent\n" - "from pyspark.sql.streaming.listener import QueryProgressEvent as QueryProgressEvent\n" - "from pyspark.sql.streaming.listener import QueryStartedEvent as QueryStartedEvent\n" - "from pyspark.sql.streaming.listener import QueryTerminatedEvent as QueryTerminatedEvent\n" - ), # The native classes of the frame bounds, which PySpark's `__all__` does not list as the # drop-in's does not, and of the triggers. "pyspark.sql.window": Extra("class FrameBound: ...\n"), - "pyspark.sql.streaming.readwriter": Extra( + "pyspark.sql.connect.streaming.readwriter": Extra( + "from pyspark.errors import PySparkAttributeError as PySparkAttributeError\n" + "from pyspark.errors import PySparkRuntimeError as PySparkRuntimeError\n" "class Trigger:\n" " @staticmethod\n" " def processingTime(interval: str) -> Trigger: ...\n" @@ -310,7 +300,6 @@ class Extra: " def availableNow() -> Trigger: ...\n" " @staticmethod\n" " def continuous(interval: str) -> Trigger: ...\n", - ("Trigger",), ), } @@ -338,9 +327,6 @@ class Extra: UNRESOLVED_IMPORTS = { "pyspark.core.context.SparkContext": _CLASSIC, "pyspark.core.rdd.RDD": _CLASSIC, - # PySpark's streaming readers and writers import it from its readwriter for the bodies of - # their option methods, which the stubs keep; the drop-in's readwriter has no to_str. - "pyspark.sql.readwriter.to_str": "a helper that only the kept bodies of methods call", "pyspark.sql.utils.StringConcat": ( "the helper of the private tree strings of PySpark's classic types, which the drop-in's " "native types build in Rust" diff --git a/tests/golden/stub_docstrings.json b/tests/golden/stub_docstrings.json index f34eb787..f8c79605 100644 --- a/tests/golden/stub_docstrings.json +++ b/tests/golden/stub_docstrings.json @@ -13,7 +13,6 @@ "pyspark.ml.connect.tuning": "f58b0ec154f78b2e0f30cc5ff90ae44ca2bd4caa63de786a872161b66f8bec35", "pyspark.ml.stat": "a0a855bb79190896cb4de3bae95fd1a59d11756e3618b3a81184a4f60fd9cbe9", "pyspark.sql.avro.functions": "ca637d002e041b77e0ad17b70f0aa1b1ba605843d3d0da0000bc174a5e4b64b0", - "pyspark.sql.catalog": "9adc35231c662855b9aeff08a4fade9e6af804639f1692e4a3dbc7f0bed570e5", "pyspark.sql.column": "c6784aa594290986d8370c4e53c0ae605ae28d4c5bbb499122ba3e82dfeb4185", "pyspark.sql.connect.avro.functions": "ca637d002e041b77e0ad17b70f0aa1b1ba605843d3d0da0000bc174a5e4b64b0", "pyspark.sql.connect.catalog": "a3c520e2b37963de540359a698b18b442848a18b15d24fd679df434405e17fbf", @@ -35,23 +34,16 @@ "pyspark.sql.connect.streaming.readwriter": "7fb5c1aab11410bd597b8ae843c5c866d6aca794b2db00361b52a959688812ff", "pyspark.sql.connect.table_arg": "e2524a3518e803d1e30b3c39a8cd0af376b7f1d3f52abd5298bf5403ad7ddf42", "pyspark.sql.connect.tvf": "bd2e8a7a4da26c0a85fe6f9cb2ac24328d5916dc4d5f331859732bbedeb6ec2f", - "pyspark.sql.connect.udf": "ed87bac9f0c10895f3f9e57648c0ac5c135c8cfc31a6671232314d19a9cffa00", - "pyspark.sql.connect.udtf": "a4773dfbfa7e53f7d5b3fc2dd62b5e1439308e655b7e28728a713e610d3a1d09", + "pyspark.sql.connect.udf": "a068c27949215576411d15b9e32d30827cab026f1b35291c759d94d4dbc2e740", + "pyspark.sql.connect.udtf": "878e044f1f0f93b5c2896aa22b05acddafef62f373e4fdae7733dd9d5d7adbe6", "pyspark.sql.connect.window": "d28e54a93ed476fbe681d0aa1e8c120936fe9e2718e99a36dc18df7841dbe1ad", "pyspark.sql.dataframe": "601e48dfdae3c5a38b3018a6d6556b628d2ca637c1a575dc53f32ed0512c1942", "pyspark.sql.functions": "02c0cb0f59f1872f7657ceff1376c71e7c8e5a6855e48b8531505ebe46738ac8", "pyspark.sql.functions.builtin": "a5662b7eba3266ddb3e7654fc84c29de5afdc01fd29338dafd9dee025ac4f99b", "pyspark.sql.functions.partitioning": "a18e916bbda260fe75a1f604273381d28923d25ab5185c92d9e03cf3a863152f", - "pyspark.sql.group": "56f9e084d3dc0f3a8d258db758131197eb29047c3d02b08d7fb5efd7fba6f823", - "pyspark.sql.pandas.group_ops": "fa099e94fba597ca8f2b63cfed7af0ec4e4fcde4e9bf8dc7a13adb30a1f11503", "pyspark.sql.protobuf.functions": "6cc39a35eeb443085292cc7c3d48700d27ec76c318e63cac062439199ec8c6a7", - "pyspark.sql.readwriter": "686a96709562f5f0444de9cf7612d3985a1b38b54dd8ea40053882a631cec7ac", - "pyspark.sql.streaming.query": "8b40269b06044ac92511d8ae89986989d20dc0cccb9b9940904749ac673d0429", - "pyspark.sql.streaming.readwriter": "8f7956cb8eadfdad7e6e7bbdc72c348efc5c8944d62809902ac5afee6e4d3e62", "pyspark.sql.table_arg": "2602b912c9c39402bbd500d612637908cd1ae965192e941c624423fd89aa96d1", "pyspark.sql.types": "5deb90a7916fd57ed43a2a520d7d32464f4b8f9331517b392d986507e1da59d3", - "pyspark.sql.udf": "41e118b4175273f1484bec456989158b08c4f1992a2afd8d8e1a4ae2fb02602a", - "pyspark.sql.udtf": "1bb8ae3d1415561ea982563185d1dfafc3eceaaa86680fed0d9a02b37cefe07a", "pyspark.sql.window": "acbfa895e964528eb717d617136176be8cea636992de0a1fe6fa4eb1e59ee268", "pyspark.storagelevel": "382513d837b7d101b0873674e27ef68fa72a5a50e2059e8e08715fc685aebb9a", "pyspark.util": "6dc21335d9af9e018b049c05a2d9a55755e8a17270776fb5db0ffac2ad7354ff" @@ -64,13 +56,12 @@ "pyspark.ml.connect:MLModel", "pyspark.ml.connect:MLModel.transform", "pyspark.sql.connect.dataframe:DataFrame.rdd", + "pyspark.sql.connect.streaming.readwriter:Trigger", + "pyspark.sql.connect.streaming.readwriter:Trigger.availableNow", + "pyspark.sql.connect.streaming.readwriter:Trigger.continuous", + "pyspark.sql.connect.streaming.readwriter:Trigger.once", + "pyspark.sql.connect.streaming.readwriter:Trigger.processingTime", "pyspark.sql.dataframe:DataFrame.rdd", - "pyspark.sql.pandas.group_ops:PandasGroupedOpsMixin.__transformWithState", - "pyspark.sql.streaming.readwriter:Trigger", - "pyspark.sql.streaming.readwriter:Trigger.availableNow", - "pyspark.sql.streaming.readwriter:Trigger.continuous", - "pyspark.sql.streaming.readwriter:Trigger.once", - "pyspark.sql.streaming.readwriter:Trigger.processingTime", "pyspark.sql.window:FrameBound" ], "samples": {