diff --git a/crates/pyspark-rs/src/dataframe.rs b/crates/pyspark-rs/src/dataframe.rs index dbccbe2e..1d4500c3 100644 --- a/crates/pyspark-rs/src/dataframe.rs +++ b/crates/pyspark-rs/src/dataframe.rs @@ -3734,7 +3734,7 @@ impl PyDataFrame { fn write_to(slf: &Bound<'_, Self>, table: &Bound<'_, PyAny>) -> PyDataFrameWriterV2 { let (name, unsent_table) = written_table(table); let frame = slf.clone().unbind(); - PyDataFrameWriterV2 { + let mut writer = PyDataFrameWriterV2 { inner: std::sync::Arc::new(WriterState::of_frame( slf.borrow().dataframe.write_to(&name), &frame, @@ -3747,8 +3747,11 @@ impl PyDataFrame { unsent_partition: std::sync::Arc::new(std::sync::Mutex::new(None)), unsent_options: std::sync::Arc::new(std::sync::Mutex::new(Default::default())), unsent_properties: std::sync::Arc::new(std::sync::Mutex::new(Default::default())), + shared: None, gc_ready: crate::gc::GcReady::new(), - } + }; + writer.shared = writer.shared_of(); + writer } /// The streaming writer interface (`df.writeStream`). @@ -5125,11 +5128,127 @@ pub struct PyDataFrameWriter { /// The format, the partitioning columns and the save mode that PySpark keeps as they are, /// and whose `WriteOperation.command` converts, or resolves, at each write ([`UnsentArgs`]). unsent: std::sync::Arc>, + /// The Python object of `unsent` and `refused_bucket_count`, which the copies of this writer + /// share too, and which reports the objects that they keep to Python's garbage collector, as + /// the `_write` of PySpark's copies holds them (see [`WriterKeptArgs`]). + shared: Option>, /// Set once the value is written, as `__traverse__` reads `unsent`, which is not valid while /// the writer object is created (see [`crate::gc::GcReady`]). gc_ready: crate::gc::GcReady, } +/// The arguments that a V1 writer keeps as they are, which the copies of the writer share, as the +/// `_write` of PySpark's copies is one object: their Python objects are reported to Python's +/// garbage collector by the Python object of them that each copy holds (see +/// [`crate::options::SharedObjects`]). +struct WriterKeptArgs { + unsent: std::sync::Arc>, + refused_bucket_count: std::sync::Arc>>>, +} + +impl crate::options::SharedObjects for WriterKeptArgs { + fn visit(&self, visit: &pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> { + if let Ok(unsent) = self.unsent.try_lock() { + for object in [ + &unsent.format, + &unsent.partition, + &unsent.clustering, + &unsent.mode, + &unsent.sort, + &unsent.bucket, + ] + .into_iter() + .flatten() + { + visit.call(object)?; + } + unsent.options.visit_objects(visit)?; + } + if let Ok(count) = self.refused_bucket_count.try_lock() { + if let Some(count) = &*count { + visit.call(count)?; + } + } + Ok(()) + } + + fn clear(&self) { + let unsent = self + .unsent + .try_lock() + .map(|mut unsent| std::mem::take(&mut *unsent)); + drop(unsent); + let count = self + .refused_bucket_count + .try_lock() + .map(|mut count| count.take()); + drop(count); + } +} + +/// The arguments that a V2 writer keeps as they are, which the copies of the writer share, as the +/// `_write` of PySpark's copies is one object (see [`WriterKeptArgs`]). +struct WriterV2KeptArgs { + kept: [std::sync::Arc>>>; 3], + options: [std::sync::Arc>; 2], +} + +impl crate::options::SharedObjects for WriterV2KeptArgs { + fn visit(&self, visit: &pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> { + for kept in &self.kept { + if let Ok(kept) = kept.try_lock() { + if let Some(object) = &*kept { + visit.call(object)?; + } + } + } + for options in &self.options { + if let Ok(options) = options.try_lock() { + options.visit_objects(visit)?; + } + } + Ok(()) + } + + fn clear(&self) { + let kept: Vec<_> = self + .kept + .iter() + .filter_map(|kept| kept.try_lock().ok().and_then(|mut kept| kept.take())) + .collect(); + drop(kept); + let options: Vec<_> = self + .options + .iter() + .filter_map(|options| { + options + .try_lock() + .ok() + .map(|mut options| std::mem::take(&mut *options)) + }) + .collect(); + drop(options); + } +} + +impl PyDataFrameWriterV2 { + /// The Python object of the arguments that this writer keeps, for a new writer that holds + /// them (see [`WriterV2KeptArgs`]). + fn shared_of(&self) -> Option> { + crate::options::PySharedObjects::of(std::sync::Arc::new(WriterV2KeptArgs { + kept: [ + std::sync::Arc::clone(&self.unsent_provider), + std::sync::Arc::clone(&self.unsent_clustering), + std::sync::Arc::clone(&self.unsent_partition), + ], + options: [ + std::sync::Arc::clone(&self.unsent_options), + std::sync::Arc::clone(&self.unsent_properties), + ], + })) + } +} + /// The arguments of a writer that PySpark keeps as they are, and that its `WriteOperation.command` /// converts at the write, which fails with the first one that does not convert. Each one is /// replaced by the next call that sets the same argument, as PySpark's attribute is. @@ -5161,15 +5280,21 @@ impl PyDataFrameWriter { writer: spark_connect::readwriter::DataFrameWriter, frame: Py, ) -> Self { + let refused_bucket_count = std::sync::Arc::new(std::sync::Mutex::new(None)); + let unsent = std::sync::Arc::new(std::sync::Mutex::new(UnsentArgs::default())); PyDataFrameWriter { inner: std::sync::Arc::new(WriterState::of_frame( writer, &frame, spark_connect::readwriter::DataFrameWriter::shares_plan_with, )), - refused_bucket_count: std::sync::Arc::new(std::sync::Mutex::new(None)), + refused_bucket_count: refused_bucket_count.clone(), refused_bucket_cols: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), - unsent: std::sync::Arc::new(std::sync::Mutex::new(UnsentArgs::default())), + shared: crate::options::PySharedObjects::of(std::sync::Arc::new(WriterKeptArgs { + unsent: unsent.clone(), + refused_bucket_count, + })), + unsent, frame, gc_ready: crate::gc::GcReady::new(), } @@ -5384,6 +5509,7 @@ impl PyDataFrameWriter { refused_bucket_count: std::sync::Arc::clone(&this.refused_bucket_count), refused_bucket_cols: std::sync::Arc::clone(&this.refused_bucket_cols), unsent: std::sync::Arc::clone(&this.unsent), + shared: this.shared.as_ref().map(|shared| shared.clone_ref(py)), frame: this.frame.clone_ref(py), gc_ready: crate::gc::GcReady::new(), }, @@ -6075,30 +6201,15 @@ impl PyDataFrameWriter { fn __traverse__(&self, visit: pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> { // The DataFrame, which reports the objects of the plan that the writer shares with it, as - // PySpark's writer holds the plan (see `crate::gc::Holders`). The kept arguments, which - // Python's garbage collector sees, as `exclusive_relation_owners` reports a DataFrame's - // owners: only when this writer holds them alone, so that a copy that shares them does not - // count one reference twice. A lock held elsewhere is skipped. Nothing of a writer object - // that is being created. + // PySpark's writer holds the plan (see `crate::gc::Holders`), and the Python object of the + // kept arguments, which the copies of the writer share and which reports them (see + // [`WriterKeptArgs`]). Nothing of a writer object that is being created. if !self.gc_ready.get() { return Ok(()); } visit.call(&self.frame)?; - if std::sync::Arc::strong_count(&self.unsent) != 1 { - return Ok(()); - } - if let Ok(unsent) = self.unsent.try_lock() { - for object in [ - &unsent.format, - &unsent.partition, - &unsent.clustering, - &unsent.mode, - ] - .into_iter() - .flatten() - { - visit.call(object)?; - } + if let Some(shared) = &self.shared { + visit.call(shared)?; } Ok(()) } @@ -6106,6 +6217,7 @@ impl PyDataFrameWriter { /// Drop this writer's references to the kept arguments and to its state, of which it keeps a /// cleared copy; a copy's are kept. fn __clear__(&mut self) { + self.shared = None; self.unsent = std::sync::Arc::new(std::sync::Mutex::new(UnsentArgs::default())); self.inner = std::sync::Arc::new( self.inner @@ -6145,9 +6257,12 @@ pub struct PyDataFrameWriterV2 { /// The table properties that the core writer does not take, which each write fails with after /// its options. unsent_properties: std::sync::Arc>, - /// Set once the value is written, as `__traverse__` reads `unsent_provider` and - /// `unsent_clustering`, which are not valid while the writer object is created (see - /// [`crate::gc::GcReady`]). + /// The Python object of the kept provider, columns, options and table properties, which the + /// copies of this writer share too, and which reports their objects to Python's garbage + /// collector, as the `_write` of PySpark's copies holds them (see [`WriterV2KeptArgs`]). + shared: Option>, + /// Set once the value is written, as `__traverse__` reads `inner`, which is not valid while + /// the writer object is created (see [`crate::gc::GcReady`]). gc_ready: crate::gc::GcReady, } @@ -6287,6 +6402,7 @@ impl PyDataFrameWriterV2 { unsent_partition: std::sync::Arc::clone(&this.unsent_partition), unsent_options: std::sync::Arc::clone(&this.unsent_options), unsent_properties: std::sync::Arc::clone(&this.unsent_properties), + shared: this.shared.as_ref().map(|shared| shared.clone_ref(py)), frame: this.frame.clone_ref(py), gc_ready: crate::gc::GcReady::new(), }, @@ -6384,7 +6500,7 @@ impl PyDataFrameWriterV2 { if !cluster_cols.is_empty() { writer = writer.cluster_by(cluster_cols); } - let restored = PyDataFrameWriterV2 { + let mut restored = PyDataFrameWriterV2 { inner: std::sync::Arc::new(WriterState::of_frame( writer, &frame, @@ -6396,9 +6512,11 @@ impl PyDataFrameWriterV2 { unsent_partition: std::sync::Arc::new(std::sync::Mutex::new(kept_partition)), unsent_options: std::sync::Arc::new(std::sync::Mutex::new(Default::default())), unsent_properties: std::sync::Arc::new(std::sync::Mutex::new(Default::default())), + shared: None, frame, gc_ready: crate::gc::GcReady::new(), }; + restored.shared = restored.shared_of(); // The options and the table properties that the core writer does not take are set in // their order, as PySpark's dicts, which keep them. for (key, value) in &kept_options { @@ -6616,18 +6734,12 @@ impl PyDataFrameWriterV2 { if let Some(table) = &self.unsent_table { visit.call(table)?; } - // The kept arguments and the state that a copy shares are reported by no owner, as the - // writer's are, and a lock held elsewhere is skipped: of the state, the functions of the - // UDFs of the partitioning columns that it alone keeps alive. - for kept in [&self.unsent_provider, &self.unsent_clustering] { - if std::sync::Arc::strong_count(kept) != 1 { - continue; - } - if let Ok(kept) = kept.try_lock() { - if let Some(object) = &*kept { - visit.call(object)?; - } - } + // The Python object of the kept arguments, which the copies of the writer share and which + // reports them (see [`WriterV2KeptArgs`]). The state that a copy shares is reported by no + // owner, and a lock held elsewhere is skipped: of the state, the functions of the UDFs of + // the partitioning columns that it alone keeps alive. + if let Some(shared) = &self.shared { + visit.call(shared)?; } if std::sync::Arc::strong_count(&self.inner) == 1 { if let Some(writer) = self.inner.try_lock() { @@ -6644,6 +6756,7 @@ impl PyDataFrameWriterV2 { /// cleared copy; a copy's are kept. fn __clear__(&mut self) { self.unsent_table = None; + self.shared = None; self.unsent_provider = std::sync::Arc::new(std::sync::Mutex::new(None)); self.unsent_clustering = std::sync::Arc::new(std::sync::Mutex::new(None)); self.inner = std::sync::Arc::new( diff --git a/crates/pyspark-rs/src/lib.rs b/crates/pyspark-rs/src/lib.rs index 07fb7241..2c5e0282 100644 --- a/crates/pyspark-rs/src/lib.rs +++ b/crates/pyspark-rs/src/lib.rs @@ -333,8 +333,10 @@ fn bare_type_names(m: &Bound<'_, PyModule>) -> PyResult<()> { /// The native classes that are not attributes of the module, of objects that it makes: the /// expression of a Column (`Column._expr`), the attribute of a param of an ML class, the exit /// hook and the active session of the sessions (`SparkSession._active_session`), the tags and the -/// user context extensions that threads share, and the listener bus of a stream manager. -fn classes_outside_module(py: Python<'_>) -> [Bound<'_, pyo3::types::PyType>; 7] { +/// user context extensions that threads share, the listener bus of a stream manager, and the +/// objects of the options and the kept arguments that the copies of a reader, a writer or a +/// builder share. +fn classes_outside_module(py: Python<'_>) -> [Bound<'_, pyo3::types::PyType>; 8] { [ py.get_type::(), py.get_type::(), @@ -343,5 +345,6 @@ fn classes_outside_module(py: Python<'_>) -> [Bound<'_, pyo3::types::PyType>; 7] py.get_type::(), py.get_type::(), py.get_type::(), + py.get_type::(), ] } diff --git a/crates/pyspark-rs/src/options.rs b/crates/pyspark-rs/src/options.rs index 5aefa62f..221d8815 100644 --- a/crates/pyspark-rs/src/options.rs +++ b/crates/pyspark-rs/src/options.rs @@ -385,6 +385,26 @@ impl UnencodableOptions { Ok(entries) } + /// Report the Python objects of the entries, the keys that are not strs and the values that + /// are objects, to Python's garbage collector, as PySpark's dict of its options holds them: + /// those that only these options hold, so that a copy of them, which a request makes of the + /// state, does not count one reference twice (see [`SharedObjects`]). + pub(crate) fn visit_objects( + &self, + visit: &pyo3::PyVisit<'_>, + ) -> Result<(), pyo3::PyTraverseError> { + for (key, value) in &self.0 { + for entry in std::iter::once(key).chain(value.as_ref()) { + if let OptionKey::Object { object, .. } = entry { + if std::sync::Arc::strong_count(object) == 1 { + visit.call(&**object)?; + } + } + } + } + Ok(()) + } + /// The entries as Python objects, which a reduce gives back: the key, and the value, if any. pub(crate) fn to_objects<'py>( &self, @@ -403,6 +423,74 @@ impl UnencodableOptions { } } +/// The Python objects of a state that the shallow copies of a reader, a writer or a session +/// builder share, as PySpark's copies share the attributes of their `__dict__`, such as the dict +/// of their options, whose keys and values may be any objects. A state that copies share has no +/// one holder among them that could report its objects to Python's garbage collector once, so the +/// state is held by a Python object too, a [`PySharedObjects`], which each copy holds and reports, +/// and which reports the objects: so each is reported once, and freed with the last copy that +/// holds the state, as in PySpark. +pub(crate) trait SharedObjects: Send + Sync { + /// Report the Python objects that the state holds, unless another thread holds its lock. + fn visit(&self, visit: &pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError>; + + /// Drop the Python objects of the state, which the garbage collector found unreachable with + /// every holder of the state. They are dropped once the lock is left, as their drop may run + /// Python code. + fn clear(&self); +} + +impl SharedObjects for WriterState { + fn visit(&self, visit: &pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> { + match self.try_lock() { + Some(options) => options.visit_objects(visit), + None => Ok(()), + } + } + + fn clear(&self) { + let cleared = self + .try_lock() + .map(|mut options| std::mem::take(&mut *options)); + drop(cleared); + } +} + +/// The Python object that holds a state that the shallow copies of an object share, for Python's +/// garbage collector (see [`SharedObjects`]): each copy holds it and reports it in its +/// `__traverse__`, and it reports the objects of the state. +#[pyclass(name = "_SharedObjects", module = "pyspark._pyspark")] +pub(crate) struct PySharedObjects(std::sync::Arc); + +impl PySharedObjects { + /// The holder of `state`. + pub(crate) fn new(state: std::sync::Arc) -> Self { + PySharedObjects(state) + } + + /// The Python object of `state`, for a new holder of it, or None when it cannot be made, which + /// reports nothing of the state: its objects are kept alive then, rather than freed while + /// reachable. + pub(crate) fn of(state: std::sync::Arc) -> Option> { + Python::attach(|py| Py::new(py, PySharedObjects(state)).ok()) + } +} + +#[pymethods] +impl PySharedObjects { + /// The objects of the state. The object has no `__dict__`, so a collection does not meet it + /// before its value is written. + fn __traverse__(&self, visit: pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> { + self.0.visit(&visit) + } + + /// Break a reference cycle that the garbage collector found unreachable: as every holder of + /// the state is unreachable then, the objects of the state are dropped. + fn __clear__(&mut self) { + self.0.clear(); + } +} + /// A reader that takes its options one at a time, and unsets one of them. pub(crate) trait ReaderOptionBuilder: OptionBuilder { fn without_option(self, key: &str) -> Self; diff --git a/crates/pyspark-rs/src/readwriter.rs b/crates/pyspark-rs/src/readwriter.rs index 976148f6..aeacfeba 100644 --- a/crates/pyspark-rs/src/readwriter.rs +++ b/crates/pyspark-rs/src/readwriter.rs @@ -43,6 +43,11 @@ pub struct PyDataFrameReader { /// The options that UTF-8 cannot encode, which the copies of this reader share, as they share /// `options` ([`UnencodableOptions`]). unencodable_options: std::sync::Arc>, + /// The Python object of `unencodable_options`, which the copies of this reader share too, and + /// which reports the keys and the values of its options that are objects to Python's garbage + /// collector, as the `_options` dict of PySpark's copies holds them (see + /// [`crate::options::SharedObjects`]). + shared: Option>, /// The UTF-32LE bytes of the schema, when it is a str that UTF-8 cannot encode, which the core /// reader leaves unset. unencodable_schema: Option>, @@ -54,13 +59,15 @@ pub struct PyDataFrameReader { impl PyDataFrameReader { pub fn new(reader: DataFrameReader) -> Self { let options = reader.get_options().clone(); + let unencodable_options = std::sync::Arc::new(crate::dataframe::WriterState::new( + UnencodableOptions::default(), + )); PyDataFrameReader { inner: without_options(reader, &options), options: std::sync::Arc::new(crate::dataframe::WriterState::new(options)), unsent_format: None, - unencodable_options: std::sync::Arc::new(crate::dataframe::WriterState::new( - UnencodableOptions::default(), - )), + shared: crate::options::PySharedObjects::of(unencodable_options.clone()), + unencodable_options, unencodable_schema: None, gc_ready: crate::gc::GcReady::new(), } @@ -716,6 +723,7 @@ impl PyDataFrameReader { .as_ref() .map(|format| format.clone_ref(py)), unencodable_options: std::sync::Arc::clone(&this.unencodable_options), + shared: this.shared.as_ref().map(|shared| shared.clone_ref(py)), unencodable_schema: this.unencodable_schema.clone(), gc_ready: crate::gc::GcReady::new(), }, @@ -908,8 +916,9 @@ impl PyDataFrameReader { Ok(slf) } - /// The format that the reader keeps, a Python object, which Python's garbage collector sees; - /// nothing of a reader object that is being created. + /// The format that the reader keeps, a Python object, which Python's garbage collector sees, + /// and the Python object of the options that it shares with its copies, which reports their + /// keys and values that are objects; nothing of a reader object that is being created. fn __traverse__(&self, visit: pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> { if !self.gc_ready.get() { return Ok(()); @@ -917,11 +926,15 @@ impl PyDataFrameReader { if let Some(format) = &self.unsent_format { visit.call(&format.object)?; } + if let Some(shared) = &self.shared { + visit.call(shared)?; + } Ok(()) } fn __clear__(&mut self) { self.unsent_format = None; + self.shared = None; } /// The session of the reader, as PySpark Connect's `DataFrameReader._client`, which its diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs index e643f2bb..e7098e01 100644 --- a/crates/pyspark-rs/src/session.rs +++ b/crates/pyspark-rs/src/session.rs @@ -42,6 +42,27 @@ pub struct PySparkSessionBuilder { /// builder, which a shallow copy has too, and which a `channelBuilder` sets on its builder /// alone. channel_builder: Option>, + /// The Python object of `options`, which the shallow copies of this builder share too, and + /// which reports the keys and the values of its options that are objects to Python's garbage + /// collector, as PySpark's `_options` dict holds them (see + /// [`crate::options::SharedObjects`]). + shared: Option>, +} + +impl crate::options::SharedObjects for std::sync::Mutex { + fn visit(&self, visit: &pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> { + match self.try_lock() { + Ok(options) => options.unencodable.visit_objects(visit), + Err(_) => Ok(()), + } + } + + fn clear(&self) { + let unencodable = self + .try_lock() + .map(|mut options| std::mem::take(&mut options.unencodable)); + drop(unencodable); + } } /// The options of a session builder, which its shallow copies share (see @@ -94,8 +115,10 @@ fn session_url(py: Python<'_>, remote_url: Option>) -> PyResult) -> Self { + let options: std::sync::Arc> = Default::default(); PySparkSessionBuilder { - options: Default::default(), + shared: Py::new(py, crate::options::PySharedObjects::new(options.clone())).ok(), + options, hook_factories: Some(PyList::empty(py).unbind()), channel_builder: None, } @@ -163,6 +186,7 @@ impl PySparkSessionBuilder { options: std::sync::Arc::clone(&this.options), hook_factories: this.hook_factories.as_ref().map(|list| list.clone_ref(py)), channel_builder: this.channel_builder.as_ref().map(|cb| cb.clone_ref(py)), + shared: this.shared.as_ref().map(|shared| shared.clone_ref(py)), } }; let copy = Py::new(py, copy)?; @@ -220,8 +244,10 @@ impl PySparkSessionBuilder { /// reference of its own to its channel builder and to the list, which the shallow copies of /// the builder share and which reports the factories itself, as PySpark's list does: so no /// reference is counted twice, and the collector frees a factory only with the last builder - /// that reaches it. Only `Option>` fields are read, which are `None` while the object - /// is created, so no [`crate::gc::GcReady`] is needed. + /// that reaches it. So does the Python object of the options that the copies share, which + /// reports the keys and the values of the options that are objects. Only `Option>` + /// fields are read, which are `None` while the object is created, so no + /// [`crate::gc::GcReady`] is needed. fn __traverse__(&self, visit: pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> { if let Some(channel_builder) = &self.channel_builder { visit.call(channel_builder)?; @@ -229,15 +255,19 @@ impl PySparkSessionBuilder { if let Some(hook_factories) = &self.hook_factories { visit.call(hook_factories)?; } + if let Some(shared) = &self.shared { + visit.call(shared)?; + } Ok(()) } /// Break a reference cycle that the garbage collector found unreachable: the builder drops - /// its channel builder and its reference to the list of its hook factories, which a copy that - /// is reachable still holds. + /// its channel builder and its references to the list of its hook factories and to the + /// object of its options, which a copy that is reachable still holds. fn __clear__(&mut self) { self.channel_builder = None; self.hook_factories = None; + self.shared = None; } /// Set the remote Spark Connect server URL. Returns the builder (chainable). diff --git a/crates/pyspark-rs/src/streaming.rs b/crates/pyspark-rs/src/streaming.rs index 9652ad4c..0b919f28 100644 --- a/crates/pyspark-rs/src/streaming.rs +++ b/crates/pyspark-rs/src/streaming.rs @@ -269,6 +269,10 @@ pub struct PyDataStreamReader { /// The options that UTF-8 cannot encode, which the copies of this reader share, as they share /// `options` ([`UnencodableOptions`]). unencodable_options: Arc>, + /// The Python object of `unencodable_options`, which the copies of this reader share too, and + /// which reports the keys of its options that are objects to Python's garbage collector, as + /// the `_options` dict of PySpark's copies holds them (see [`crate::options::SharedObjects`]). + shared: Option>, /// The UTF-32LE bytes of the schema, when it is a str that UTF-8 cannot encode, which the core /// reader leaves unset. unencodable_schema: Option>, @@ -296,11 +300,13 @@ fn stream_reader_format( impl PyDataStreamReader { pub fn new(reader: DataStreamReader) -> Self { let options = reader.get_options().clone(); + let unencodable_options = Arc::new(WriterState::new(UnencodableOptions::default())); PyDataStreamReader { inner: without_options(reader, &options), options: Arc::new(WriterState::new(options)), unsent_format: None, - unencodable_options: Arc::new(WriterState::new(UnencodableOptions::default())), + shared: crate::options::PySharedObjects::of(unencodable_options.clone()), + unencodable_options, unencodable_schema: None, gc_ready: crate::gc::GcReady::new(), } @@ -481,6 +487,7 @@ impl PyDataStreamReader { .as_ref() .map(|format| format.clone_ref(py)), unencodable_options: Arc::clone(&this.unencodable_options), + shared: this.shared.as_ref().map(|shared| shared.clone_ref(py)), unencodable_schema: this.unencodable_schema.clone(), gc_ready: crate::gc::GcReady::new(), }, @@ -691,11 +698,15 @@ impl PyDataStreamReader { if let Some(format) = &self.unsent_format { visit.call(&format.object)?; } + if let Some(shared) = &self.shared { + visit.call(shared)?; + } Ok(()) } fn __clear__(&mut self) { self.unsent_format = None; + self.shared = None; } /// The session of the reader, as PySpark Connect's `DataStreamReader._client`, which its diff --git a/dev/parity/known-differences.md b/dev/parity/known-differences.md index a3429af3..5ee6cb20 100644 --- a/dev/parity/known-differences.md +++ b/dev/parity/known-differences.md @@ -366,7 +366,9 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan rows of `toLocalIterator`, and the expression of a Column and of a table argument (`_expr`, `_subquery_expr`) and its copies, of the Columns and the plans that they hold, and an `ArrayType`, a `MapType`, a `StructField` and a `StructType`, of the types, flags, metadata and fields that - they hold, as PySpark's `__dict__` holds them. Those that hold their DataFrame report it, which + they hold, as PySpark's `__dict__` holds them, and the readers, the writers and the session + builder, of the objects of their options and of the arguments that the writers keep, through one + object of them that their shallow copies share. Those that hold their DataFrame report it, which reports the functions of the plan that they share (`crate::gc::Holders`). A function that a Column and a DataFrame of it share, or two DataFrames, or a DataFrame and a table argument of it, which holds the plan and not the DataFrame, is reported by none: the holders share one reference to it, diff --git a/docs/compatibility.md b/docs/compatibility.md index 43820e3c..2e372d54 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -2727,11 +2727,17 @@ collects it. A stream writer holds its DataFrame, where PySpark's holds the plan that keeps its stream writer is freed by the collector, where PySpark frees it with its last reference. The functions of `foreachBatch` and `foreach` are kept as their pickles, as PySpark's stream writer keeps them, so a function that keeps its writer is freed with its last reference, and -a copy or a pickle of the writer carries the pickle of the call. The registration of `dataSource` reports its session, and the builder of -`pyspark.sql.connect.session.SparkSession` its channel builder and the list of its hook factories, -which its shallow copies share, as PySpark's plain objects hold them, so a session that keeps its -`dataSource`, or a channel builder or a hook factory that keeps its builder or a copy of it, is -collected. +a copy or a pickle of the writer carries the pickle of the call. The registration of `dataSource` +reports its session, and the builder of `pyspark.sql.connect.session.SparkSession` its channel +builder and the list of its hook factories, which its shallow copies share, as PySpark's plain +objects hold them, so a session that keeps its `dataSource`, or a channel builder or a hook factory +that keeps its builder or a copy of it, is collected. The readers, the writers and the builder +report the keys and the values of their options that are objects, which PySpark's dicts of options +keep as they are, such as a key that is not a str, and the format, the save mode, the partitioning +and clustering columns and the provider that the writers keep as they are, through one object of +them that their shallow copies share, as PySpark's copies share the dict of their options and the +`_write` of a writer: so a cycle through one, such as a key of an option that keeps its reader, is +collected, also with a copy of the reader, and freed with the last of the copies. The condition and the assignments of a clause of `mergeInto` are built, and the functions of their UDFs pickled, when the clause is added, which raises there, and the merge sends them as they were diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 11a97c69..a8239125 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -13254,6 +13254,71 @@ def __call__(self, session): session.stop() +def test_cycles_through_the_option_keys_and_kept_arguments_are_collected_as_pyspark(mock_connect): + # As PySpark's readers, writers and session builder, plain objects whose dicts keep the keys of + # their options as they are, and a table property's value, and whose writers keep a format, a + # mode, partitioning and clustering columns and a provider that are not strs as they are: + # a reference cycle through such an object is collected, also while a shallow copy shares the + # dict, as the copies of PySpark's do, and freed with the last of the copies. + import copy + import gc + import weakref + + from pyspark.sql.connect.session import SparkSession as ConnectSession + + spark, _ = mock_connect + df = spark.range(1) + + class Key: + # A hashable key of an option, which keeps what it is a key of. + def __hash__(self): + return 1 + + def __eq__(self, other): + return self is other + + def with_copy(make): + return lambda key: (lambda holder: (holder, copy.copy(holder)))(make(key)) + + makers = [ + lambda key: spark.read.option(key, "v"), + lambda key: spark.readStream.option(key, "v"), + lambda key: df.write.option(key, "v"), + lambda key: df.write.format(key), + lambda key: df.write.mode(key), + lambda key: df.write.partitionBy(key), + lambda key: df.write.clusterBy(key), + lambda key: df.writeTo("t").option(key, "v"), + lambda key: df.writeTo("t").tableProperty(key, "v"), + lambda key: df.writeTo("t").tableProperty("p", key), + lambda key: df.writeTo("t").using(key), + lambda key: df.writeTo("t").clusterBy(key), + lambda key: ConnectSession.builder.config(key, "v"), + ] + for make in makers + [with_copy(make) for make in makers]: + key = Key() + key.held = make(key) + freed = weakref.ref(key) + del key + gc.collect() + assert freed() is None + + # A copy that stays keeps the options, which the copy that is collected shares. + for make in (lambda key: spark.read.option(key, "v"), lambda key: df.write.option(key, "v"), + lambda key: df.writeTo("t").option(key, "v"), + lambda key: ConnectSession.builder.config(key, "v")): + key = Key() + key.held = make(key) + kept = copy.copy(key.held) + freed = weakref.ref(key) + del key + gc.collect() + assert freed() is not None + del kept + gc.collect() + assert freed() is None + + def test_stream_writers_keep_the_pickles_of_their_functions_as_pyspark(mock_connect): # As PySpark's DataStreamWriter, whose WriteStreamOperationStart keeps the pickle of the # function of foreachBatch and foreach and not the function: an object that keeps the writer