From 33bebdc75024991c318a00780848820a10195d69 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Sat, 7 Nov 2026 10:03:12 +0900 Subject: [PATCH] [SPARK-61062][CONNECT] Upload large local relations at the cache threshold, whichever path encodes them createDataFrame reads the server's local-relation settings for a relation of at least 32 MiB of Arrow data, from local rows, the pandas row conversion, columns and the inline IPC of the pandas fast paths, and then sends it as the Arrow importer does: inline below spark.sql.session.localRelationCacheThreshold and uploaded as cache artifacts at it, as PySpark does. Such relations used to be sent inline at any size, which failed beyond the server's gRPC message size limit. Co-authored-by: Isaac --- crates/pyspark-rs/src/session.rs | 22 +++-- crates/spark-connect/src/arrow_input.rs | 2 +- crates/spark-connect/src/session.rs | 117 +++++++++++++++++------- python/tests/test_arrow_handoff.py | 48 ++++++++++ 4 files changed, 150 insertions(+), 39 deletions(-) diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs index d17c8c37..4d82c2ee 100644 --- a/crates/pyspark-rs/src/session.rs +++ b/crates/pyspark-rs/src/session.rs @@ -1039,9 +1039,14 @@ impl PySparkSession { let schema = DataType::Struct { fields: fields.clone(), }; - let frame = self - .session - .create_dataframe_from_inline_arrow_ipc(ipc.extract()?, schema); + let ipc: Vec = ipc.extract()?; + let frame = py + .detach(|| { + self.session + .create_dataframe_from_inline_arrow_ipc(ipc, schema) + .map_err(Box::new) + }) + .to_pyerr()?; // Match pandas row input: explicit nonempty field names travel // through toDF as well as the local relation's logical schema. return Ok(PyDataFrame::new( @@ -1064,9 +1069,14 @@ impl PySparkSession { let schema = DataType::Struct { fields: fields.clone(), }; - let frame = self - .session - .create_dataframe_from_inline_arrow_ipc(ipc.extract()?, schema); + let ipc: Vec = ipc.extract()?; + let frame = py + .detach(|| { + self.session + .create_dataframe_from_inline_arrow_ipc(ipc, schema) + .map_err(Box::new) + }) + .to_pyerr()?; // Match pandas row input: explicit nonempty field names travel // through toDF as well as the local relation's logical schema. return Ok(PyDataFrame::new( diff --git a/crates/spark-connect/src/arrow_input.rs b/crates/spark-connect/src/arrow_input.rs index 1592468a..7d707b9e 100644 --- a/crates/spark-connect/src/arrow_input.rs +++ b/crates/spark-connect/src/arrow_input.rs @@ -120,7 +120,7 @@ fn validate(batches: &[RecordBatch]) -> Result<()> { Ok(()) } -fn batch_data_size(batch: &RecordBatch) -> usize { +pub(crate) fn batch_data_size(batch: &RecordBatch) -> usize { batch .columns() .iter() diff --git a/crates/spark-connect/src/session.rs b/crates/spark-connect/src/session.rs index 5b6698e7..6b0844a3 100644 --- a/crates/spark-connect/src/session.rs +++ b/crates/spark-connect/src/session.rs @@ -418,16 +418,42 @@ impl SparkSession { // Empty data -> send the schema only (no Arrow `data`); the server builds an empty // relation from the schema. A zero-row Arrow payload would be empty bytes, which the // server cannot parse ("Unexpected end of input. Missing schema."). - let data = if rows.is_empty() { - None - } else { - Some(rows_to_arrow_ipc(&rows, &schema)?) - }; - let plan = LogicalPlan::LocalRelation { + if rows.is_empty() { + let plan = LogicalPlan::LocalRelation { + schema: Some(schema), + data: None, + }; + return Ok(DataFrame::new(self.clone(), plan)); + } + let batch = rows_to_arrow_batch(&rows, &schema)?; + Ok(DataFrame::new( + self.clone(), + self.local_relation(batch, schema)?, + )) + } + + /// The local relation of `batch`: inline, without reading the server's configuration, when + /// it is smaller than [`LOCAL_RELATION_SETTINGS_BYTES`]; otherwise as the Arrow importer + /// sends one, which reads the server's local-relation settings and uploads the data as + /// cache artifacts at `spark.sql.session.localRelationCacheThreshold`, as PySpark does. A + /// single gRPC message cannot carry a relation beyond the server's message size limit. + fn local_relation( + &self, + batch: arrow::record_batch::RecordBatch, + schema: DataType, + ) -> Result { + if crate::arrow_input::batch_data_size(&batch) >= LOCAL_RELATION_SETTINGS_BYTES { + return crate::arrow_input::create_relation( + &self.client, + vec![batch], + Some(schema), + None, + ); + } + Ok(LogicalPlan::LocalRelation { schema: Some(schema), - data, - }; - Ok(DataFrame::new(self.clone(), plan)) + data: Some(batch_to_arrow_ipc(&batch)?), + }) } /// Create an inline local relation from columns with the same encoding as local rows. @@ -441,29 +467,39 @@ impl SparkSession { ) -> Result { let arrow_schema = arrow::datatypes::Schema::new(schema_to_arrow_fields(&schema)?); let row_count = columns.first().map_or(0, |column| column.len()); - let data = Some(columns_to_arrow_ipc(columns, arrow_schema, row_count)?); - let plan = LogicalPlan::LocalRelation { - schema: Some(schema), - data, - }; - Ok(DataFrame::new(self.clone(), plan)) + let batch = columns_to_arrow_batch(columns, arrow_schema, row_count)?; + Ok(DataFrame::new( + self.clone(), + self.local_relation(batch, schema)?, + )) } /// Internal binding bridge for Arrow IPC already validated and encoded from - /// local rows or columns. Retains the row path's inline transport policy. + /// local rows or columns. Retains the row path's transport policy: inline, unless the + /// IPC is large enough to consult the server's local-relation settings. #[doc(hidden)] pub fn create_dataframe_from_inline_arrow_ipc( &self, data: Vec, schema: DataType, - ) -> DataFrame { - DataFrame::new( - self.clone(), + ) -> Result { + let plan = if data.len() >= LOCAL_RELATION_SETTINGS_BYTES { + let reader = + arrow::ipc::reader::StreamReader::try_new(std::io::Cursor::new(data), None) + .map_err(|e| { + SparkError::connect_msg(format!("Failed to read Arrow IPC: {e}")) + })?; + let batches = reader + .collect::, _>>() + .map_err(|e| SparkError::connect_msg(format!("Failed to read Arrow IPC: {e}")))?; + crate::arrow_input::create_relation(&self.client, batches, Some(schema), None)? + } else { LogicalPlan::LocalRelation { schema: Some(schema), data: Some(data), - }, - ) + } + }; + Ok(DataFrame::new(self.clone(), plan)) } /// Create a local DataFrame from Arrow batches without materializing rows. @@ -895,13 +931,18 @@ fn row_width_mismatch(row_index: usize, actual: usize, expected: usize) -> Spark /// Convert Arrow RecordBatch data to IPC bytes. /// Used by createDataFrame to package local data. -fn rows_to_arrow_ipc(rows: &[Row], schema: &DataType) -> Result> { +/// The size of a local relation, in Arrow data, from which `createDataFrame` reads the +/// server's local-relation settings to decide whether to upload it: half of the default +/// `spark.sql.session.localRelationCacheThreshold`, and well below the default gRPC message +/// size limit of the server (128 MiB). Smaller relations are sent inline without that RPC. +const LOCAL_RELATION_SETTINGS_BYTES: usize = 32 << 20; + +fn rows_to_arrow_batch( + rows: &[Row], + schema: &DataType, +) -> Result { use arrow::datatypes::Schema as ArrowSchema; - if rows.is_empty() { - return Ok(vec![]); - } - // Convert our schema to Arrow schema let arrow_schema_vec = schema_to_arrow_fields(schema)?; let arrow_schema = ArrowSchema::new(arrow_schema_vec); @@ -925,21 +966,33 @@ fn rows_to_arrow_ipc(rows: &[Row], schema: &DataType) -> Result> { columns.push(values_to_arrow(&column, &field.data_type)?); } - columns_to_arrow_ipc(columns, arrow_schema, rows.len()) + columns_to_arrow_batch(columns, arrow_schema, rows.len()) +} + +/// The Arrow IPC stream of local rows, as the server reads a local relation. +#[cfg(test)] +fn rows_to_arrow_ipc(rows: &[Row], schema: &DataType) -> Result> { + if rows.is_empty() { + return Ok(vec![]); + } + batch_to_arrow_ipc(&rows_to_arrow_batch(rows, schema)?) } -fn columns_to_arrow_ipc( +fn columns_to_arrow_batch( columns: Vec, arrow_schema: arrow::datatypes::Schema, row_count: usize, -) -> Result> { - use arrow::ipc::writer::StreamWriter; +) -> Result { use arrow::record_batch::RecordBatch; // Preserve the row count even for a batch without columns. let options = arrow::record_batch::RecordBatchOptions::new().with_row_count(Some(row_count)); - let batch = RecordBatch::try_new_with_options(Arc::new(arrow_schema), columns, &options) - .map_err(|e| SparkError::connect_msg(format!("Failed to create Arrow batch: {}", e)))?; + RecordBatch::try_new_with_options(Arc::new(arrow_schema), columns, &options) + .map_err(|e| SparkError::connect_msg(format!("Failed to create Arrow batch: {}", e))) +} + +fn batch_to_arrow_ipc(batch: &arrow::record_batch::RecordBatch) -> Result> { + use arrow::ipc::writer::StreamWriter; // Write to IPC stream let mut buffer = Vec::new(); diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index 9424ac5e..9a719c79 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -1177,6 +1177,54 @@ def test_add_artifacts_names_each_artifact_as_pyspark(protocol, tmp_path): } +# Local relations of at least 32 MiB read the server's local-relation settings, and are uploaded +# as cache artifacts at its threshold, whichever path encoded them; a single gRPC message +# cannot carry a relation beyond the server's message size limit. + +@pytest.mark.parametrize("path", ["local_rows", "pandas_rows", "float64", "strings"]) +def test_large_local_relations_follow_the_cache_threshold(protocol, path): + # (The mock service keeps gRPC's default 4 MiB message limit, so the inline variant of + # these inputs, below a higher threshold, can only be checked against a real server.) + import numpy as np + + if path == "local_rows": + data = [("y" * 2000, i) for i in range(20_000)] + schema = T.StructType([T.StructField("v", T.StringType()), T.StructField("i", T.LongType())]) + expected_rows = 20_000 + elif path == "pandas_rows": + # float32 columns into DoubleType keep the row path. + data = pd.DataFrame(np.ones((100_000, 45), dtype=np.float32), + columns=[f"c{i}" for i in range(45)]) + schema = T.StructType([T.StructField(c, T.DoubleType()) for c in data.columns]) + expected_rows = 100_000 + elif path == "float64": + data = pd.DataFrame(np.ones((100_000, 45)), columns=[f"c{i}" for i in range(45)]) + schema = T.StructType([T.StructField(c, T.DoubleType()) for c in data.columns]) + expected_rows = 100_000 + else: + data = pd.DataFrame({"v": ["x" * 3500] * 10_000}, dtype=object) + schema = T.StructType([T.StructField("v", T.StringType())]) + expected_rows = 10_000 + overrides = {"spark.sql.session.localRelationCacheThreshold": "1"} + with _arrow_input_service(protocol, **overrides) as (spark, state): + df = spark.createDataFrame(data, schema) + df.toArrow() + assert sum(table.num_rows for table in state["input_tables"]) == expected_rows + assert state["uploads"] + assert any("spark.sql.session.localRelationCacheThreshold" in keys + for _, keys in state["config_requests"]) + + +def test_small_local_relations_stay_inline_without_reading_settings(protocol): + with _arrow_input_service(protocol) as (spark, state): + schema = T.StructType([T.StructField("v", T.StringType()), T.StructField("i", T.LongType())]) + spark.createDataFrame([("y" * 100, i) for i in range(100)], schema).toArrow() + assert not state["uploads"] + assert not any("spark.sql.session.localRelationCacheThreshold" in keys + for _, keys in state["config_requests"]) + assert state["input_tables"][0].num_rows == 100 + + # Signed-int64 pandas input uses the same Arrow importer and mock service. @pytest.mark.parametrize("storage", ["object", "default", "python", "pyarrow", "python_nan", "pyarrow_nan"])