diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs index d17c8c37..4d82c2ee 100644 --- a/crates/pyspark-rs/src/session.rs +++ b/crates/pyspark-rs/src/session.rs @@ -1039,9 +1039,14 @@ impl PySparkSession { let schema = DataType::Struct { fields: fields.clone(), }; - let frame = self - .session - .create_dataframe_from_inline_arrow_ipc(ipc.extract()?, schema); + let ipc: Vec = ipc.extract()?; + let frame = py + .detach(|| { + self.session + .create_dataframe_from_inline_arrow_ipc(ipc, schema) + .map_err(Box::new) + }) + .to_pyerr()?; // Match pandas row input: explicit nonempty field names travel // through toDF as well as the local relation's logical schema. return Ok(PyDataFrame::new( @@ -1064,9 +1069,14 @@ impl PySparkSession { let schema = DataType::Struct { fields: fields.clone(), }; - let frame = self - .session - .create_dataframe_from_inline_arrow_ipc(ipc.extract()?, schema); + let ipc: Vec = ipc.extract()?; + let frame = py + .detach(|| { + self.session + .create_dataframe_from_inline_arrow_ipc(ipc, schema) + .map_err(Box::new) + }) + .to_pyerr()?; // Match pandas row input: explicit nonempty field names travel // through toDF as well as the local relation's logical schema. return Ok(PyDataFrame::new( diff --git a/crates/spark-connect/src/arrow_input.rs b/crates/spark-connect/src/arrow_input.rs index 1592468a..7d707b9e 100644 --- a/crates/spark-connect/src/arrow_input.rs +++ b/crates/spark-connect/src/arrow_input.rs @@ -120,7 +120,7 @@ fn validate(batches: &[RecordBatch]) -> Result<()> { Ok(()) } -fn batch_data_size(batch: &RecordBatch) -> usize { +pub(crate) fn batch_data_size(batch: &RecordBatch) -> usize { batch .columns() .iter() diff --git a/crates/spark-connect/src/session.rs b/crates/spark-connect/src/session.rs index 5b6698e7..6b0844a3 100644 --- a/crates/spark-connect/src/session.rs +++ b/crates/spark-connect/src/session.rs @@ -418,16 +418,42 @@ impl SparkSession { // Empty data -> send the schema only (no Arrow `data`); the server builds an empty // relation from the schema. A zero-row Arrow payload would be empty bytes, which the // server cannot parse ("Unexpected end of input. Missing schema."). - let data = if rows.is_empty() { - None - } else { - Some(rows_to_arrow_ipc(&rows, &schema)?) - }; - let plan = LogicalPlan::LocalRelation { + if rows.is_empty() { + let plan = LogicalPlan::LocalRelation { + schema: Some(schema), + data: None, + }; + return Ok(DataFrame::new(self.clone(), plan)); + } + let batch = rows_to_arrow_batch(&rows, &schema)?; + Ok(DataFrame::new( + self.clone(), + self.local_relation(batch, schema)?, + )) + } + + /// The local relation of `batch`: inline, without reading the server's configuration, when + /// it is smaller than [`LOCAL_RELATION_SETTINGS_BYTES`]; otherwise as the Arrow importer + /// sends one, which reads the server's local-relation settings and uploads the data as + /// cache artifacts at `spark.sql.session.localRelationCacheThreshold`, as PySpark does. A + /// single gRPC message cannot carry a relation beyond the server's message size limit. + fn local_relation( + &self, + batch: arrow::record_batch::RecordBatch, + schema: DataType, + ) -> Result { + if crate::arrow_input::batch_data_size(&batch) >= LOCAL_RELATION_SETTINGS_BYTES { + return crate::arrow_input::create_relation( + &self.client, + vec![batch], + Some(schema), + None, + ); + } + Ok(LogicalPlan::LocalRelation { schema: Some(schema), - data, - }; - Ok(DataFrame::new(self.clone(), plan)) + data: Some(batch_to_arrow_ipc(&batch)?), + }) } /// Create an inline local relation from columns with the same encoding as local rows. @@ -441,29 +467,39 @@ impl SparkSession { ) -> Result { let arrow_schema = arrow::datatypes::Schema::new(schema_to_arrow_fields(&schema)?); let row_count = columns.first().map_or(0, |column| column.len()); - let data = Some(columns_to_arrow_ipc(columns, arrow_schema, row_count)?); - let plan = LogicalPlan::LocalRelation { - schema: Some(schema), - data, - }; - Ok(DataFrame::new(self.clone(), plan)) + let batch = columns_to_arrow_batch(columns, arrow_schema, row_count)?; + Ok(DataFrame::new( + self.clone(), + self.local_relation(batch, schema)?, + )) } /// Internal binding bridge for Arrow IPC already validated and encoded from - /// local rows or columns. Retains the row path's inline transport policy. + /// local rows or columns. Retains the row path's transport policy: inline, unless the + /// IPC is large enough to consult the server's local-relation settings. #[doc(hidden)] pub fn create_dataframe_from_inline_arrow_ipc( &self, data: Vec, schema: DataType, - ) -> DataFrame { - DataFrame::new( - self.clone(), + ) -> Result { + let plan = if data.len() >= LOCAL_RELATION_SETTINGS_BYTES { + let reader = + arrow::ipc::reader::StreamReader::try_new(std::io::Cursor::new(data), None) + .map_err(|e| { + SparkError::connect_msg(format!("Failed to read Arrow IPC: {e}")) + })?; + let batches = reader + .collect::, _>>() + .map_err(|e| SparkError::connect_msg(format!("Failed to read Arrow IPC: {e}")))?; + crate::arrow_input::create_relation(&self.client, batches, Some(schema), None)? + } else { LogicalPlan::LocalRelation { schema: Some(schema), data: Some(data), - }, - ) + } + }; + Ok(DataFrame::new(self.clone(), plan)) } /// Create a local DataFrame from Arrow batches without materializing rows. @@ -895,13 +931,18 @@ fn row_width_mismatch(row_index: usize, actual: usize, expected: usize) -> Spark /// Convert Arrow RecordBatch data to IPC bytes. /// Used by createDataFrame to package local data. -fn rows_to_arrow_ipc(rows: &[Row], schema: &DataType) -> Result> { +/// The size of a local relation, in Arrow data, from which `createDataFrame` reads the +/// server's local-relation settings to decide whether to upload it: half of the default +/// `spark.sql.session.localRelationCacheThreshold`, and well below the default gRPC message +/// size limit of the server (128 MiB). Smaller relations are sent inline without that RPC. +const LOCAL_RELATION_SETTINGS_BYTES: usize = 32 << 20; + +fn rows_to_arrow_batch( + rows: &[Row], + schema: &DataType, +) -> Result { use arrow::datatypes::Schema as ArrowSchema; - if rows.is_empty() { - return Ok(vec![]); - } - // Convert our schema to Arrow schema let arrow_schema_vec = schema_to_arrow_fields(schema)?; let arrow_schema = ArrowSchema::new(arrow_schema_vec); @@ -925,21 +966,33 @@ fn rows_to_arrow_ipc(rows: &[Row], schema: &DataType) -> Result> { columns.push(values_to_arrow(&column, &field.data_type)?); } - columns_to_arrow_ipc(columns, arrow_schema, rows.len()) + columns_to_arrow_batch(columns, arrow_schema, rows.len()) +} + +/// The Arrow IPC stream of local rows, as the server reads a local relation. +#[cfg(test)] +fn rows_to_arrow_ipc(rows: &[Row], schema: &DataType) -> Result> { + if rows.is_empty() { + return Ok(vec![]); + } + batch_to_arrow_ipc(&rows_to_arrow_batch(rows, schema)?) } -fn columns_to_arrow_ipc( +fn columns_to_arrow_batch( columns: Vec, arrow_schema: arrow::datatypes::Schema, row_count: usize, -) -> Result> { - use arrow::ipc::writer::StreamWriter; +) -> Result { use arrow::record_batch::RecordBatch; // Preserve the row count even for a batch without columns. let options = arrow::record_batch::RecordBatchOptions::new().with_row_count(Some(row_count)); - let batch = RecordBatch::try_new_with_options(Arc::new(arrow_schema), columns, &options) - .map_err(|e| SparkError::connect_msg(format!("Failed to create Arrow batch: {}", e)))?; + RecordBatch::try_new_with_options(Arc::new(arrow_schema), columns, &options) + .map_err(|e| SparkError::connect_msg(format!("Failed to create Arrow batch: {}", e))) +} + +fn batch_to_arrow_ipc(batch: &arrow::record_batch::RecordBatch) -> Result> { + use arrow::ipc::writer::StreamWriter; // Write to IPC stream let mut buffer = Vec::new(); diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index 9424ac5e..9a719c79 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -1177,6 +1177,54 @@ def test_add_artifacts_names_each_artifact_as_pyspark(protocol, tmp_path): } +# Local relations of at least 32 MiB read the server's local-relation settings, and are uploaded +# as cache artifacts at its threshold, whichever path encoded them; a single gRPC message +# cannot carry a relation beyond the server's message size limit. + +@pytest.mark.parametrize("path", ["local_rows", "pandas_rows", "float64", "strings"]) +def test_large_local_relations_follow_the_cache_threshold(protocol, path): + # (The mock service keeps gRPC's default 4 MiB message limit, so the inline variant of + # these inputs, below a higher threshold, can only be checked against a real server.) + import numpy as np + + if path == "local_rows": + data = [("y" * 2000, i) for i in range(20_000)] + schema = T.StructType([T.StructField("v", T.StringType()), T.StructField("i", T.LongType())]) + expected_rows = 20_000 + elif path == "pandas_rows": + # float32 columns into DoubleType keep the row path. + data = pd.DataFrame(np.ones((100_000, 45), dtype=np.float32), + columns=[f"c{i}" for i in range(45)]) + schema = T.StructType([T.StructField(c, T.DoubleType()) for c in data.columns]) + expected_rows = 100_000 + elif path == "float64": + data = pd.DataFrame(np.ones((100_000, 45)), columns=[f"c{i}" for i in range(45)]) + schema = T.StructType([T.StructField(c, T.DoubleType()) for c in data.columns]) + expected_rows = 100_000 + else: + data = pd.DataFrame({"v": ["x" * 3500] * 10_000}, dtype=object) + schema = T.StructType([T.StructField("v", T.StringType())]) + expected_rows = 10_000 + overrides = {"spark.sql.session.localRelationCacheThreshold": "1"} + with _arrow_input_service(protocol, **overrides) as (spark, state): + df = spark.createDataFrame(data, schema) + df.toArrow() + assert sum(table.num_rows for table in state["input_tables"]) == expected_rows + assert state["uploads"] + assert any("spark.sql.session.localRelationCacheThreshold" in keys + for _, keys in state["config_requests"]) + + +def test_small_local_relations_stay_inline_without_reading_settings(protocol): + with _arrow_input_service(protocol) as (spark, state): + schema = T.StructType([T.StructField("v", T.StringType()), T.StructField("i", T.LongType())]) + spark.createDataFrame([("y" * 100, i) for i in range(100)], schema).toArrow() + assert not state["uploads"] + assert not any("spark.sql.session.localRelationCacheThreshold" in keys + for _, keys in state["config_requests"]) + assert state["input_tables"][0].num_rows == 100 + + # Signed-int64 pandas input uses the same Arrow importer and mock service. @pytest.mark.parametrize("storage", ["object", "default", "python", "pyarrow", "python_nan", "pyarrow_nan"])