From c7e59418d087af9dfec0b489c743a2a8772e34ff Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 28 Oct 2026 20:31:56 -0700 Subject: [PATCH 01/10] [SPARK-60675][CONNECT] Support PyArrow Table input through the C Stream interface Normalize Arrow schemas and timestamps with the existing PySpark helpers, then serialize imported Rust batches directly for local relations without Python row materialization. Co-authored-by: Isaac --- crates/pyspark-rs/src/lib.rs | 1 + crates/pyspark-rs/src/pyarrow_import.rs | 87 +++++++++++++++++++ crates/pyspark-rs/src/session.rs | 6 ++ crates/spark-connect/src/session.rs | 85 +++++++++++++++++++ docs/architecture.md | 7 ++ python/pyspark/sql/_arrow_input.py | 66 +++++++++++++++ python/tests/test_dropin_offline.py | 108 ++++++++++++++++++++++++ 7 files changed, 360 insertions(+) create mode 100644 crates/pyspark-rs/src/pyarrow_import.rs create mode 100644 python/pyspark/sql/_arrow_input.py diff --git a/crates/pyspark-rs/src/lib.rs b/crates/pyspark-rs/src/lib.rs index fec1a7d0..68507bd5 100644 --- a/crates/pyspark-rs/src/lib.rs +++ b/crates/pyspark-rs/src/lib.rs @@ -38,6 +38,7 @@ mod observation; mod pipelines; mod profiler; mod pyarrow_export; +mod pyarrow_import; mod readwriter; mod resource; mod row; diff --git a/crates/pyspark-rs/src/pyarrow_import.rs b/crates/pyspark-rs/src/pyarrow_import.rs new file mode 100644 index 00000000..f1b083be --- /dev/null +++ b/crates/pyspark-rs/src/pyarrow_import.rs @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! PyArrow Table input through arrow-rs's owned C Stream importer. + +use arrow::pyarrow::{FromPyArrow, Table}; +use pyo3::prelude::*; +use spark_connect::session::SparkSession; + +use crate::dataframe::PyDataFrame; +use crate::errors::ResultExt; + +pub(crate) fn create_dataframe( + py: Python<'_>, + session: &SparkSession, + table: &Bound<'_, PyAny>, + schema: Option<&Bound<'_, PyAny>>, +) -> PyResult { + let helper = py.import("pyspark.sql._arrow_input")?; + // Resolve DDL with the same converter as the existing local-data path. + let schema = match schema { + Some(s) if s.extract::().is_ok() => Some(crate::types::data_type_to_py( + py, + &crate::types::py_to_data_type(s)?, + )?), + Some(s) => Some(s.clone().unbind()), + None => None, + }; + let schema = schema.unwrap_or_else(|| py.None()); + // Empty input needs only the explicit schema, and no server configuration. + let empty: bool = table.call_method0("__len__")?.extract::()? == 0; + let (timestamp_type, timezone, large_types) = if empty { + (None, None, None) + } else { + py.detach(|| -> PyResult<_> { + let conf = session.conf(); + Ok(( + conf.get("spark.sql.timestampType").to_pyerr()?, + conf.get("spark.sql.session.timeZone").to_pyerr()?, + conf.get("spark.sql.execution.arrow.useLargeVarTypes") + .to_pyerr()?, + )) + })? + }; + let result = helper.getattr("prepare_table")?.call1(( + table, + schema, + timestamp_type, + timezone, + large_types, + ))?; + let normalized = result.get_item(0)?; + let spark_schema = crate::types::py_to_data_type(&result.get_item(1)?)?; + let names: Option> = result.get_item(2)?.extract()?; + let batches = if empty { + Vec::new() + } else { + // PyArrow 10 readers support _export_to_c, even without PyCapsules. + Table::from_pyarrow_bound(&normalized.call_method0("to_reader")?)? + .into_inner() + .0 + }; + let dataframe = py.detach(|| { + session + .create_dataframe_from_arrow(batches, spark_schema) + .to_pyerr() + })?; + let dataframe = match names { + Some(names) => dataframe.to_df(names.iter().map(String::as_str).collect()), + None => dataframe, + }; + Ok(PyDataFrame::new(dataframe)) +} diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs index 5f6302e3..0e271dbd 100644 --- a/crates/pyspark-rs/src/session.rs +++ b/crates/pyspark-rs/src/session.rs @@ -348,6 +348,12 @@ impl PySparkSession { // signature parity. See PARITY_GAP_ANALYSIS.md. let _ = (samplingRatio, verifySchema); + if let Ok(pa) = py.import("pyarrow") { + if data.is_instance(&pa.getattr("Table")?)? { + return crate::pyarrow_import::create_dataframe(py, &self.session, data, schema); + } + } + // Accept a pandas DataFrame (as pyspark does): convert it to a list of row lists // with NaN/NaT mapped to None, and take its column names as the default schema. // pandas-on-Spark always passes an explicit `schema`, so this mainly needs to diff --git a/crates/spark-connect/src/session.rs b/crates/spark-connect/src/session.rs index 7eb29ace..31843d53 100644 --- a/crates/spark-connect/src/session.rs +++ b/crates/spark-connect/src/session.rs @@ -290,6 +290,44 @@ impl SparkSession { Ok(DataFrame::new(self.clone(), plan)) } + /// Create a local DataFrame from Arrow batches without materializing rows. + /// + /// `schema` is the Spark schema applied by the server. All batches must share + /// the same Arrow schema; IPC serialization is still required for transport. + pub fn create_dataframe_from_arrow( + &self, + batches: Vec, + schema: DataType, + ) -> Result { + let data = if let Some(first) = batches.first() { + let arrow_schema = first.schema(); + let mut buffer = Vec::new(); + { + let mut writer = + arrow::ipc::writer::StreamWriter::try_new(&mut buffer, &arrow_schema) + .map_err(|e| SparkError::connect_msg(e.to_string()))?; + for batch in &batches { + if batch.schema() != arrow_schema { + return Err(SparkError::connect_msg("Arrow batch schemas must match")); + } + writer + .write(batch) + .map_err(|e| SparkError::connect_msg(e.to_string()))?; + } + writer + .finish() + .map_err(|e| SparkError::connect_msg(e.to_string()))?; + } + Some(buffer) + } else { + None + }; + Ok(DataFrame::new( + self.clone(), + LogicalPlan::LocalRelation { schema, data }, + )) + } + /// Create a DataFrameReader for reading data from various sources. /// /// Mirrors `pyspark.sql.SparkSession.read`. @@ -1224,6 +1262,53 @@ mod tests { .expect("failed to build session") } + #[test] + fn arrow_local_relation_preserves_batches_and_rejects_schema_mismatch() { + use arrow::array::Array; + use arrow::datatypes::{DataType as ArrowType, Field, Schema}; + let arrow_schema = Arc::new(Schema::new(vec![Field::new("x", ArrowType::Int32, true)])); + let batch = RecordBatch::try_new( + arrow_schema, + vec![Arc::new(Int32Array::from(vec![Some(1), None]))], + ) + .unwrap(); + let schema = DataType::Struct { + fields: vec![struct_field("x", DataType::Integer)], + }; + let spark = session(); + let df = spark + .create_dataframe_from_arrow(vec![batch.clone(), batch.clone()], schema.clone()) + .unwrap(); + let LogicalPlan::LocalRelation { + data: Some(data), .. + } = df.plan() + else { + panic!("expected Arrow local relation") + }; + let batches = StreamReader::try_new(Cursor::new(data), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(batches.len(), 2); + assert_eq!(batches[0], batch); + assert!(batches[1].column(0).is_null(1)); + let other = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("y", ArrowType::Int32, true)])), + batch.columns().to_vec(), + ) + .unwrap(); + assert!(spark + .create_dataframe_from_arrow(vec![batch, other], schema.clone()) + .is_err()); + assert!(matches!( + spark + .create_dataframe_from_arrow(vec![], schema) + .unwrap() + .plan(), + LogicalPlan::LocalRelation { data: None, .. } + )); + } + #[test] fn session_is_not_stopped_initially() { let spark = session(); diff --git a/docs/architecture.md b/docs/architecture.md index 17559ed9..1fa9261c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -111,6 +111,13 @@ Batch schema consistency is validated before export. Field names or timezones containing NUL use IPC over the collected batches, without another RPC. Empty batch vectors retain the empty-schema behavior. +`createDataFrame(pyarrow_table)` uses the reverse boundary: Spark's schema and +timestamp conversion helpers normalize the table, then arrow-rs imports its +`RecordBatchReader` through the C Stream interface. The Rust core serializes +those batches to IPC for the local relation sent to Spark. No Python rows are +materialized; casts, buffer alignment, and IPC serialization may still copy. +Empty tables require an explicit schema, matching PySpark Connect. + ## Two front-ends, one core === "Rust (native)" diff --git a/python/pyspark/sql/_arrow_input.py b/python/pyspark/sql/_arrow_input.py new file mode 100644 index 00000000..f77540c8 --- /dev/null +++ b/python/pyspark/sql/_arrow_input.py @@ -0,0 +1,66 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +"""Schema normalization for the native Arrow Table input path.""" + +from pyspark.errors import PySparkValueError +from pyspark.sql.pandas.types import ( + _check_arrow_table_timestamps_localize, + from_arrow_schema, + to_arrow_schema, +) +from pyspark.sql.types import StructType + + +def prepare_table(table, schema, timestamp_type, timezone, large_types): + """Match Connect's Arrow casts without converting any column to Python rows.""" + names = list(schema) if isinstance(schema, (list, tuple)) else None + explicit = schema is not None and names is None + if len(table) == 0: + if not explicit: + raise PySparkValueError(errorClass="CANNOT_INFER_EMPTY_SCHEMA", messageParameters={}) + return table, schema, None + + if names is not None: + names.extend(f"_{i + 1}" for i in range(len(names), table.num_columns)) + inferred = ( + schema + if isinstance(schema, StructType) + else from_arrow_schema(table.schema, prefer_timestamp_ntz=timestamp_type == "TIMESTAMP_NTZ") + ) + normalized = ( + _check_arrow_table_timestamps_localize(table, inferred, True, timezone) + .cast( + to_arrow_schema( + inferred, + error_on_duplicated_field_names_in_struct=True, + timezone="UTC", + prefers_large_types=large_types == "true", + ) + ) + .rename_columns(inferred.names) + ) + if names is not None and len(names) != normalized.num_columns: + raise PySparkValueError( + errorClass="AXIS_LENGTH_MISMATCH", + messageParameters={ + "expected_length": str(len(names)), + "actual_length": str(normalized.num_columns), + }, + ) + return normalized, schema if explicit else inferred, names diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 9988f655..7735741f 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -3085,3 +3085,111 @@ def allow_response_after_request(): else: assert result == [("id", "bigint")] assert service.calls == 1 + + +@pytest.mark.parametrize("names,expected", [(None, None), (["a"], ["a", "_2"]), ([], ["_1", "_2"])]) +def test_arrow_input_normalization(names, expected): + pa = pytest.importorskip("pyarrow") + from pyspark.sql._arrow_input import prepare_table + + table = pa.table({"x": pa.chunked_array([[1], [2]]), "y": ["a", None]}) + normalized, schema, columns = prepare_table(table, names, "TIMESTAMP_LTZ", "UTC", "false") + assert normalized.equals(table) + assert schema.names == ["x", "y"] + assert columns == expected + + +def test_arrow_input_empty_and_width_errors(): + pa = pytest.importorskip("pyarrow") + from pyspark.errors import PySparkValueError + from pyspark.sql._arrow_input import prepare_table + + empty = pa.table({"x": pa.array([], type=pa.int64())}) + with pytest.raises(PySparkValueError, match="CANNOT_INFER_EMPTY_SCHEMA"): + prepare_table(empty, None, None, None, None) + schema = T.StructType([T.StructField("x", T.LongType())]) + assert prepare_table(empty, schema, None, None, None)[1] is schema + with pytest.raises(PySparkValueError, match="AXIS_LENGTH_MISMATCH"): + prepare_table(pa.table({"x": [1]}), ["a", "b"], "TIMESTAMP_LTZ", "UTC", "false") + + +@pytest.mark.parametrize("timestamp_type,expected", [("TIMESTAMP_NTZ", T.TimestampNTZType), ("TIMESTAMP_LTZ", T.TimestampType)]) +def test_arrow_input_timestamps(timestamp_type, expected): + pa = pytest.importorskip("pyarrow") + from pyspark.sql._arrow_input import prepare_table + + table = pa.table({"t": pa.array([1001], type=pa.timestamp("ns"))}) + normalized, schema, _ = prepare_table(table, None, timestamp_type, "UTC", "false") + assert isinstance(schema.fields[0].dataType, expected) + assert normalized.column(0).type.unit == "us" + assert normalized.column(0).cast(pa.int64()).to_pylist() == [1] + + +def test_arrow_input_explicit_cast_and_nested_values(): + pa = pytest.importorskip("pyarrow") + from pyspark.sql._arrow_input import prepare_table + + table = pa.table({"x": [1, 2], "nested": [[1, None], []]}) + schema = T.StructType([T.StructField("x", T.IntegerType()), + T.StructField("nested", T.ArrayType(T.LongType()))]) + normalized, result, _ = prepare_table(table, schema, "TIMESTAMP_LTZ", "UTC", "false") + assert result is schema + assert normalized.column(0).type == pa.int32() + assert normalized.to_pylist() == table.to_pylist() + with pytest.raises(pa.ArrowInvalid): + prepare_table(pa.table({"x": [2**40]}), T.StructType([T.StructField("x", T.IntegerType())]), + "TIMESTAMP_LTZ", "UTC", "false") + + +def test_arrow_input_roundtrip_live_server(): + """Run explicitly with SPARK_REMOTE; also checks imported buffers own their lifetime.""" + import gc + import os + pa = pytest.importorskip("pyarrow") + remote = os.environ.get("SPARK_REMOTE") + if not remote: + pytest.skip("requires SPARK_REMOTE") + from pyspark.sql import SparkSession + + spark = SparkSession.builder.remote(remote).getOrCreate() + try: + table = pa.table({"x": pa.chunked_array([[1, 2], [3]]), "nested": [[1], None, []]}) + df = spark.createDataFrame(table, ["id", "items"]) + del table + gc.collect() + assert df.columns == ["id", "items"] + assert [(r[0], r[1]) for r in df.collect()] == [(1, [1]), (2, None), (3, [])] + empty = spark.createDataFrame(pa.table({"x": pa.array([], type=pa.int64())}), + T.StructType([T.StructField("x", T.LongType())])) + assert empty.count() == 0 + assert empty.columns == ["x"] + finally: + spark.stop() + + +@pytest.mark.parametrize("kind", ["slice", "dictionary", "decimal", "null", "struct", "nul"]) +def test_arrow_input_types_live_server(kind): + import os + from decimal import Decimal + pa = pytest.importorskip("pyarrow") + remote = os.environ.get("SPARK_REMOTE") + if not remote: + pytest.skip("requires SPARK_REMOTE") + from pyspark.sql import SparkSession + + tables = { + "slice": pa.table({"x": pa.array([0, 1, 2, 3]).slice(1, 2)}), + "dictionary": pa.table({"x": pa.array(["a", None, "b", "a"]).dictionary_encode()}), + "decimal": pa.table({"x": pa.array([Decimal("1.23"), None], type=pa.decimal128(10, 2))}), + "null": pa.table({"x": pa.nulls(2)}), + "struct": pa.table({"x": [{"a": [1, None]}, None]}), + "nul": pa.table({"a\0b": [1, 2]}), + } + spark = SparkSession.builder.remote(remote).getOrCreate() + try: + table = tables[kind] + df = spark.createDataFrame(table) + assert df.columns == table.column_names + assert df.toArrow().to_pylist() == table.to_pylist() + finally: + spark.stop() From 97bd97b3636087c8ee4be79f24a859000a6f9d4a Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 28 Oct 2026 21:02:38 -0700 Subject: [PATCH 02/10] [SPARK-60675][CONNECT] Upload large Arrow inputs as cached relations Honor server local-relation settings, split Arrow IPC into independently readable chunks, and upload session-scoped cache artifacts with SHA-256 deduplication and CRC validation. Keep large input payloads out of query plans and test uploads above the gRPC message limit. Co-authored-by: Isaac --- Cargo.lock | 1 + crates/spark-connect-core/Cargo.toml | 1 + crates/spark-connect-core/src/client.rs | 67 +++++++ crates/spark-connect/src/arrow_input.rs | 235 ++++++++++++++++++++++++ crates/spark-connect/src/dataframe.rs | 5 +- crates/spark-connect/src/lib.rs | 1 + crates/spark-connect/src/plan.rs | 16 ++ crates/spark-connect/src/session.rs | 39 +--- docs/architecture.md | 6 + python/tests/test_arrow_handoff.py | 210 +++++++++++++++++++++ python/tests/test_dropin_offline.py | 21 +++ 11 files changed, 570 insertions(+), 32 deletions(-) create mode 100644 crates/spark-connect/src/arrow_input.rs diff --git a/Cargo.lock b/Cargo.lock index 29e7d3fe..6238cb96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -87,6 +87,7 @@ dependencies = [ "prost-types", "regex", "serde_json", + "sha2", "tempfile", "thiserror", "tokio", diff --git a/crates/spark-connect-core/Cargo.toml b/crates/spark-connect-core/Cargo.toml index c6a0c088..6cdf0e01 100644 --- a/crates/spark-connect-core/Cargo.toml +++ b/crates/spark-connect-core/Cargo.toml @@ -26,6 +26,7 @@ serde_json = "1" regex = "1" prost-types.workspace = true crc32fast = "1" +sha2 = "0.10" [dev-dependencies] tempfile = "3" diff --git a/crates/spark-connect-core/src/client.rs b/crates/spark-connect-core/src/client.rs index 31820ed5..8f658bdc 100644 --- a/crates/spark-connect-core/src/client.rs +++ b/crates/spark-connect-core/src/client.rs @@ -800,6 +800,73 @@ impl SparkConnectClient { Ok(()) } + /// Upload session-scoped cache blobs, returning SHA-256 hashes in input order. + /// Existing artifacts and duplicate blobs are uploaded only once. + pub async fn cache_artifacts(&self, blobs: Vec>) -> Result> { + use crate::artifact::InMemoryArtifact; + use sha2::{Digest, Sha256}; + use std::collections::HashSet; + + let hashes: Vec = blobs + .iter() + .map(|blob| format!("{:x}", Sha256::digest(blob))) + .collect(); + let names: Vec = hashes.iter().map(|hash| format!("cache/{hash}")).collect(); + let unique: HashSet<&str> = names.iter().map(String::as_str).collect(); + if unique.is_empty() { + return Ok(hashes); + } + let cached = self + .artifact_status(&unique.into_iter().collect::>()) + .await?; + let mut seen = HashSet::new(); + let mut artifacts = Vec::new(); + for (blob, name) in blobs.into_iter().zip(names) { + if !cached.get(&name).copied().unwrap_or(false) && seen.insert(name.clone()) { + artifacts.push(NamedArtifact::new( + name, + Box::new(InMemoryArtifact::new(blob)), + )); + } + } + if artifacts.is_empty() { + return Ok(hashes); + } + let expected: HashSet = artifacts.iter().map(|a| a.name.clone()).collect(); + let mut requests = build_artifact_request_stream(self.session_id.clone(), artifacts)?; + for request in &mut requests { + request.user_context = Some(UserContext { + user_id: self.user_id.clone().unwrap_or_default(), + ..Default::default() + }); + } + let mut req = Request::new(futures::stream::iter(requests)); + self._attach_metadata(&mut req); + let response = self + .stub + .clone() + .add_artifacts(req) + .await + .map_err(SparkError::from_grpc_status)? + .into_inner(); + let mut received = HashSet::new(); + for summary in response.artifacts { + if !summary.is_crc_successful { + return Err(SparkError::connect_msg(format!( + "CRC check failed for artifact: {}", + summary.name + ))); + } + received.insert(summary.name); + } + if !expected.is_subset(&received) { + return Err(SparkError::connect_msg( + "Missing cache artifact upload acknowledgement", + )); + } + Ok(hashes) + } + /// Check artifact status on the server. /// /// Mirrors functionality from `pyspark.sql.connect.client.artifact.ArtifactManager.is_cached_artifact`. diff --git a/crates/spark-connect/src/arrow_input.rs b/crates/spark-connect/src/arrow_input.rs new file mode 100644 index 00000000..8ef091e3 --- /dev/null +++ b/crates/spark-connect/src/arrow_input.rs @@ -0,0 +1,235 @@ +//! Arrow local relations and session-scoped chunked uploads. + +use arrow::datatypes::SchemaRef; +use arrow::ipc::writer::StreamWriter; +use arrow::record_batch::RecordBatch; +use spark_connect_core::client::SparkConnectClient; +use spark_connect_core::error::{Result, SparkError, SparkErrorKind}; +use spark_connect_core::runtime::block_on; + +use crate::plan::LogicalPlan; +use crate::types::DataType; + +struct Options { + cache_threshold: usize, + size_limit: usize, + chunk_rows: usize, + chunk_bytes: usize, + upload_batch_bytes: usize, +} + +impl Options { + fn from_client(client: &SparkConnectClient) -> Result { + let keys = [ + "spark.sql.session.localRelationCacheThreshold", + "spark.sql.session.localRelationSizeLimit", + "spark.sql.session.localRelationChunkSizeRows", + "spark.sql.session.localRelationChunkSizeBytes", + "spark.sql.session.localRelationBatchOfChunksSizeBytes", + ]; + let values = block_on(client.get_configs(&keys))?; + let mut parsed = Vec::new(); + for (i, key) in keys.iter().enumerate() { + let value = values + .get(i) + .and_then(Option::as_deref) + .and_then(|v| v.parse::().ok()) + .filter(|v| i == 0 || *v > 0) + .ok_or_else(|| { + SparkError::connect_msg(format!("Invalid or missing configuration: {key}")) + })?; + parsed.push(value); + } + Ok(Self { + cache_threshold: parsed[0], + size_limit: parsed[1], + chunk_rows: parsed[2], + chunk_bytes: parsed[3].min(parsed[4]), + upload_batch_bytes: parsed[4], + }) + } +} + +fn arrow_error(error: arrow::error::ArrowError) -> SparkError { + SparkError::connect_msg(error.to_string()) +} + +fn validate(batches: &[RecordBatch]) -> Result<()> { + if let Some(first) = batches.first() { + if batches.iter().any(|batch| batch.schema() != first.schema()) { + return Err(SparkError::connect_msg("Arrow batch schemas must match")); + } + } + Ok(()) +} + +pub(crate) fn inline_relation(batches: &[RecordBatch], schema: DataType) -> Result { + validate(batches)?; + let data = if let Some(first) = batches.first() { + let mut writer = StreamWriter::try_new(Vec::new(), &first.schema()).map_err(arrow_error)?; + for batch in batches { + writer.write(batch).map_err(arrow_error)?; + } + Some(writer.into_inner().map_err(arrow_error)?) + } else { + None + }; + Ok(LogicalPlan::LocalRelation { schema, data }) +} + +pub(crate) fn create_relation( + client: &SparkConnectClient, + batches: Vec, + schema: DataType, +) -> Result { + // Validate before any RPC, and preserve the offline empty-input path. + validate(&batches)?; + if batches.iter().all(|batch| batch.num_rows() == 0) { + return inline_relation(&batches, schema); + } + let options = Options::from_client(client)?; + // Count the logical slices rather than their potentially much larger backing buffers. + let bytes: usize = batches + .iter() + .flat_map(|b| b.columns()) + .map(|a| { + a.to_data() + .get_slice_memory_size() + .unwrap_or_else(|_| a.get_buffer_memory_size()) + }) + .sum(); + if bytes < options.cache_threshold { + return inline_relation(&batches, schema); + } + + let mut pending = Vec::new(); + let mut pending_size = 0; + let mut total_size = 0; + let mut hashes = Vec::new(); + let mut enqueue = |chunk: Vec| -> Result<()> { + total_size += chunk.len(); + if total_size > options.size_limit { + return Err(SparkError::classed( + SparkErrorKind::Analysis, + "LOCAL_RELATION_SIZE_LIMIT_EXCEEDED", + &[ + ("actualSize", &total_size.to_string()), + ("sizeLimit", &options.size_limit.to_string()), + ], + )); + } + if !pending.is_empty() && pending_size + chunk.len() > options.upload_batch_bytes { + hashes.extend(block_on( + client.cache_artifacts(std::mem::take(&mut pending)), + )?); + pending_size = 0; + } + pending_size += chunk.len(); + pending.push(chunk); + Ok(()) + }; + enqueue(schema.json().into_bytes())?; + serialize_chunks(&batches, &options, &mut enqueue)?; + hashes.extend(block_on(client.cache_artifacts(pending))?); + let schema_hash = hashes.remove(0); + Ok(LogicalPlan::ChunkedCachedLocalRelation { + data_hashes: hashes, + schema_hash: Some(schema_hash), + }) +} + +fn new_writer(schema: &SchemaRef) -> Result>> { + StreamWriter::try_new(Vec::new(), schema).map_err(arrow_error) +} + +/// Each chunk is an independently readable IPC stream. Check limits at most every +/// 1024 rows, as reference PySpark does; a single oversized row remains indivisible. +fn serialize_chunks( + batches: &[RecordBatch], + options: &Options, + emit: &mut impl FnMut(Vec) -> Result<()>, +) -> Result<()> { + let schema = batches[0].schema(); + let mut writer = new_writer(&schema)?; + let mut rows = 0; + for batch in batches { + let mut offset = 0; + while offset < batch.num_rows() { + let length = (batch.num_rows() - offset) + .min(1024) + .min(options.chunk_rows - rows); + writer + .write(&batch.slice(offset, length)) + .map_err(arrow_error)?; + offset += length; + rows += length; + if rows >= options.chunk_rows || writer.get_ref().len() >= options.chunk_bytes { + emit(writer.into_inner().map_err(arrow_error)?)?; + writer = new_writer(&schema)?; + rows = 0; + } + } + } + if rows > 0 { + emit(writer.into_inner().map_err(arrow_error)?)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::Int64Array; + use arrow::datatypes::{DataType as ArrowType, Field, Schema}; + use arrow::ipc::reader::StreamReader; + use std::io::Cursor; + use std::sync::Arc; + + #[test] + fn chunks_preserve_rows_schema_and_order() { + let schema = Arc::new(Schema::new(vec![Field::new("id", ArrowType::Int64, true)])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int64Array::from_iter_values(0..2500))], + ) + .unwrap(); + for (chunk_rows, chunk_bytes) in [(700, usize::MAX), (10000, 1024)] { + let options = Options { + cache_threshold: 0, + size_limit: usize::MAX, + chunk_rows, + chunk_bytes, + upload_batch_bytes: usize::MAX, + }; + let mut chunks = Vec::new(); + serialize_chunks(std::slice::from_ref(&batch), &options, &mut |c| { + chunks.push(c); + Ok(()) + }) + .unwrap(); + assert!(chunks.len() > 1); + let mut actual = Vec::new(); + for chunk in chunks { + let reader = StreamReader::try_new(Cursor::new(chunk), None).unwrap(); + assert_eq!(reader.schema(), schema); + let mut rows = 0; + for batch in reader { + let batch = batch.unwrap(); + rows += batch.num_rows(); + actual.extend( + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .iter() + .copied(), + ); + } + assert!(rows <= chunk_rows); + } + assert_eq!(actual, (0..2500).collect::>()); + } + } +} diff --git a/crates/spark-connect/src/dataframe.rs b/crates/spark-connect/src/dataframe.rs index 4ce38983..7baa8671 100644 --- a/crates/spark-connect/src/dataframe.rs +++ b/crates/spark-connect/src/dataframe.rs @@ -1694,7 +1694,10 @@ impl DataFrame { /// Check if this DataFrame is local (collected). pub fn is_local(&self) -> bool { - matches!(self.plan.as_ref(), LogicalPlan::LocalRelation { .. }) + matches!( + self.plan.as_ref(), + LogicalPlan::LocalRelation { .. } | LogicalPlan::ChunkedCachedLocalRelation { .. } + ) } /// Check if this DataFrame is streaming. diff --git a/crates/spark-connect/src/lib.rs b/crates/spark-connect/src/lib.rs index 5f1a8faf..180bf089 100644 --- a/crates/spark-connect/src/lib.rs +++ b/crates/spark-connect/src/lib.rs @@ -5,6 +5,7 @@ #![allow(clippy::result_large_err)] // SparkError is large; changing the type would break the API +mod arrow_input; pub mod catalog; pub mod column; pub mod conf; diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs index b24309bf..86dacf27 100644 --- a/crates/spark-connect/src/plan.rs +++ b/crates/spark-connect/src/plan.rs @@ -228,6 +228,11 @@ pub enum LogicalPlan { schema: DataType, data: Option>, }, + /// Local Arrow data uploaded as session-scoped cache artifacts. + ChunkedCachedLocalRelation { + data_hashes: Vec, + schema_hash: Option, + }, /// CachedRemoteRelation: a reference to a cached remote relation. CachedRemoteRelation { relation_id: String }, /// Read: `spark.read.format(...).load(...)` or `spark.read.table(...)`. @@ -944,6 +949,17 @@ impl LogicalPlan { Some(proto::relation::RelType::SubqueryAlias(Box::new(sq_alias))); } + LogicalPlan::ChunkedCachedLocalRelation { + data_hashes, + schema_hash, + } => { + relation.rel_type = Some(proto::relation::RelType::ChunkedCachedLocalRelation( + proto::ChunkedCachedLocalRelation { + data_hashes: data_hashes.clone(), + schema_hash: schema_hash.clone(), + }, + )); + } LogicalPlan::LocalRelation { schema, data } => { let mut local = proto::LocalRelation::default(); if let Some(d) = data { diff --git a/crates/spark-connect/src/session.rs b/crates/spark-connect/src/session.rs index 31843d53..ce92949e 100644 --- a/crates/spark-connect/src/session.rs +++ b/crates/spark-connect/src/session.rs @@ -294,38 +294,15 @@ impl SparkSession { /// /// `schema` is the Spark schema applied by the server. All batches must share /// the same Arrow schema; IPC serialization is still required for transport. + /// Non-empty inputs read the server's local-relation configuration. Large inputs + /// are uploaded as cache artifacts and referenced by hash in the resulting plan. pub fn create_dataframe_from_arrow( &self, batches: Vec, schema: DataType, ) -> Result { - let data = if let Some(first) = batches.first() { - let arrow_schema = first.schema(); - let mut buffer = Vec::new(); - { - let mut writer = - arrow::ipc::writer::StreamWriter::try_new(&mut buffer, &arrow_schema) - .map_err(|e| SparkError::connect_msg(e.to_string()))?; - for batch in &batches { - if batch.schema() != arrow_schema { - return Err(SparkError::connect_msg("Arrow batch schemas must match")); - } - writer - .write(batch) - .map_err(|e| SparkError::connect_msg(e.to_string()))?; - } - writer - .finish() - .map_err(|e| SparkError::connect_msg(e.to_string()))?; - } - Some(buffer) - } else { - None - }; - Ok(DataFrame::new( - self.clone(), - LogicalPlan::LocalRelation { schema, data }, - )) + let plan = crate::arrow_input::create_relation(&self.client, batches, schema)?; + Ok(DataFrame::new(self.clone(), plan)) } /// Create a DataFrameReader for reading data from various sources. @@ -1276,12 +1253,12 @@ mod tests { fields: vec![struct_field("x", DataType::Integer)], }; let spark = session(); - let df = spark - .create_dataframe_from_arrow(vec![batch.clone(), batch.clone()], schema.clone()) - .unwrap(); + let plan = + crate::arrow_input::inline_relation(&[batch.clone(), batch.clone()], schema.clone()) + .unwrap(); let LogicalPlan::LocalRelation { data: Some(data), .. - } = df.plan() + } = &plan else { panic!("expected Arrow local relation") }; diff --git a/docs/architecture.md b/docs/architecture.md index 1fa9261c..93107c3e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -117,6 +117,12 @@ timestamp conversion helpers normalize the table, then arrow-rs imports its those batches to IPC for the local relation sent to Spark. No Python rows are materialized; casts, buffer alignment, and IPC serialization may still copy. Empty tables require an explicit schema, matching PySpark Connect. +Inputs at or above `spark.sql.session.localRelationCacheThreshold` are serialized +into independent IPC streams and uploaded as session-scoped cache artifacts. +The plan carries `ChunkedCachedLocalRelation` hashes instead of embedding the +entire payload. Chunk sizes, upload batch sizes, and the total relation size +limit follow the server's local-relation settings; artifact RPC messages use +32 KiB pieces, with SHA-256 deduplication and CRC checks. ## Two front-ends, one core diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index 214a09a0..70dbea0b 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -395,3 +395,213 @@ def test_malformed_ipc_is_reported(protocol): with pytest.raises(RuntimeError, match="Arrow"): frame.toArrow() assert calls == {"execute": 1} + + +@contextmanager +def _arrow_input_service(protocol, *, upload_error=None, **overrides): + """Resolve cache artifacts at a mock service with a 4 MiB receive limit.""" + import hashlib + import zlib + + config = { + "spark.sql.timestampType": "TIMESTAMP_LTZ", + "spark.sql.session.timeZone": "UTC", + "spark.sql.execution.arrow.useLargeVarTypes": "false", + "spark.sql.session.localRelationCacheThreshold": "1048576", + "spark.sql.session.localRelationSizeLimit": str(3 * 1024**3), + "spark.sql.session.localRelationChunkSizeRows": "10000", + "spark.sql.session.localRelationChunkSizeBytes": str(16 * 1024**2), + "spark.sql.session.localRelationBatchOfChunksSizeBytes": str(1024**3), + } + config.update(overrides) + state = {"cache": {}, "uploads": 0, "relations": [], "message_sizes": []} + server_id = "00000000-0000-0000-0000-000000000001" + + def configure(raw, context): + request = protocol("ConfigRequest").FromString(raw) + return protocol("ConfigResponse", session_id=request.session_id, + pairs=[dict(key=k, value=config[k]) for k in request.operation.get.keys] + ).SerializeToString() + + def statuses(raw, context): + request = protocol("ArtifactStatusesRequest").FromString(raw) + return protocol("ArtifactStatusesResponse", session_id=request.session_id, + statuses={name: dict(exists=name in state["cache"]) + for name in request.names}).SerializeToString() + + def upload(requests, context): + summaries = [] + name, expected, buffer = None, 0, bytearray() + session_id = None + + def save(name, data): + assert name == "cache/" + hashlib.sha256(data).hexdigest() + assert name not in state["cache"], "cached artifacts should not be uploaded again" + state["cache"][name] = bytes(data) + summaries.append(dict(name=name, is_crc_successful=True)) + + def checked(chunk): + assert zlib.crc32(chunk.data) == chunk.crc + return chunk.data + + for raw in requests: + state["message_sizes"].append(len(raw)) + request = protocol("AddArtifactsRequest").FromString(raw) + session_id = request.session_id + if request.HasField("batch"): + for artifact in request.batch.artifacts: + save(artifact.name, checked(artifact.data)) + elif request.HasField("begin_chunk"): + begin = request.begin_chunk + name, expected = begin.name, begin.total_bytes + buffer = bytearray(checked(begin.initial_chunk)) + else: + buffer.extend(checked(request.chunk)) + if name is not None and len(buffer) == expected: + save(name, buffer) + name, buffer = None, bytearray() + assert name is None + state["uploads"] += 1 + if upload_error == "crc": + summaries[0]["is_crc_successful"] = False + elif upload_error == "missing": + summaries = [] + return protocol("AddArtifactsResponse", session_id=session_id, + artifacts=summaries).SerializeToString() + + def execute(raw, context): + request = protocol("ExecutePlanRequest").FromString(raw) + relation = request.plan.root + state["relations"].append(relation) + if relation.HasField("chunked_cached_local_relation"): + cached = relation.chunked_cached_local_relation + payloads = [state["cache"]["cache/" + h] for h in cached.dataHashes] + state["spark_schema"] = state["cache"]["cache/" + cached.schemaHash].decode() + else: + payloads = [relation.local_relation.data] + rows, first, last = 0, None, None + for payload in payloads: + for batch in pa.ipc.open_stream(payload): + rows += batch.num_rows + if batch.num_rows: + if first is None: + first = batch.column(0)[0].as_py() + last = batch.column(0)[-1].as_py() + state["result"] = (rows, first, last) + result = pa.table({"rows": [rows]}) + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, result.schema) as writer: + writer.write_table(result) + header = dict(session_id=request.session_id, operation_id=request.operation_id, + server_side_session_id=server_id) + yield protocol("ExecutePlanResponse", **header, response_id="0", + arrow_batch=dict(row_count=1, data=sink.getvalue().to_pybytes()) + ).SerializeToString() + yield protocol("ExecutePlanResponse", **header, response_id="1", + result_complete={}).SerializeToString() + + def release(raw, context): + request = protocol("ReleaseExecuteRequest").FromString(raw) + return protocol("ReleaseExecuteResponse", session_id=request.session_id, + operation_id=request.operation_id, + server_side_session_id=server_id).SerializeToString() + + def stop(raw, context): + request = protocol("ReleaseSessionRequest").FromString(raw) + return protocol("ReleaseSessionResponse", session_id=request.session_id, + server_side_session_id=server_id).SerializeToString() + + with ThreadPoolExecutor(max_workers=2) as executor: + server = grpc.server(executor, options=[("grpc.max_receive_message_length", 4 * 1024**2)]) + server.add_generic_rpc_handlers([grpc.method_handlers_generic_handler( + "spark.connect.SparkConnectService", { + "Config": grpc.unary_unary_rpc_method_handler(configure), + "ArtifactStatus": grpc.unary_unary_rpc_method_handler(statuses), + "AddArtifacts": grpc.stream_unary_rpc_method_handler(upload), + "ExecutePlan": grpc.unary_stream_rpc_method_handler(execute), + "ReleaseExecute": grpc.unary_unary_rpc_method_handler(release), + "ReleaseSession": grpc.unary_unary_rpc_method_handler(stop), + })]) + port = server.add_insecure_port("127.0.0.1:0") + server.start() + session = SparkSession.builder.remote(f"sc://127.0.0.1:{port}").getOrCreate() + try: + yield session, state + finally: + session.stop() + server.stop(0).wait(timeout=5) + + +@pytest.mark.parametrize("rows", [2, 17000000]) +def test_arrow_input_upload_above_grpc_limit(protocol, rows): + import numpy as np + import json + + table = pa.table({"id": np.arange(rows, dtype=np.int64)}) + with _arrow_input_service(protocol) as (spark, state): + df = spark.createDataFrame(table) + assert df.isLocal() + assert df.toArrow().column(0).to_pylist() == [rows] + assert state["result"] == (rows, 0, rows - 1) + if rows == 2: + assert state["uploads"] == 0 + assert state["relations"][0].HasField("local_relation") + else: + assert table.nbytes > 128 * 1024**2 + assert state["relations"][0].HasField("chunked_cached_local_relation") + assert state["relations"][0].ByteSize() < 1024**2 + assert max(state["message_sizes"]) < 33 * 1024 + assert json.loads(state["spark_schema"])["fields"][0]["name"] == "id" + uploads = state["uploads"] + spark.createDataFrame(table).toArrow() + assert state["uploads"] == uploads + + +def test_arrow_input_upload_limits_and_chunk_order(protocol): + import numpy as np + from pyspark.errors import AnalysisException + table = pa.table({"id": np.arange(100, dtype=np.int64)}) + options = { + "spark.sql.session.localRelationCacheThreshold": "0", + "spark.sql.session.localRelationChunkSizeRows": "7", + "spark.sql.session.localRelationChunkSizeBytes": "1000", + "spark.sql.session.localRelationBatchOfChunksSizeBytes": "1000", + } + with _arrow_input_service(protocol, **options) as (spark, state): + df = spark.createDataFrame(table) + df.toArrow() + assert state["result"] == (100, 0, 99) + assert state["uploads"] > 1 + cached = state["relations"][0].chunked_cached_local_relation + assert len(cached.dataHashes) > 1 + for hash in cached.dataHashes: + assert pa.ipc.open_stream(state["cache"]["cache/" + hash]).read_all().num_rows <= 7 + + options["spark.sql.session.localRelationSizeLimit"] = "10" + with _arrow_input_service(protocol, **options) as (spark, state): + with pytest.raises(AnalysisException, match="LOCAL_RELATION_SIZE_LIMIT_EXCEEDED"): + spark.createDataFrame(table) + assert not state["relations"] + assert state["uploads"] == 0 + + +@pytest.mark.parametrize("failure,match", [("crc", "CRC check failed"), + ("missing", "Missing cache artifact")]) +def test_arrow_input_upload_failure(protocol, failure, match): + with _arrow_input_service(protocol, upload_error=failure, + **{"spark.sql.session.localRelationCacheThreshold": "0"}) as (spark, state): + with pytest.raises(Exception, match=match): + spark.createDataFrame(pa.table({"id": [1, 2]})) + assert not state["relations"] + + +def test_arrow_input_duplicate_chunks_preserve_multiplicity(protocol): + options = {"spark.sql.session.localRelationCacheThreshold": "0", + "spark.sql.session.localRelationChunkSizeRows": "5"} + with _arrow_input_service(protocol, **options) as (spark, state): + spark.createDataFrame(pa.table({"id": [1] * 20})).toArrow() + cached = state["relations"][0].chunked_cached_local_relation + assert len(cached.dataHashes) == 4 + assert len(set(cached.dataHashes)) == 1 + assert len(state["cache"]) == 2 # One schema and one unique data chunk. + assert state["result"] == (20, 1, 1) diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 7735741f..6cfddd32 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -3193,3 +3193,24 @@ def test_arrow_input_types_live_server(kind): assert df.toArrow().to_pylist() == table.to_pylist() finally: spark.stop() + + +def test_arrow_input_large_upload_live_server(): + """A table larger than one gRPC message must use uploaded cache artifacts.""" + import os + pa = pytest.importorskip("pyarrow") + remote = os.environ.get("SPARK_REMOTE") + if not remote: + pytest.skip("requires SPARK_REMOTE") + from pyspark.sql import SparkSession + + rows = 140000 + table = pa.table({"payload": [b"x" * 1024] * rows}) + assert table.nbytes > 128 * 1024**2 + spark = SparkSession.builder.remote(remote).getOrCreate() + try: + df = spark.createDataFrame(table) + assert df.count() == rows + assert df.selectExpr("sum(length(payload)) AS total").collect()[0][0] == rows * 1024 + finally: + spark.stop() From 3f5fde1d4ff16e21a6432bc9323511542358160a Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 30 Oct 2026 12:13:34 -0700 Subject: [PATCH 03/10] [SPARK-60675][CONNECT] Reduce cached Arrow upload overhead Use accelerated SHA-256 and shared payload storage, and send cache artifacts in 1 MiB frames while keeping sequential uploads. Document real remote Spark benchmarking as required before merge. Co-authored-by: Isaac --- Cargo.lock | 2 +- crates/spark-connect-core/Cargo.toml | 3 +- crates/spark-connect-core/src/artifact.rs | 164 ++++++++++++++++++---- crates/spark-connect-core/src/client.rs | 16 ++- dev/design/arrow-input-benchmark.md | 81 +++++++++++ python/tests/test_arrow_handoff.py | 27 ++-- 6 files changed, 254 insertions(+), 39 deletions(-) create mode 100644 dev/design/arrow-input-benchmark.md diff --git a/Cargo.lock b/Cargo.lock index 6238cb96..7c7cb4b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -86,8 +86,8 @@ dependencies = [ "prost", "prost-types", "regex", + "ring", "serde_json", - "sha2", "tempfile", "thiserror", "tokio", diff --git a/crates/spark-connect-core/Cargo.toml b/crates/spark-connect-core/Cargo.toml index 6cdf0e01..a180ae90 100644 --- a/crates/spark-connect-core/Cargo.toml +++ b/crates/spark-connect-core/Cargo.toml @@ -26,7 +26,8 @@ serde_json = "1" regex = "1" prost-types.workspace = true crc32fast = "1" -sha2 = "0.10" +# Reuse the TLS backend's accelerated SHA-256, including on ARM64. +ring = "0.17" [dev-dependencies] tempfile = "3" diff --git a/crates/spark-connect-core/src/artifact.rs b/crates/spark-connect-core/src/artifact.rs index 20affd29..4c40bbf0 100644 --- a/crates/spark-connect-core/src/artifact.rs +++ b/crates/spark-connect-core/src/artifact.rs @@ -20,8 +20,17 @@ use crate::error::SparkError; /// https://github.com/grpc/grpc.github.io/issues/371. pub const CHUNK_SIZE: usize = 32 * 1024; +/// Cache uploads amortize message overhead while staying below the default +/// 4 MiB gRPC receive limit. Other artifact uploads retain their existing size. +pub(crate) const CACHE_CHUNK_SIZE: usize = 1024 * 1024; + /// Local artifact data source (file or in-memory). pub trait ArtifactData: Send + Sync { + /// Share an in-memory payload without reading or copying it. + fn bytes(&self) -> Option { + None + } + /// Get the size of this artifact in bytes. fn size(&self) -> Result; @@ -56,16 +65,22 @@ impl ArtifactData for FileArtifact { /// An in-memory artifact. pub struct InMemoryArtifact { - data: Vec, + data: Bytes, } impl InMemoryArtifact { pub fn new(data: impl Into>) -> Self { - Self { data: data.into() } + Self { + data: Bytes::from(data.into()), + } } } impl ArtifactData for InMemoryArtifact { + fn bytes(&self) -> Option { + Some(self.data.clone()) + } + fn size(&self) -> Result { Ok(self.data.len() as u64) } @@ -100,6 +115,16 @@ impl NamedArtifact { pub fn build_artifact_request_stream( session_id: String, artifacts: Vec, +) -> Result> { + build_artifact_request_stream_with_chunk_size(session_id, artifacts, CHUNK_SIZE) +} + +/// Cache payloads can use larger messages to amortize framing overhead while +/// keeping each message well below the server's default 4 MiB receive limit. +pub(crate) fn build_artifact_request_stream_with_chunk_size( + session_id: String, + artifacts: Vec, + chunk_size: usize, ) -> Result> { let mut requests = Vec::new(); let mut batch_artifacts = Vec::new(); @@ -108,7 +133,7 @@ pub fn build_artifact_request_stream( for artifact in artifacts { let size = artifact.data.size()? as usize; - if size > CHUNK_SIZE { + if size > chunk_size { // Flush batch if non-empty if !batch_artifacts.is_empty() { let mut request = AddArtifactsRequest::default(); @@ -127,10 +152,14 @@ pub fn build_artifact_request_stream( requests.extend(build_chunked_artifact_requests( session_id.clone(), artifact, + chunk_size, )?); } else { // Add to batch - if batch_size + size > CHUNK_SIZE && !batch_artifacts.is_empty() { + // Include names and protobuf framing so many tiny cache blobs cannot + // turn a small payload batch into an oversized gRPC message. + let framed_size = size + artifact.name.len() + 32; + if batch_size + framed_size > chunk_size && !batch_artifacts.is_empty() { // Flush current batch let mut request = AddArtifactsRequest::default(); request.session_id = session_id.clone(); @@ -145,21 +174,24 @@ pub fn build_artifact_request_stream( } // Read artifact data - let mut stream = artifact.data.stream()?; - let mut data = Vec::new(); - stream - .read_to_end(&mut data) - .map_err(|e| SparkError::connect_msg(format!("Failed to read artifact: {}", e)))?; + let data = match artifact.data.bytes() { + Some(data) => data, + None => { + let mut stream = artifact.data.stream()?; + let mut data = Vec::new(); + stream.read_to_end(&mut data).map_err(|e| { + SparkError::connect_msg(format!("Failed to read artifact: {}", e)) + })?; + Bytes::from(data) + } + }; let crc = compute_crc32(&data) as i64; batch_artifacts.push(add_artifacts_request::SingleChunkArtifact { name: artifact.name, - data: Some(add_artifacts_request::ArtifactChunk { - data: Bytes::from(data), - crc, - }), + data: Some(add_artifacts_request::ArtifactChunk { data, crc }), }); - batch_size += size; + batch_size += framed_size; } } @@ -182,24 +214,41 @@ pub fn build_artifact_request_stream( fn build_chunked_artifact_requests( session_id: String, artifact: NamedArtifact, + chunk_size: usize, ) -> Result> { let mut requests = Vec::new(); let total_size = artifact.data.size()? as i64; - let num_chunks = (total_size as usize).div_ceil(CHUNK_SIZE); - - let mut stream = artifact.data.stream()?; + let num_chunks = (total_size as usize).div_ceil(chunk_size); + + let shared = artifact.data.bytes(); + let mut stream = if shared.is_none() { + Some(artifact.data.stream()?) + } else { + None + }; let mut first_chunk = true; - let mut buf = vec![0u8; CHUNK_SIZE]; + let mut buf = if shared.is_none() { + vec![0u8; chunk_size] + } else { + Vec::new() + }; + let mut offset = 0; loop { - let n = stream.read(&mut buf).map_err(|e| { - SparkError::connect_msg(format!("Failed to read artifact chunk: {}", e)) - })?; - if n == 0 { + let chunk_data = if let Some(data) = &shared { + let end = (offset + chunk_size).min(data.len()); + let chunk = data.slice(offset..end); + offset = end; + chunk + } else { + let n = stream.as_mut().unwrap().read(&mut buf).map_err(|e| { + SparkError::connect_msg(format!("Failed to read artifact chunk: {}", e)) + })?; + Bytes::copy_from_slice(&buf[..n]) + }; + if chunk_data.is_empty() { break; } - - let chunk_data = buf[..n].to_vec(); let crc = compute_crc32(&chunk_data) as i64; if first_chunk { @@ -212,7 +261,7 @@ fn build_chunked_artifact_requests( total_bytes: total_size, num_chunks: num_chunks as i64, initial_chunk: Some(add_artifacts_request::ArtifactChunk { - data: Bytes::from(chunk_data), + data: chunk_data, crc, }), }, @@ -225,7 +274,7 @@ fn build_chunked_artifact_requests( request.session_id = session_id.clone(); request.payload = Some(add_artifacts_request::Payload::Chunk( add_artifacts_request::ArtifactChunk { - data: Bytes::from(chunk_data), + data: chunk_data, crc, }, )); @@ -344,4 +393,67 @@ mod tests { // Multiple small artifacts should be batched into one request assert_eq!(requests.len(), 1); } + + #[test] + fn shared_payload_matches_file_framing_and_survives_source_drop() { + for chunk_size in [CHUNK_SIZE, CACHE_CHUNK_SIZE] { + for size in [0, chunk_size, chunk_size + 1, chunk_size * 3 + 17] { + let data: Vec = (0..size).map(|i| (i % 251) as u8).collect(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("payload"); + std::fs::write(&path, &data).unwrap(); + let memory = InMemoryArtifact::new(data); + let shared = memory.bytes().unwrap(); + let requests = build_artifact_request_stream_with_chunk_size( + "session".into(), + vec![NamedArtifact::new("cache/hash".into(), Box::new(memory))], + chunk_size, + ) + .unwrap(); + let file_requests = build_artifact_request_stream_with_chunk_size( + "session".into(), + vec![NamedArtifact::new( + "cache/hash".into(), + Box::new(FileArtifact::new(path)), + )], + chunk_size, + ) + .unwrap(); + assert_eq!(requests, file_requests); + if size > chunk_size { + let Some(add_artifacts_request::Payload::BeginChunk(first)) = + &requests[0].payload + else { + panic!("expected chunked artifact"); + }; + assert_eq!( + first.initial_chunk.as_ref().unwrap().data.as_ptr(), + shared.as_ptr() + ); + } + } + } + } + + #[test] + fn small_artifact_batches_include_name_and_framing_overhead() { + use prost::Message; + + let artifacts = (0..100) + .map(|i| { + NamedArtifact::new( + format!("cache/{i:064x}"), + Box::new(InMemoryArtifact::new(vec![i as u8; 2])), + ) + }) + .collect(); + let requests = + build_artifact_request_stream_with_chunk_size("session".into(), artifacts, 1024) + .unwrap(); + assert!(requests.len() > 1); + for request in requests { + // Allow the session/enclosing-message fields above the batch budget. + assert!(request.encoded_len() <= 1024 + 32); + } + } } diff --git a/crates/spark-connect-core/src/client.rs b/crates/spark-connect-core/src/client.rs index 8f658bdc..c3faf1e3 100644 --- a/crates/spark-connect-core/src/client.rs +++ b/crates/spark-connect-core/src/client.rs @@ -804,12 +804,18 @@ impl SparkConnectClient { /// Existing artifacts and duplicate blobs are uploaded only once. pub async fn cache_artifacts(&self, blobs: Vec>) -> Result> { use crate::artifact::InMemoryArtifact; - use sha2::{Digest, Sha256}; + use ring::digest::{digest, SHA256}; use std::collections::HashSet; let hashes: Vec = blobs .iter() - .map(|blob| format!("{:x}", Sha256::digest(blob))) + .map(|blob| { + digest(&SHA256, blob) + .as_ref() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::() + }) .collect(); let names: Vec = hashes.iter().map(|hash| format!("cache/{hash}")).collect(); let unique: HashSet<&str> = names.iter().map(String::as_str).collect(); @@ -833,7 +839,11 @@ impl SparkConnectClient { return Ok(hashes); } let expected: HashSet = artifacts.iter().map(|a| a.name.clone()).collect(); - let mut requests = build_artifact_request_stream(self.session_id.clone(), artifacts)?; + let mut requests = crate::artifact::build_artifact_request_stream_with_chunk_size( + self.session_id.clone(), + artifacts, + crate::artifact::CACHE_CHUNK_SIZE, + )?; for request in &mut requests { request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), diff --git a/dev/design/arrow-input-benchmark.md b/dev/design/arrow-input-benchmark.md new file mode 100644 index 00000000..23c733c1 --- /dev/null +++ b/dev/design/arrow-input-benchmark.md @@ -0,0 +1,81 @@ +# Arrow input upload benchmarking + +## Current decision + +Keep sequential cache uploads with 1 MiB artifact frames, accelerated SHA-256, +and shared payload buffers. Generic artifact uploads retain 32 KiB frames. +Keep hash order, CRC validation, acknowledgements, and server-side deduplication. + +Two concurrent upload streams did not improve the localhost experiment. +Overlapping serialization and upload reduced latency slightly, but increased +peak client RSS growth from approximately 93 MiB to 145 MiB for a 136 MB input. +Neither concurrency prototype is retained. + +The selected framing change reduced the 136 MB creation-plus-query median from +230.58 ms to 138.31 ms; official PySpark measured 307.30 ms in the same experiment. +These are **localhost fixture measurements without Spark query execution**, +using ten fresh-process samples per client. They do not establish remote Spark +performance. A separate live localhost Spark check verified the uploaded values. + +## Required before merging: real remote Spark measurements + +Remote performance validation is **pending**. Benchmark against an actual Spark +Connect server on another machine over the deployment's normal network and TLS +path. A local JVM, a scripted fixture, or injected latency alone does not satisfy +this requirement. Obtain the remote endpoint and configured authentication from +the operator; keep credentials out of commands, reports, and checked-in files. + +Compare all three clients against that same server: + +- Official PySpark, matching the server version. +- The previous optimized sequential client with 32 KiB frames, including the + accelerated hash and shared-buffer changes. Comparing only against the earlier + software-hash implementation would overstate the framing improvement. +- The proposed sequential client with 1 MiB cache frames. + +Use the same Python interpreter and dependency versions. Record exact native +binary hashes and source revisions or patches. Run samples serially, rotating +client order, with at least ten fresh-process samples per client and case. + +### Workloads and measurement boundaries + +Use eight int64 columns at these sizes: + +| Rows | Unencoded input size | +| ---: | ---: | +| 12,500 | 0.8 MB | +| 250,000 | 16 MB | +| 2,125,000 | 136 MB | + +Create the input and establish a fresh session outside the measurement. Warm +server execution with an unrelated small query, without uploading the measured +input. Do not reuse a fixed session ID across cold-input samples: a cached hash +would conceal the upload cost. + +Measure creation plus a materializing query as the primary comparison, including +all uploads. Aggregate all eight columns, checking row count and column sums +against the generated data, so server execution consumes the complete relation. +Also measure repeated queries on a prepared DataFrame after one untimed query; +report these separately from the initial upload. Never warm up cold-input timing +by creating that same input in the same session. + +Keep real server configuration unchanged for the primary comparison. Record +the cache threshold, local relation size limit, chunk row/byte settings, and +upload-batch cap. The fixture used a 64 MiB upload-batch cap; the real server's +default is 1 GiB, so fixture memory and RPC counts cannot be extrapolated directly. + +### Report and decision + +Record client/server versions and hardware, client/server locations, TLS mode, +small-query round-trip latency, driver heap, and concurrent server activity. +Report raw samples, medians and ranges, client CPU, sampled peak RSS above the +prepared-input baseline, failures, and upload/network counters where available. +Keep server execution time visible rather than attributing the entire query +latency to the client. Record sampling intervals and any unavailable counters. + +The 0.8 MB input is a control: the framing change does not optimize its inline +path, which remains slower than PySpark in the localhost results. Preserve this +result alongside larger-input gains. + +Reassess frame size if the remote workload regresses. Do not enable parallel +uploads or claim an end-to-end remote speedup solely from the localhost results. diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index 70dbea0b..f969a130 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -533,12 +533,15 @@ def stop(raw, context): @pytest.mark.parametrize("rows", [2, 17000000]) -def test_arrow_input_upload_above_grpc_limit(protocol, rows): +@pytest.mark.parametrize("batch_bytes", [64 * 1024**2, 1024**3]) +def test_arrow_input_upload_above_grpc_limit(protocol, rows, batch_bytes): import numpy as np import json table = pa.table({"id": np.arange(rows, dtype=np.int64)}) - with _arrow_input_service(protocol) as (spark, state): + with _arrow_input_service(protocol, **{ + "spark.sql.session.localRelationBatchOfChunksSizeBytes": str(batch_bytes) + }) as (spark, state): df = spark.createDataFrame(table) assert df.isLocal() assert df.toArrow().column(0).to_pylist() == [rows] @@ -550,7 +553,7 @@ def test_arrow_input_upload_above_grpc_limit(protocol, rows): assert table.nbytes > 128 * 1024**2 assert state["relations"][0].HasField("chunked_cached_local_relation") assert state["relations"][0].ByteSize() < 1024**2 - assert max(state["message_sizes"]) < 33 * 1024 + assert max(state["message_sizes"]) < 1025 * 1024 assert json.loads(state["spark_schema"])["fields"][0]["name"] == "id" uploads = state["uploads"] spark.createDataFrame(table).toArrow() @@ -587,17 +590,25 @@ def test_arrow_input_upload_limits_and_chunk_order(protocol): @pytest.mark.parametrize("failure,match", [("crc", "CRC check failed"), ("missing", "Missing cache artifact")]) -def test_arrow_input_upload_failure(protocol, failure, match): +@pytest.mark.parametrize("rows", [2, 4000000]) +def test_arrow_input_upload_failure(protocol, failure, match, rows): + import numpy as np + + # The larger input spans multiple upload batches. with _arrow_input_service(protocol, upload_error=failure, - **{"spark.sql.session.localRelationCacheThreshold": "0"}) as (spark, state): + **{"spark.sql.session.localRelationCacheThreshold": "0", + "spark.sql.session.localRelationBatchOfChunksSizeBytes": + str(8 * 1024**2)}) as (spark, state): with pytest.raises(Exception, match=match): - spark.createDataFrame(pa.table({"id": [1, 2]})) + spark.createDataFrame(pa.table({"id": np.arange(rows, dtype=np.int64)})) assert not state["relations"] -def test_arrow_input_duplicate_chunks_preserve_multiplicity(protocol): +@pytest.mark.parametrize("batch_bytes", [500, 1024**3]) +def test_arrow_input_duplicate_chunks_preserve_multiplicity(protocol, batch_bytes): options = {"spark.sql.session.localRelationCacheThreshold": "0", - "spark.sql.session.localRelationChunkSizeRows": "5"} + "spark.sql.session.localRelationChunkSizeRows": "5", + "spark.sql.session.localRelationBatchOfChunksSizeBytes": str(batch_bytes)} with _arrow_input_service(protocol, **options) as (spark, state): spark.createDataFrame(pa.table({"id": [1] * 20})).toArrow() cached = state["relations"][0].chunked_cached_local_relation From efd5d750cab6b63789e556d782c1498d3ff7394b Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 30 Oct 2026 12:53:47 -0700 Subject: [PATCH 04/10] [SPARK-60675][CONNECT] Batch Arrow input settings with safe fallbacks Read normalization and upload settings in one optional configuration RPC, preserve server defaults, and defer chunk validation until caching. Avoid importing PyArrow for other input types and share artifact context and response validation. Co-authored-by: Isaac --- crates/pyspark-rs/src/pyarrow_import.rs | 31 +++---- crates/pyspark-rs/src/session.rs | 11 ++- crates/spark-connect-core/src/client.rs | 112 ++++++++++++------------ crates/spark-connect/src/arrow_input.rs | 99 +++++++++++++++------ crates/spark-connect/src/lib.rs | 1 + crates/spark-connect/src/session.rs | 19 +++- dev/design/arrow-input-benchmark.md | 4 +- python/tests/test_arrow_handoff.py | 89 ++++++++++++++++++- 8 files changed, 261 insertions(+), 105 deletions(-) diff --git a/crates/pyspark-rs/src/pyarrow_import.rs b/crates/pyspark-rs/src/pyarrow_import.rs index f1b083be..6e2e37b2 100644 --- a/crates/pyspark-rs/src/pyarrow_import.rs +++ b/crates/pyspark-rs/src/pyarrow_import.rs @@ -43,27 +43,20 @@ pub(crate) fn create_dataframe( let schema = schema.unwrap_or_else(|| py.None()); // Empty input needs only the explicit schema, and no server configuration. let empty: bool = table.call_method0("__len__")?.extract::()? == 0; - let (timestamp_type, timezone, large_types) = if empty { - (None, None, None) + let options = if empty { + None } else { - py.detach(|| -> PyResult<_> { - let conf = session.conf(); - Ok(( - conf.get("spark.sql.timestampType").to_pyerr()?, - conf.get("spark.sql.session.timeZone").to_pyerr()?, - conf.get("spark.sql.execution.arrow.useLargeVarTypes") - .to_pyerr()?, - )) - })? + Some(py.detach(|| session.arrow_input_options().to_pyerr())?) }; let result = helper.getattr("prepare_table")?.call1(( table, schema, - timestamp_type, - timezone, - large_types, + options.as_ref().and_then(|o| o.timestamp_type()), + options.as_ref().and_then(|o| o.timezone()), + options.as_ref().and_then(|o| o.use_large_var_types()), ))?; let normalized = result.get_item(0)?; + // Native local relations carry inferred schemas to preserve timestamp-type normalization. let spark_schema = crate::types::py_to_data_type(&result.get_item(1)?)?; let names: Option> = result.get_item(2)?.extract()?; let batches = if empty { @@ -75,9 +68,13 @@ pub(crate) fn create_dataframe( .0 }; let dataframe = py.detach(|| { - session - .create_dataframe_from_arrow(batches, spark_schema) - .to_pyerr() + match options.as_ref() { + Some(options) => { + session.create_dataframe_from_arrow_with_options(batches, spark_schema, options) + } + None => session.create_dataframe_from_arrow(batches, spark_schema), + } + .to_pyerr() })?; let dataframe = match names { Some(names) => dataframe.to_df(names.iter().map(String::as_str).collect()), diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs index 0e271dbd..80fb6058 100644 --- a/crates/pyspark-rs/src/session.rs +++ b/crates/pyspark-rs/src/session.rs @@ -348,10 +348,13 @@ impl PySparkSession { // signature parity. See PARITY_GAP_ANALYSIS.md. let _ = (samplingRatio, verifySchema); - if let Ok(pa) = py.import("pyarrow") { - if data.is_instance(&pa.getattr("Table")?)? { - return crate::pyarrow_import::create_dataframe(py, &self.session, data, schema); - } + // A caller holding a Table has already imported PyArrow. + let pa = py + .import("sys")? + .getattr("modules")? + .call_method1("get", ("pyarrow",))?; + if !pa.is_none() && data.is_instance(&pa.getattr("Table")?)? { + return crate::pyarrow_import::create_dataframe(py, &self.session, data, schema); } // Accept a pandas DataFrame (as pyspark does): convert it to a list of row lists diff --git a/crates/spark-connect-core/src/client.rs b/crates/spark-connect-core/src/client.rs index c3faf1e3..a0e7a646 100644 --- a/crates/spark-connect-core/src/client.rs +++ b/crates/spark-connect-core/src/client.rs @@ -445,6 +445,38 @@ impl SparkConnectClient { Ok(keys.iter().map(|k| config_dict.get(*k).cloned()).collect()) } + /// Get registered defaults and session overrides, returning None for unknown keys. + pub async fn get_config_options(&self, keys: &[&str]) -> Result>> { + let request = ConfigRequest { + session_id: self.session_id.clone(), + user_context: Some(UserContext { + user_id: self.user_id.clone().unwrap_or_default(), + ..Default::default() + }), + operation: Some(spark_connect_proto::config_request::Operation { + op_type: Some( + spark_connect_proto::config_request::operation::OpType::GetOption( + spark_connect_proto::config_request::GetOption { + keys: keys.iter().map(|key| key.to_string()).collect(), + }, + ), + ), + }), + ..Default::default() + }; + let values: BTreeMap<_, _> = self + .config(request) + .await? + .pairs + .into_iter() + .map(|pair| (pair.key, pair.value)) + .collect(); + Ok(keys + .iter() + .map(|key| values.get(*key).cloned().flatten()) + .collect()) + } + /// Set configuration values for the session. /// /// Mirrors `pyspark.sql.connect.client.core.SparkConnectClient.set_config`. @@ -739,33 +771,7 @@ impl SparkConnectClient { // Build request stream with proper chunking/batching let requests = build_artifact_request_stream(self.session_id.clone(), artifacts)?; - - // Create a futures stream from the requests - let request_stream = futures::stream::iter(requests); - - // Call the AddArtifacts streaming RPC - let mut req = Request::new(request_stream); - self._attach_metadata(&mut req); - - let response = self - .stub - .clone() - .add_artifacts(req) - .await - .map_err(SparkError::from_grpc_status)? - .into_inner(); - - // Check that all artifacts were successfully received - for summary in response.artifacts { - if !summary.is_crc_successful { - return Err(SparkError::connect_msg(format!( - "CRC check failed for artifact: {}", - summary.name - ))); - } - } - - Ok(()) + self.upload_artifact_requests(requests, None).await } /// Upload a single local file as an artifact under an explicit artifact name @@ -779,25 +785,7 @@ impl SparkConnectClient { Box::new(FileArtifact::new(local_path)), )]; let requests = build_artifact_request_stream(self.session_id.clone(), artifacts)?; - let request_stream = futures::stream::iter(requests); - let mut req = Request::new(request_stream); - self._attach_metadata(&mut req); - let response = self - .stub - .clone() - .add_artifacts(req) - .await - .map_err(SparkError::from_grpc_status)? - .into_inner(); - for summary in response.artifacts { - if !summary.is_crc_successful { - return Err(SparkError::connect_msg(format!( - "CRC check failed for artifact: {}", - summary.name - ))); - } - } - Ok(()) + self.upload_artifact_requests(requests, None).await } /// Upload session-scoped cache blobs, returning SHA-256 hashes in input order. @@ -839,11 +827,23 @@ impl SparkConnectClient { return Ok(hashes); } let expected: HashSet = artifacts.iter().map(|a| a.name.clone()).collect(); - let mut requests = crate::artifact::build_artifact_request_stream_with_chunk_size( + let requests = crate::artifact::build_artifact_request_stream_with_chunk_size( self.session_id.clone(), artifacts, crate::artifact::CACHE_CHUNK_SIZE, )?; + self.upload_artifact_requests(requests, Some(&expected)) + .await?; + Ok(hashes) + } + + /// Send artifact frames with the same user context and validate the response. + /// Cache uploads also require an acknowledgement for every requested artifact. + async fn upload_artifact_requests( + &self, + mut requests: Vec, + expected_cache_names: Option<&std::collections::HashSet>, + ) -> Result<()> { for request in &mut requests { request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), @@ -859,7 +859,7 @@ impl SparkConnectClient { .await .map_err(SparkError::from_grpc_status)? .into_inner(); - let mut received = HashSet::new(); + let mut received = std::collections::HashSet::new(); for summary in response.artifacts { if !summary.is_crc_successful { return Err(SparkError::connect_msg(format!( @@ -867,14 +867,18 @@ impl SparkConnectClient { summary.name ))); } - received.insert(summary.name); + if expected_cache_names.is_some() { + received.insert(summary.name); + } } - if !expected.is_subset(&received) { - return Err(SparkError::connect_msg( - "Missing cache artifact upload acknowledgement", - )); + if let Some(expected) = expected_cache_names { + if !expected.is_subset(&received) { + return Err(SparkError::connect_msg( + "Missing cache artifact upload acknowledgement", + )); + } } - Ok(hashes) + Ok(()) } /// Check artifact status on the server. diff --git a/crates/spark-connect/src/arrow_input.rs b/crates/spark-connect/src/arrow_input.rs index 8ef091e3..a6aa6c81 100644 --- a/crates/spark-connect/src/arrow_input.rs +++ b/crates/spark-connect/src/arrow_input.rs @@ -10,8 +10,62 @@ use spark_connect_core::runtime::block_on; use crate::plan::LogicalPlan; use crate::types::DataType; +/// A snapshot of server settings used to normalize and upload one Arrow input. +/// Reuse it when normalization and IPC serialization happen in separate layers. +pub struct ArrowInputOptions { + values: Vec>, +} + +impl ArrowInputOptions { + const KEYS: [&'static str; 8] = [ + "spark.sql.timestampType", + "spark.sql.session.timeZone", + "spark.sql.execution.arrow.useLargeVarTypes", + "spark.sql.session.localRelationCacheThreshold", + "spark.sql.session.localRelationSizeLimit", + "spark.sql.session.localRelationChunkSizeRows", + "spark.sql.session.localRelationChunkSizeBytes", + "spark.sql.session.localRelationBatchOfChunksSizeBytes", + ]; + // SQLConf defaults for servers that do not expose the upload settings. + const UPLOAD_DEFAULTS: [&'static str; 5] = + ["1048576", "3221225472", "10000", "16777216", "1073741824"]; + + pub(crate) fn from_client(client: &SparkConnectClient) -> Result { + Ok(Self { + values: block_on(client.get_config_options(&Self::KEYS))?, + }) + } + + /// Server timestamp type, including its registered default. + pub fn timestamp_type(&self) -> Option<&str> { + self.values[0].as_deref() + } + + /// Server session timezone; do not substitute a client-local timezone. + pub fn timezone(&self) -> Option<&str> { + self.values[1].as_deref() + } + + /// Whether the server prefers large Arrow string and binary offsets. + pub fn use_large_var_types(&self) -> Option<&str> { + self.values[2].as_deref() + } + + fn upload_value(&self, index: usize) -> Result { + self.values[index + 3] + .as_deref() + .unwrap_or(Self::UPLOAD_DEFAULTS[index]) + .parse::() + .ok() + .filter(|value| index == 0 || *value > 0) + .ok_or_else(|| { + SparkError::connect_msg(format!("Invalid configuration: {}", Self::KEYS[index + 3])) + }) + } +} + struct Options { - cache_threshold: usize, size_limit: usize, chunk_rows: usize, chunk_bytes: usize, @@ -19,31 +73,14 @@ struct Options { } impl Options { - fn from_client(client: &SparkConnectClient) -> Result { - let keys = [ - "spark.sql.session.localRelationCacheThreshold", - "spark.sql.session.localRelationSizeLimit", - "spark.sql.session.localRelationChunkSizeRows", - "spark.sql.session.localRelationChunkSizeBytes", - "spark.sql.session.localRelationBatchOfChunksSizeBytes", - ]; - let values = block_on(client.get_configs(&keys))?; - let mut parsed = Vec::new(); - for (i, key) in keys.iter().enumerate() { - let value = values - .get(i) - .and_then(Option::as_deref) - .and_then(|v| v.parse::().ok()) - .filter(|v| i == 0 || *v > 0) - .ok_or_else(|| { - SparkError::connect_msg(format!("Invalid or missing configuration: {key}")) - })?; - parsed.push(value); - } + fn from_input(options: &ArrowInputOptions) -> Result { + let parsed = (0..5) + .map(|index| options.upload_value(index)) + .collect::>>()?; Ok(Self { - cache_threshold: parsed[0], size_limit: parsed[1], chunk_rows: parsed[2], + // Match upstream's min(chunk size, upload batch size). chunk_bytes: parsed[3].min(parsed[4]), upload_batch_bytes: parsed[4], }) @@ -81,13 +118,21 @@ pub(crate) fn create_relation( client: &SparkConnectClient, batches: Vec, schema: DataType, + options: Option<&ArrowInputOptions>, ) -> Result { // Validate before any RPC, and preserve the offline empty-input path. validate(&batches)?; if batches.iter().all(|batch| batch.num_rows() == 0) { return inline_relation(&batches, schema); } - let options = Options::from_client(client)?; + let fetched; + let options = match options { + Some(options) => options, + None => { + fetched = ArrowInputOptions::from_client(client)?; + &fetched + } + }; // Count the logical slices rather than their potentially much larger backing buffers. let bytes: usize = batches .iter() @@ -98,9 +143,11 @@ pub(crate) fn create_relation( .unwrap_or_else(|_| a.get_buffer_memory_size()) }) .sum(); - if bytes < options.cache_threshold { + if bytes < options.upload_value(0)? { return inline_relation(&batches, schema); } + // Chunk and batch settings are irrelevant to inline inputs. + let options = Options::from_input(options)?; let mut pending = Vec::new(); let mut pending_size = 0; @@ -128,6 +175,7 @@ pub(crate) fn create_relation( pending.push(chunk); Ok(()) }; + // Carry inferred schemas too, preserving the client's timestamp-type normalization. enqueue(schema.json().into_bytes())?; serialize_chunks(&batches, &options, &mut enqueue)?; hashes.extend(block_on(client.cache_artifacts(pending))?); @@ -195,7 +243,6 @@ mod tests { .unwrap(); for (chunk_rows, chunk_bytes) in [(700, usize::MAX), (10000, 1024)] { let options = Options { - cache_threshold: 0, size_limit: usize::MAX, chunk_rows, chunk_bytes, diff --git a/crates/spark-connect/src/lib.rs b/crates/spark-connect/src/lib.rs index 180bf089..60018059 100644 --- a/crates/spark-connect/src/lib.rs +++ b/crates/spark-connect/src/lib.rs @@ -36,6 +36,7 @@ pub mod wasm_udf; pub mod window; // Re-export commonly used types +pub use arrow_input::ArrowInputOptions; pub use column::{col, lit, lit_boolean, lit_double, lit_string, Column}; pub use dataframe::{DataFrame, LocalRowIterator}; pub use datasource::{CommonInlineUserDefinedDataSourceExpression, PythonDataSourcePayload}; diff --git a/crates/spark-connect/src/session.rs b/crates/spark-connect/src/session.rs index ce92949e..789039e6 100644 --- a/crates/spark-connect/src/session.rs +++ b/crates/spark-connect/src/session.rs @@ -301,7 +301,24 @@ impl SparkSession { batches: Vec, schema: DataType, ) -> Result { - let plan = crate::arrow_input::create_relation(&self.client, batches, schema)?; + let plan = crate::arrow_input::create_relation(&self.client, batches, schema, None)?; + Ok(DataFrame::new(self.clone(), plan)) + } + + /// Read Arrow normalization and upload settings together in one configuration RPC. + pub fn arrow_input_options(&self) -> Result { + crate::ArrowInputOptions::from_client(&self.client) + } + + /// Create an Arrow DataFrame using the same settings snapshot as normalization. + pub fn create_dataframe_from_arrow_with_options( + &self, + batches: Vec, + schema: DataType, + options: &crate::ArrowInputOptions, + ) -> Result { + let plan = + crate::arrow_input::create_relation(&self.client, batches, schema, Some(options))?; Ok(DataFrame::new(self.clone(), plan)) } diff --git a/dev/design/arrow-input-benchmark.md b/dev/design/arrow-input-benchmark.md index 23c733c1..f63fa9bf 100644 --- a/dev/design/arrow-input-benchmark.md +++ b/dev/design/arrow-input-benchmark.md @@ -16,6 +16,8 @@ The selected framing change reduced the 136 MB creation-plus-query median from These are **localhost fixture measurements without Spark query execution**, using ten fresh-process samples per client. They do not establish remote Spark performance. A separate live localhost Spark check verified the uploaded values. +These measurements preceded the follow-up that batches normalization and upload +settings into one configuration RPC; they are not a rerun of that follow-up. ## Required before merging: real remote Spark measurements @@ -74,7 +76,7 @@ Keep server execution time visible rather than attributing the entire query latency to the client. Record sampling intervals and any unavailable counters. The 0.8 MB input is a control: the framing change does not optimize its inline -path, which remains slower than PySpark in the localhost results. Preserve this +path, which was slower than PySpark in these localhost results. Preserve this result alongside larger-input gains. Reassess frame size if the remote workload regresses. Do not enable parallel diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index f969a130..07fa7168 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -414,13 +414,19 @@ def _arrow_input_service(protocol, *, upload_error=None, **overrides): "spark.sql.session.localRelationBatchOfChunksSizeBytes": str(1024**3), } config.update(overrides) - state = {"cache": {}, "uploads": 0, "relations": [], "message_sizes": []} + state = {"cache": {}, "uploads": 0, "relations": [], "message_sizes": [], + "config_requests": []} server_id = "00000000-0000-0000-0000-000000000001" def configure(raw, context): request = protocol("ConfigRequest").FromString(raw) + state["user_context"] = request.user_context + operation = request.operation.WhichOneof("op_type") + keys = getattr(request.operation, operation).keys + state["config_requests"].append((operation, list(keys))) return protocol("ConfigResponse", session_id=request.session_id, - pairs=[dict(key=k, value=config[k]) for k in request.operation.get.keys] + pairs=[dict(key=k, value=config[k]) for k in keys + if config.get(k) is not None] ).SerializeToString() def statuses(raw, context): @@ -447,6 +453,8 @@ def checked(chunk): for raw in requests: state["message_sizes"].append(len(raw)) request = protocol("AddArtifactsRequest").FromString(raw) + assert request.HasField("user_context") + assert request.user_context == state["user_context"] session_id = request.session_id if request.HasField("batch"): for artifact in request.batch.artifacts: @@ -543,6 +551,9 @@ def test_arrow_input_upload_above_grpc_limit(protocol, rows, batch_bytes): "spark.sql.session.localRelationBatchOfChunksSizeBytes": str(batch_bytes) }) as (spark, state): df = spark.createDataFrame(table) + assert len(state["config_requests"]) == 1 + assert state["config_requests"][0][0] == "get_option" + assert len(state["config_requests"][0][1]) == 8 assert df.isLocal() assert df.toArrow().column(0).to_pylist() == [rows] assert state["result"] == (rows, 0, rows - 1) @@ -616,3 +627,77 @@ def test_arrow_input_duplicate_chunks_preserve_multiplicity(protocol, batch_byte assert len(set(cached.dataHashes)) == 1 assert len(state["cache"]) == 2 # One schema and one unique data chunk. assert state["result"] == (20, 1, 1) + + +@pytest.mark.parametrize("data", [[[1]], pd.DataFrame({"id": [1]})]) +def test_non_arrow_input_does_not_import_pyarrow(protocol, monkeypatch, data): + import builtins + import sys + + imported = [] + original_import = builtins.__import__ + + def import_module(name, *args, **kwargs): + if name == "pyarrow": + imported.append(name) + raise ImportError("PyArrow is unavailable") + return original_import(name, *args, **kwargs) + + with _arrow_input_service(protocol) as (spark, state): + monkeypatch.setitem(sys.modules, "pyarrow", None) + monkeypatch.setattr(builtins, "__import__", import_module) + spark.createDataFrame(data) + assert imported == [] + assert state["config_requests"] == [] + + +@pytest.mark.parametrize("cached", [False, True]) +@pytest.mark.parametrize("missing", [ + "spark.sql.session.localRelationSizeLimit", + "spark.sql.session.localRelationChunkSizeRows", + "spark.sql.session.localRelationChunkSizeBytes", + "spark.sql.session.localRelationBatchOfChunksSizeBytes", + "all", +]) +def test_arrow_input_missing_upload_settings_use_defaults(protocol, missing, cached): + options = {missing: None} if missing != "all" else { + f"spark.sql.session.localRelation{suffix}": None + for suffix in ["SizeLimit", "ChunkSizeRows", "ChunkSizeBytes", "BatchOfChunksSizeBytes"] + } + options["spark.sql.session.localRelationCacheThreshold"] = "0" if cached else "1048576" + with _arrow_input_service(protocol, **options) as (spark, state): + spark.createDataFrame(pa.table({"id": [1, 2]})).toArrow() + assert state["result"] == (2, 1, 2) + assert len(state["config_requests"]) == 1 + assert bool(state["uploads"]) == cached + + +def test_arrow_input_ignores_chunk_settings_for_inline_input(protocol): + options = { + "spark.sql.session.localRelationChunkSizeRows": "invalid", + "spark.sql.session.localRelationChunkSizeBytes": "0", + } + with _arrow_input_service(protocol, **options) as (spark, state): + spark.createDataFrame(pa.table({"id": [1, 2]})).toArrow() + assert state["result"] == (2, 1, 2) + assert state["uploads"] == 0 + assert len(state["config_requests"]) == 1 + options["spark.sql.session.localRelationCacheThreshold"] = "0" + with _arrow_input_service(protocol, **options) as (spark, state): + with pytest.raises(Exception, match="Invalid configuration"): + spark.createDataFrame(pa.table({"id": [1, 2]})) + assert state["uploads"] == 0 + + +@pytest.mark.parametrize("rows", [2, 200000]) +def test_arrow_input_missing_cache_threshold_uses_default(protocol, rows): + import numpy as np + + table = pa.table({"id": np.arange(rows, dtype=np.int64)}) + with _arrow_input_service(protocol, **{ + "spark.sql.session.localRelationCacheThreshold": None + }) as (spark, state): + spark.createDataFrame(table).toArrow() + assert state["result"] == (rows, 0, rows - 1) + assert bool(state["uploads"]) == (table.nbytes >= 1048576) + assert len(state["config_requests"]) == 1 From c1b4cb5c31e7b08c38b05591a79eb45ace793918 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 30 Oct 2026 13:20:59 -0700 Subject: [PATCH 05/10] [SPARK-60675][CONNECT] Validate Arrow schema widths and test stock uploads Reject incompatible explicit schemas before Arrow casts and hide the internal settings handoff from Rust documentation. Exercise cached Arrow input with stock local-relation settings in the live CI suite. Co-authored-by: Isaac --- crates/spark-connect/src/arrow_input.rs | 4 +- crates/spark-connect/src/lib.rs | 1 + crates/spark-connect/src/session.rs | 6 ++- python/pyspark/sql/_arrow_input.py | 16 ++++++- python/tests/test_arrow_e2e.py | 41 ++++++++++++++++- python/tests/test_dropin_offline.py | 59 +++++++++++++++++++++++++ 6 files changed, 121 insertions(+), 6 deletions(-) diff --git a/crates/spark-connect/src/arrow_input.rs b/crates/spark-connect/src/arrow_input.rs index a6aa6c81..31c7cb71 100644 --- a/crates/spark-connect/src/arrow_input.rs +++ b/crates/spark-connect/src/arrow_input.rs @@ -10,8 +10,8 @@ use spark_connect_core::runtime::block_on; use crate::plan::LogicalPlan; use crate::types::DataType; -/// A snapshot of server settings used to normalize and upload one Arrow input. -/// Reuse it when normalization and IPC serialization happen in separate layers. +/// Internal settings snapshot shared with the Python Arrow importer; not a stable API. +#[doc(hidden)] pub struct ArrowInputOptions { values: Vec>, } diff --git a/crates/spark-connect/src/lib.rs b/crates/spark-connect/src/lib.rs index 60018059..4ae2263a 100644 --- a/crates/spark-connect/src/lib.rs +++ b/crates/spark-connect/src/lib.rs @@ -36,6 +36,7 @@ pub mod wasm_udf; pub mod window; // Re-export commonly used types +#[doc(hidden)] pub use arrow_input::ArrowInputOptions; pub use column::{col, lit, lit_boolean, lit_double, lit_string, Column}; pub use dataframe::{DataFrame, LocalRowIterator}; diff --git a/crates/spark-connect/src/session.rs b/crates/spark-connect/src/session.rs index 789039e6..9f5a8cc9 100644 --- a/crates/spark-connect/src/session.rs +++ b/crates/spark-connect/src/session.rs @@ -305,12 +305,14 @@ impl SparkSession { Ok(DataFrame::new(self.clone(), plan)) } - /// Read Arrow normalization and upload settings together in one configuration RPC. + /// Internal settings handoff for the Python Arrow importer; not a stable API. + #[doc(hidden)] pub fn arrow_input_options(&self) -> Result { crate::ArrowInputOptions::from_client(&self.client) } - /// Create an Arrow DataFrame using the same settings snapshot as normalization. + /// Internal Arrow import entry point using the normalization settings snapshot. + #[doc(hidden)] pub fn create_dataframe_from_arrow_with_options( &self, batches: Vec, diff --git a/python/pyspark/sql/_arrow_input.py b/python/pyspark/sql/_arrow_input.py index f77540c8..3f467f4c 100644 --- a/python/pyspark/sql/_arrow_input.py +++ b/python/pyspark/sql/_arrow_input.py @@ -24,7 +24,7 @@ from_arrow_schema, to_arrow_schema, ) -from pyspark.sql.types import StructType +from pyspark.sql.types import AtomicType, StructType def prepare_table(table, schema, timestamp_type, timezone, large_types): @@ -36,6 +36,20 @@ def prepare_table(table, schema, timestamp_type, timezone, large_types): raise PySparkValueError(errorClass="CANNOT_INFER_EMPTY_SCHEMA", messageParameters={}) return table, schema, None + expected_columns = None + if isinstance(schema, StructType): + expected_columns = len(schema.fields) + elif isinstance(schema, AtomicType): + expected_columns = 1 + if expected_columns is not None and expected_columns != table.num_columns: + raise PySparkValueError( + errorClass="AXIS_LENGTH_MISMATCH", + messageParameters={ + "expected_length": str(expected_columns), + "actual_length": str(table.num_columns), + }, + ) + if names is not None: names.extend(f"_{i + 1}" for i in range(len(names), table.num_columns)) inferred = ( diff --git a/python/tests/test_arrow_e2e.py b/python/tests/test_arrow_e2e.py index dc76866d..f76876e3 100644 --- a/python/tests/test_arrow_e2e.py +++ b/python/tests/test_arrow_e2e.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""Native toArrow smoke tests against a live Spark Connect server.""" +"""Native Arrow input and output checks against a live Spark Connect server.""" import os from datetime import date, datetime, timezone @@ -65,3 +65,42 @@ def test_to_arrow_empty_result(session): table = session.range(0).toArrow() assert isinstance(table, pa.Table) assert table.num_rows == 0 + + +def test_create_dataframe_arrow_cached_upload(session): + """Exercise real-server config values and cached input above the default 1 MiB.""" + import numpy as np + + rows = 262144 + table = pa.table({"id": np.arange(rows, dtype=np.int64)}) + assert table.nbytes == 2 * 1024**2 + # Clear inherited session overrides so CI exercises registered server defaults. + session.conf.unset("spark.sql.session.localRelationCacheThreshold") + session.conf.unset("spark.sql.session.localRelationSizeLimit") + threshold = int(session.conf.get("spark.sql.session.localRelationCacheThreshold")) + assert threshold == 1024**2 + assert table.nbytes >= threshold + # Spark 4.2 declares this setting as longConf and returns decimal bytes. + assert session.conf.get("spark.sql.session.localRelationSizeLimit") == "3221225472" + + frame = session.createDataFrame(table) + actual = frame.selectExpr("count(*) AS rows", "sum(id) AS checksum").collect()[0] + assert actual[0] == rows + assert actual[1] == rows * (rows - 1) // 2 + assert frame.isLocal() + + +@pytest.mark.parametrize("schema,expected_width", [ + ("x long", "1"), + ("x long, y long, z long", "3"), + ("long", "1"), +]) +def test_create_dataframe_arrow_explicit_schema_width(session, schema, expected_width): + from pyspark.errors import PySparkValueError + + with pytest.raises(PySparkValueError) as error: + session.createDataFrame(pa.table({"x": [1], "y": [2]}), schema) + assert error.value.getCondition() == "AXIS_LENGTH_MISMATCH" + assert error.value.getMessageParameters() == { + "expected_length": expected_width, "actual_length": "2", + } diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 6cfddd32..1e278fd0 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -3113,6 +3113,65 @@ def test_arrow_input_empty_and_width_errors(): prepare_table(pa.table({"x": [1]}), ["a", "b"], "TIMESTAMP_LTZ", "UTC", "false") +@pytest.mark.parametrize("schema,expected_length", [ + pytest.param(T.StructType([T.StructField("x", T.IntegerType())]), "1", id="too-few-fields"), + pytest.param(T.StructType([ + T.StructField("x", T.LongType()), T.StructField("y", T.LongType()), + T.StructField("z", T.LongType()), + ]), "3", id="too-many-fields"), + pytest.param(T.LongType(), "1", id="atomic-multiple-columns"), +]) +def test_arrow_input_explicit_schema_width_errors(schema, expected_length): + pa = pytest.importorskip("pyarrow") + from pyspark.errors import PySparkValueError + from pyspark.sql._arrow_input import prepare_table + + # The first value would overflow the one-field IntegerType schema. Width + # validation must produce the structured Spark error before an Arrow cast. + table = pa.table({"x": [2**40], "y": [2]}) + with pytest.raises(PySparkValueError) as error: + prepare_table(table, schema, "TIMESTAMP_LTZ", "UTC", "false") + assert error.value.getCondition() == "AXIS_LENGTH_MISMATCH" + assert error.value.getMessageParameters() == { + "expected_length": expected_length, "actual_length": "2", + } + + +@pytest.mark.parametrize("schema,values", [ + pytest.param(T.StructType([T.StructField("x", T.LongType())]), + {"x": [1, 2]}, id="single-field"), + pytest.param(T.StructType([ + T.StructField("x", T.LongType()), T.StructField("y", T.LongType()), + ]), {"x": [1, 2], "y": [3, 4]}, id="multiple-fields"), + pytest.param(T.LongType(), {"x": [1, 2]}, id="atomic-single-column"), +]) +def test_arrow_input_explicit_schema_matching_width(schema, values): + pa = pytest.importorskip("pyarrow") + from pyspark.sql._arrow_input import prepare_table + + table = pa.table(values) + normalized, result, names = prepare_table(table, schema, "TIMESTAMP_LTZ", "UTC", "false") + assert normalized.equals(table) + assert result is schema + assert names is None + + +@pytest.mark.parametrize("schema", [ + T.StructType([]), + T.StructType([T.StructField("x", T.LongType()), T.StructField("y", T.LongType())]), + T.LongType(), +]) +def test_arrow_input_empty_explicit_schema_skips_width_validation(schema): + pa = pytest.importorskip("pyarrow") + from pyspark.sql._arrow_input import prepare_table + + table = pa.table({"x": pa.array([], type=pa.int64())}) + normalized, result, names = prepare_table(table, schema, None, None, None) + assert normalized is table + assert result is schema + assert names is None + + @pytest.mark.parametrize("timestamp_type,expected", [("TIMESTAMP_NTZ", T.TimestampNTZType), ("TIMESTAMP_LTZ", T.TimestampType)]) def test_arrow_input_timestamps(timestamp_type, expected): pa = pytest.importorskip("pyarrow") From 36189cc12a54523fe518ec518464eb19c29376a7 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 30 Oct 2026 13:41:36 -0700 Subject: [PATCH 06/10] [SPARK-60675][CONNECT] Harden Arrow input settings and schema validation Co-authored-by: Isaac --- crates/spark-connect/src/arrow_input.rs | 31 ++++++++++++-- python/pyspark/sql/_arrow_input.py | 11 ++++- python/tests/test_arrow_e2e.py | 54 +++++++++++++++++------- python/tests/test_arrow_handoff.py | 48 +++++++++++++++++++++ python/tests/test_dropin_offline.py | 55 +++++++++++++++++++++++++ 5 files changed, 180 insertions(+), 19 deletions(-) diff --git a/crates/spark-connect/src/arrow_input.rs b/crates/spark-connect/src/arrow_input.rs index 31c7cb71..5e26373f 100644 --- a/crates/spark-connect/src/arrow_input.rs +++ b/crates/spark-connect/src/arrow_input.rs @@ -53,11 +53,16 @@ impl ArrowInputOptions { } fn upload_value(&self, index: usize) -> Result { - self.values[index + 3] + let value = self.values[index + 3] .as_deref() - .unwrap_or(Self::UPLOAD_DEFAULTS[index]) - .parse::() - .ok() + .unwrap_or(Self::UPLOAD_DEFAULTS[index]); + // Spark 4.2 uses decimal bytes; older bytesConf values may include units. + let parsed = if index == 1 { + parse_byte_size(value) + } else { + value.parse::().ok() + }; + parsed .filter(|value| index == 0 || *value > 0) .ok_or_else(|| { SparkError::connect_msg(format!("Invalid configuration: {}", Self::KEYS[index + 3])) @@ -65,6 +70,24 @@ impl ArrowInputOptions { } } +fn parse_byte_size(value: &str) -> Option { + let value = value.trim().to_ascii_lowercase(); + let number = value.trim_end_matches(|c: char| c.is_ascii_alphabetic()); + let exponent = match &value[number.len()..] { + "" | "b" => 0, + "k" | "kb" => 1, + "m" | "mb" => 2, + "g" | "gb" => 3, + "t" | "tb" => 4, + "p" | "pb" => 5, + _ => return None, + }; + number + .parse::() + .ok()? + .checked_mul(1024usize.checked_pow(exponent)?) +} + struct Options { size_limit: usize, chunk_rows: usize, diff --git a/python/pyspark/sql/_arrow_input.py b/python/pyspark/sql/_arrow_input.py index 3f467f4c..3da01f70 100644 --- a/python/pyspark/sql/_arrow_input.py +++ b/python/pyspark/sql/_arrow_input.py @@ -18,7 +18,7 @@ """Schema normalization for the native Arrow Table input path.""" -from pyspark.errors import PySparkValueError +from pyspark.errors import PySparkTypeError, PySparkValueError from pyspark.sql.pandas.types import ( _check_arrow_table_timestamps_localize, from_arrow_schema, @@ -29,6 +29,15 @@ def prepare_table(table, schema, timestamp_type, timezone, large_types): """Match Connect's Arrow casts without converting any column to Python rows.""" + if schema is not None and not isinstance(schema, (AtomicType, StructType, list, tuple)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "schema", + "expected_type": "list, None or StructType", + "arg_type": type(schema).__name__, + }, + ) names = list(schema) if isinstance(schema, (list, tuple)) else None explicit = schema is not None and names is None if len(table) == 0: diff --git a/python/tests/test_arrow_e2e.py b/python/tests/test_arrow_e2e.py index f76876e3..1b7b998d 100644 --- a/python/tests/test_arrow_e2e.py +++ b/python/tests/test_arrow_e2e.py @@ -74,20 +74,24 @@ def test_create_dataframe_arrow_cached_upload(session): rows = 262144 table = pa.table({"id": np.arange(rows, dtype=np.int64)}) assert table.nbytes == 2 * 1024**2 - # Clear inherited session overrides so CI exercises registered server defaults. - session.conf.unset("spark.sql.session.localRelationCacheThreshold") - session.conf.unset("spark.sql.session.localRelationSizeLimit") - threshold = int(session.conf.get("spark.sql.session.localRelationCacheThreshold")) - assert threshold == 1024**2 - assert table.nbytes >= threshold - # Spark 4.2 declares this setting as longConf and returns decimal bytes. - assert session.conf.get("spark.sql.session.localRelationSizeLimit") == "3221225472" - - frame = session.createDataFrame(table) - actual = frame.selectExpr("count(*) AS rows", "sum(id) AS checksum").collect()[0] - assert actual[0] == rows - assert actual[1] == rows * (rows - 1) // 2 - assert frame.isLocal() + isolated = session.newSession() + try: + # Clear inherited session overrides so CI exercises registered server defaults. + isolated.conf.unset("spark.sql.session.localRelationCacheThreshold") + isolated.conf.unset("spark.sql.session.localRelationSizeLimit") + threshold = int(isolated.conf.get("spark.sql.session.localRelationCacheThreshold")) + assert threshold == 1024**2 + assert table.nbytes >= threshold + # Spark 4.2 declares this setting as longConf and returns decimal bytes. + assert isolated.conf.get("spark.sql.session.localRelationSizeLimit") == "3221225472" + + frame = isolated.createDataFrame(table) + actual = frame.selectExpr("count(*) AS rows", "sum(id) AS checksum").collect()[0] + assert actual[0] == rows + assert actual[1] == rows * (rows - 1) // 2 + assert frame.isLocal() + finally: + isolated.stop() @pytest.mark.parametrize("schema,expected_width", [ @@ -104,3 +108,25 @@ def test_create_dataframe_arrow_explicit_schema_width(session, schema, expected_ assert error.value.getMessageParameters() == { "expected_length": expected_width, "actual_length": "2", } + + +@pytest.mark.parametrize("schema_kind", ["array", "map"]) +@pytest.mark.parametrize("ddl", [False, True], ids=["datatype", "ddl"]) +@pytest.mark.parametrize("empty", [False, True], ids=["nonempty", "empty"]) +def test_create_dataframe_arrow_unsupported_schema(session, schema_kind, ddl, empty): + from pyspark.errors import PySparkTypeError + from pyspark.sql import types as T + + schema, expected_type = { + "array": (("array" if ddl else T.ArrayType(T.LongType())), "ArrayType"), + "map": (("map" if ddl else T.MapType(T.StringType(), T.LongType())), + "MapType"), + }[schema_kind] + table = pa.table({"x": pa.array([] if empty else [1], type=pa.int64())}) + with pytest.raises(PySparkTypeError) as error: + session.createDataFrame(table, schema) + assert error.value.getCondition() == "NOT_EXPECTED_TYPE" + assert error.value.getMessageParameters() == { + "arg_name": "schema", "expected_type": "list, None or StructType", + "arg_type": expected_type, + } diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index 07fa7168..c83ef9fc 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -599,6 +599,54 @@ def test_arrow_input_upload_limits_and_chunk_order(protocol): assert state["uploads"] == 0 +@pytest.mark.parametrize("size_limit", ["3221225472", "3221225472b", "3GB", " 3gb "]) +def test_arrow_input_upload_accepts_byte_size_limits(protocol, size_limit): + import numpy as np + + rows = 262144 + table = pa.table({"id": np.arange(rows, dtype=np.int64)}) + with _arrow_input_service(protocol, **{ + "spark.sql.session.localRelationSizeLimit": size_limit, + }) as (spark, state): + spark.createDataFrame(table).toArrow() + assert state["uploads"] > 0 + assert state["relations"][0].HasField("chunked_cached_local_relation") + assert state["result"] == (rows, 0, rows - 1) + + +@pytest.mark.parametrize("size_limit", ["1024", "1024b", "1KB"]) +def test_arrow_input_byte_size_limits_are_enforced(protocol, size_limit): + from pyspark.errors import AnalysisException + + with _arrow_input_service(protocol, **{ + "spark.sql.session.localRelationCacheThreshold": "0", + "spark.sql.session.localRelationSizeLimit": size_limit, + }) as (spark, state): + with pytest.raises(AnalysisException, match="LOCAL_RELATION_SIZE_LIMIT_EXCEEDED") as error: + spark.createDataFrame(pa.table({"id": list(range(1000))})) + assert "limit (1024 bytes)" in str(error.value) + assert state["uploads"] == 0 + + +@pytest.mark.parametrize("setting,value", [ + ("SizeLimit", "0"), + ("SizeLimit", "-1GB"), + ("SizeLimit", "1.5GB"), + ("SizeLimit", "3XB"), + ("SizeLimit", "18446744073709551616"), + ("SizeLimit", "18446744073709551615GB"), + ("ChunkSizeRows", "1KB"), +]) +def test_arrow_input_rejects_invalid_upload_sizes(protocol, setting, value): + key = f"spark.sql.session.localRelation{setting}" + with _arrow_input_service(protocol, **{ + "spark.sql.session.localRelationCacheThreshold": "0", key: value, + }) as (spark, state): + with pytest.raises(Exception, match=f"Invalid configuration: {key}"): + spark.createDataFrame(pa.table({"id": [1]})) + assert state["uploads"] == 0 + + @pytest.mark.parametrize("failure,match", [("crc", "CRC check failed"), ("missing", "Missing cache artifact")]) @pytest.mark.parametrize("rows", [2, 4000000]) diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 1e278fd0..8041221d 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -3113,6 +3113,61 @@ def test_arrow_input_empty_and_width_errors(): prepare_table(pa.table({"x": [1]}), ["a", "b"], "TIMESTAMP_LTZ", "UTC", "false") +@pytest.mark.parametrize("schema,expected_type", [ + pytest.param(T.ArrayType(T.LongType()), "ArrayType", id="array"), + pytest.param(T.MapType(T.StringType(), T.LongType()), "MapType", id="map"), +]) +@pytest.mark.parametrize("empty", [False, True], ids=["nonempty", "empty"]) +def test_arrow_input_unsupported_schema_types(schema, expected_type, empty): + pa = pytest.importorskip("pyarrow") + from pyspark.errors import PySparkTypeError + from pyspark.sql._arrow_input import prepare_table + + table = pa.table({"x": pa.array([] if empty else [1], type=pa.int64())}) + with pytest.raises(PySparkTypeError) as error: + prepare_table(table, schema, "TIMESTAMP_LTZ", "UTC", "false") + assert error.value.getCondition() == "NOT_EXPECTED_TYPE" + assert error.value.getMessageParameters() == { + "arg_name": "schema", "expected_type": "list, None or StructType", + "arg_type": expected_type, + } + + +@pytest.mark.parametrize("schema", [ + None, T.LongType(), T.StructType([T.StructField("x", T.LongType())]), ["x"], ("x",), +]) +def test_arrow_input_supported_schema_types(schema): + pa = pytest.importorskip("pyarrow") + from pyspark.sql._arrow_input import prepare_table + + table = pa.table({"x": [1]}) + normalized, result, names = prepare_table(table, schema, "TIMESTAMP_LTZ", "UTC", "false") + assert normalized.equals(table) + if isinstance(schema, (T.AtomicType, T.StructType)): + assert result is schema + else: + assert result.names == ["x"] + assert names == (["x"] if isinstance(schema, (list, tuple)) else None) + + +def test_arrow_input_array_and_map_struct_fields_allowed(): + pa = pytest.importorskip("pyarrow") + from pyspark.sql._arrow_input import prepare_table + + table = pa.table({ + "items": pa.array([[1, None]], type=pa.list_(pa.int64())), + "lookup": pa.array([[("a", 1)]], type=pa.map_(pa.string(), pa.int64())), + }) + schema = T.StructType([ + T.StructField("items", T.ArrayType(T.LongType())), + T.StructField("lookup", T.MapType(T.StringType(), T.LongType())), + ]) + normalized, result, names = prepare_table(table, schema, "TIMESTAMP_LTZ", "UTC", "false") + assert normalized.equals(table) + assert result is schema + assert names is None + + @pytest.mark.parametrize("schema,expected_length", [ pytest.param(T.StructType([T.StructField("x", T.IntegerType())]), "1", id="too-few-fields"), pytest.param(T.StructType([ From 0c335de567d2d8b27ad760f00d9e22f01412b670 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Sat, 31 Oct 2026 15:47:51 -0700 Subject: [PATCH 07/10] [SPARK-60675][CONNECT] Address Arrow input review feedback Match upstream inferred-schema and chunk serialization behavior, parse strict configuration snapshots into named fields, and narrow artifact changes to the cache uploads needed by Arrow input. Remove unrelated frame-size, buffer-sharing, hashing backend, user-context, and batching optimizations. Keep client-only regressions offline and test live timestamp normalization. Co-authored-by: Isaac --- Cargo.lock | 2 +- crates/pyspark-rs/src/pyarrow_import.rs | 8 +- crates/spark-connect-core/Cargo.toml | 3 +- crates/spark-connect-core/src/artifact.rs | 164 ++--------- crates/spark-connect-core/src/client.rs | 53 +--- crates/spark-connect/src/arrow_input.rs | 343 ++++++++++++++-------- crates/spark-connect/src/dataframe.rs | 7 +- crates/spark-connect/src/plan.rs | 13 +- crates/spark-connect/src/session.rs | 34 ++- dev/design/arrow-input-benchmark.md | 83 ------ docs/architecture.md | 4 +- python/pyspark/sql/_arrow_input.py | 6 +- python/tests/test_arrow_e2e.py | 77 ++--- python/tests/test_arrow_handoff.py | 229 +++++++-------- python/tests/test_dropin_offline.py | 11 +- 15 files changed, 447 insertions(+), 590 deletions(-) delete mode 100644 dev/design/arrow-input-benchmark.md diff --git a/Cargo.lock b/Cargo.lock index 7c7cb4b5..6238cb96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -86,8 +86,8 @@ dependencies = [ "prost", "prost-types", "regex", - "ring", "serde_json", + "sha2", "tempfile", "thiserror", "tokio", diff --git a/crates/pyspark-rs/src/pyarrow_import.rs b/crates/pyspark-rs/src/pyarrow_import.rs index 6e2e37b2..2ef7e528 100644 --- a/crates/pyspark-rs/src/pyarrow_import.rs +++ b/crates/pyspark-rs/src/pyarrow_import.rs @@ -56,8 +56,12 @@ pub(crate) fn create_dataframe( options.as_ref().and_then(|o| o.use_large_var_types()), ))?; let normalized = result.get_item(0)?; - // Native local relations carry inferred schemas to preserve timestamp-type normalization. - let spark_schema = crate::types::py_to_data_type(&result.get_item(1)?)?; + let schema = result.get_item(1)?; + let spark_schema = if schema.is_none() { + None + } else { + Some(crate::types::py_to_data_type(&schema)?) + }; let names: Option> = result.get_item(2)?.extract()?; let batches = if empty { Vec::new() diff --git a/crates/spark-connect-core/Cargo.toml b/crates/spark-connect-core/Cargo.toml index a180ae90..6cdf0e01 100644 --- a/crates/spark-connect-core/Cargo.toml +++ b/crates/spark-connect-core/Cargo.toml @@ -26,8 +26,7 @@ serde_json = "1" regex = "1" prost-types.workspace = true crc32fast = "1" -# Reuse the TLS backend's accelerated SHA-256, including on ARM64. -ring = "0.17" +sha2 = "0.10" [dev-dependencies] tempfile = "3" diff --git a/crates/spark-connect-core/src/artifact.rs b/crates/spark-connect-core/src/artifact.rs index 4c40bbf0..20affd29 100644 --- a/crates/spark-connect-core/src/artifact.rs +++ b/crates/spark-connect-core/src/artifact.rs @@ -20,17 +20,8 @@ use crate::error::SparkError; /// https://github.com/grpc/grpc.github.io/issues/371. pub const CHUNK_SIZE: usize = 32 * 1024; -/// Cache uploads amortize message overhead while staying below the default -/// 4 MiB gRPC receive limit. Other artifact uploads retain their existing size. -pub(crate) const CACHE_CHUNK_SIZE: usize = 1024 * 1024; - /// Local artifact data source (file or in-memory). pub trait ArtifactData: Send + Sync { - /// Share an in-memory payload without reading or copying it. - fn bytes(&self) -> Option { - None - } - /// Get the size of this artifact in bytes. fn size(&self) -> Result; @@ -65,22 +56,16 @@ impl ArtifactData for FileArtifact { /// An in-memory artifact. pub struct InMemoryArtifact { - data: Bytes, + data: Vec, } impl InMemoryArtifact { pub fn new(data: impl Into>) -> Self { - Self { - data: Bytes::from(data.into()), - } + Self { data: data.into() } } } impl ArtifactData for InMemoryArtifact { - fn bytes(&self) -> Option { - Some(self.data.clone()) - } - fn size(&self) -> Result { Ok(self.data.len() as u64) } @@ -115,16 +100,6 @@ impl NamedArtifact { pub fn build_artifact_request_stream( session_id: String, artifacts: Vec, -) -> Result> { - build_artifact_request_stream_with_chunk_size(session_id, artifacts, CHUNK_SIZE) -} - -/// Cache payloads can use larger messages to amortize framing overhead while -/// keeping each message well below the server's default 4 MiB receive limit. -pub(crate) fn build_artifact_request_stream_with_chunk_size( - session_id: String, - artifacts: Vec, - chunk_size: usize, ) -> Result> { let mut requests = Vec::new(); let mut batch_artifacts = Vec::new(); @@ -133,7 +108,7 @@ pub(crate) fn build_artifact_request_stream_with_chunk_size( for artifact in artifacts { let size = artifact.data.size()? as usize; - if size > chunk_size { + if size > CHUNK_SIZE { // Flush batch if non-empty if !batch_artifacts.is_empty() { let mut request = AddArtifactsRequest::default(); @@ -152,14 +127,10 @@ pub(crate) fn build_artifact_request_stream_with_chunk_size( requests.extend(build_chunked_artifact_requests( session_id.clone(), artifact, - chunk_size, )?); } else { // Add to batch - // Include names and protobuf framing so many tiny cache blobs cannot - // turn a small payload batch into an oversized gRPC message. - let framed_size = size + artifact.name.len() + 32; - if batch_size + framed_size > chunk_size && !batch_artifacts.is_empty() { + if batch_size + size > CHUNK_SIZE && !batch_artifacts.is_empty() { // Flush current batch let mut request = AddArtifactsRequest::default(); request.session_id = session_id.clone(); @@ -174,24 +145,21 @@ pub(crate) fn build_artifact_request_stream_with_chunk_size( } // Read artifact data - let data = match artifact.data.bytes() { - Some(data) => data, - None => { - let mut stream = artifact.data.stream()?; - let mut data = Vec::new(); - stream.read_to_end(&mut data).map_err(|e| { - SparkError::connect_msg(format!("Failed to read artifact: {}", e)) - })?; - Bytes::from(data) - } - }; + let mut stream = artifact.data.stream()?; + let mut data = Vec::new(); + stream + .read_to_end(&mut data) + .map_err(|e| SparkError::connect_msg(format!("Failed to read artifact: {}", e)))?; let crc = compute_crc32(&data) as i64; batch_artifacts.push(add_artifacts_request::SingleChunkArtifact { name: artifact.name, - data: Some(add_artifacts_request::ArtifactChunk { data, crc }), + data: Some(add_artifacts_request::ArtifactChunk { + data: Bytes::from(data), + crc, + }), }); - batch_size += framed_size; + batch_size += size; } } @@ -214,41 +182,24 @@ pub(crate) fn build_artifact_request_stream_with_chunk_size( fn build_chunked_artifact_requests( session_id: String, artifact: NamedArtifact, - chunk_size: usize, ) -> Result> { let mut requests = Vec::new(); let total_size = artifact.data.size()? as i64; - let num_chunks = (total_size as usize).div_ceil(chunk_size); - - let shared = artifact.data.bytes(); - let mut stream = if shared.is_none() { - Some(artifact.data.stream()?) - } else { - None - }; + let num_chunks = (total_size as usize).div_ceil(CHUNK_SIZE); + + let mut stream = artifact.data.stream()?; let mut first_chunk = true; - let mut buf = if shared.is_none() { - vec![0u8; chunk_size] - } else { - Vec::new() - }; - let mut offset = 0; + let mut buf = vec![0u8; CHUNK_SIZE]; loop { - let chunk_data = if let Some(data) = &shared { - let end = (offset + chunk_size).min(data.len()); - let chunk = data.slice(offset..end); - offset = end; - chunk - } else { - let n = stream.as_mut().unwrap().read(&mut buf).map_err(|e| { - SparkError::connect_msg(format!("Failed to read artifact chunk: {}", e)) - })?; - Bytes::copy_from_slice(&buf[..n]) - }; - if chunk_data.is_empty() { + let n = stream.read(&mut buf).map_err(|e| { + SparkError::connect_msg(format!("Failed to read artifact chunk: {}", e)) + })?; + if n == 0 { break; } + + let chunk_data = buf[..n].to_vec(); let crc = compute_crc32(&chunk_data) as i64; if first_chunk { @@ -261,7 +212,7 @@ fn build_chunked_artifact_requests( total_bytes: total_size, num_chunks: num_chunks as i64, initial_chunk: Some(add_artifacts_request::ArtifactChunk { - data: chunk_data, + data: Bytes::from(chunk_data), crc, }), }, @@ -274,7 +225,7 @@ fn build_chunked_artifact_requests( request.session_id = session_id.clone(); request.payload = Some(add_artifacts_request::Payload::Chunk( add_artifacts_request::ArtifactChunk { - data: chunk_data, + data: Bytes::from(chunk_data), crc, }, )); @@ -393,67 +344,4 @@ mod tests { // Multiple small artifacts should be batched into one request assert_eq!(requests.len(), 1); } - - #[test] - fn shared_payload_matches_file_framing_and_survives_source_drop() { - for chunk_size in [CHUNK_SIZE, CACHE_CHUNK_SIZE] { - for size in [0, chunk_size, chunk_size + 1, chunk_size * 3 + 17] { - let data: Vec = (0..size).map(|i| (i % 251) as u8).collect(); - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("payload"); - std::fs::write(&path, &data).unwrap(); - let memory = InMemoryArtifact::new(data); - let shared = memory.bytes().unwrap(); - let requests = build_artifact_request_stream_with_chunk_size( - "session".into(), - vec![NamedArtifact::new("cache/hash".into(), Box::new(memory))], - chunk_size, - ) - .unwrap(); - let file_requests = build_artifact_request_stream_with_chunk_size( - "session".into(), - vec![NamedArtifact::new( - "cache/hash".into(), - Box::new(FileArtifact::new(path)), - )], - chunk_size, - ) - .unwrap(); - assert_eq!(requests, file_requests); - if size > chunk_size { - let Some(add_artifacts_request::Payload::BeginChunk(first)) = - &requests[0].payload - else { - panic!("expected chunked artifact"); - }; - assert_eq!( - first.initial_chunk.as_ref().unwrap().data.as_ptr(), - shared.as_ptr() - ); - } - } - } - } - - #[test] - fn small_artifact_batches_include_name_and_framing_overhead() { - use prost::Message; - - let artifacts = (0..100) - .map(|i| { - NamedArtifact::new( - format!("cache/{i:064x}"), - Box::new(InMemoryArtifact::new(vec![i as u8; 2])), - ) - }) - .collect(); - let requests = - build_artifact_request_stream_with_chunk_size("session".into(), artifacts, 1024) - .unwrap(); - assert!(requests.len() > 1); - for request in requests { - // Allow the session/enclosing-message fields above the batch budget. - assert!(request.encoded_len() <= 1024 + 32); - } - } } diff --git a/crates/spark-connect-core/src/client.rs b/crates/spark-connect-core/src/client.rs index a0e7a646..9ad97b13 100644 --- a/crates/spark-connect-core/src/client.rs +++ b/crates/spark-connect-core/src/client.rs @@ -445,38 +445,6 @@ impl SparkConnectClient { Ok(keys.iter().map(|k| config_dict.get(*k).cloned()).collect()) } - /// Get registered defaults and session overrides, returning None for unknown keys. - pub async fn get_config_options(&self, keys: &[&str]) -> Result>> { - let request = ConfigRequest { - session_id: self.session_id.clone(), - user_context: Some(UserContext { - user_id: self.user_id.clone().unwrap_or_default(), - ..Default::default() - }), - operation: Some(spark_connect_proto::config_request::Operation { - op_type: Some( - spark_connect_proto::config_request::operation::OpType::GetOption( - spark_connect_proto::config_request::GetOption { - keys: keys.iter().map(|key| key.to_string()).collect(), - }, - ), - ), - }), - ..Default::default() - }; - let values: BTreeMap<_, _> = self - .config(request) - .await? - .pairs - .into_iter() - .map(|pair| (pair.key, pair.value)) - .collect(); - Ok(keys - .iter() - .map(|key| values.get(*key).cloned().flatten()) - .collect()) - } - /// Set configuration values for the session. /// /// Mirrors `pyspark.sql.connect.client.core.SparkConnectClient.set_config`. @@ -792,14 +760,13 @@ impl SparkConnectClient { /// Existing artifacts and duplicate blobs are uploaded only once. pub async fn cache_artifacts(&self, blobs: Vec>) -> Result> { use crate::artifact::InMemoryArtifact; - use ring::digest::{digest, SHA256}; + use sha2::{Digest, Sha256}; use std::collections::HashSet; let hashes: Vec = blobs .iter() .map(|blob| { - digest(&SHA256, blob) - .as_ref() + Sha256::digest(blob) .iter() .map(|byte| format!("{byte:02x}")) .collect::() @@ -827,29 +794,19 @@ impl SparkConnectClient { return Ok(hashes); } let expected: HashSet = artifacts.iter().map(|a| a.name.clone()).collect(); - let requests = crate::artifact::build_artifact_request_stream_with_chunk_size( - self.session_id.clone(), - artifacts, - crate::artifact::CACHE_CHUNK_SIZE, - )?; + let requests = build_artifact_request_stream(self.session_id.clone(), artifacts)?; self.upload_artifact_requests(requests, Some(&expected)) .await?; Ok(hashes) } - /// Send artifact frames with the same user context and validate the response. + /// Send artifact frames and validate the response. /// Cache uploads also require an acknowledgement for every requested artifact. async fn upload_artifact_requests( &self, - mut requests: Vec, + requests: Vec, expected_cache_names: Option<&std::collections::HashSet>, ) -> Result<()> { - for request in &mut requests { - request.user_context = Some(UserContext { - user_id: self.user_id.clone().unwrap_or_default(), - ..Default::default() - }); - } let mut req = Request::new(futures::stream::iter(requests)); self._attach_metadata(&mut req); let response = self diff --git a/crates/spark-connect/src/arrow_input.rs b/crates/spark-connect/src/arrow_input.rs index 5e26373f..1592468a 100644 --- a/crates/spark-connect/src/arrow_input.rs +++ b/crates/spark-connect/src/arrow_input.rs @@ -6,6 +6,7 @@ use arrow::record_batch::RecordBatch; use spark_connect_core::client::SparkConnectClient; use spark_connect_core::error::{Result, SparkError, SparkErrorKind}; use spark_connect_core::runtime::block_on; +use std::collections::HashMap; use crate::plan::LogicalPlan; use crate::types::DataType; @@ -13,7 +14,11 @@ use crate::types::DataType; /// Internal settings snapshot shared with the Python Arrow importer; not a stable API. #[doc(hidden)] pub struct ArrowInputOptions { - values: Vec>, + timestamp_type: String, + timezone: String, + use_large_var_types: bool, + cache_threshold: usize, + upload: Options, } impl ArrowInputOptions { @@ -27,67 +32,74 @@ impl ArrowInputOptions { "spark.sql.session.localRelationChunkSizeBytes", "spark.sql.session.localRelationBatchOfChunksSizeBytes", ]; - // SQLConf defaults for servers that do not expose the upload settings. - const UPLOAD_DEFAULTS: [&'static str; 5] = - ["1048576", "3221225472", "10000", "16777216", "1073741824"]; - pub(crate) fn from_client(client: &SparkConnectClient) -> Result { + let values = block_on(client.get_configs(&Self::KEYS))?; + let values = Self::KEYS + .into_iter() + .zip(values) + .filter_map(|(key, value)| value.map(|value| (key, value))) + .collect(); + Self::from_values(values) + } + + fn from_values(values: HashMap<&str, String>) -> Result { + let required = |key: &str| { + values.get(key).map(String::as_str).ok_or_else(|| { + SparkError::classed( + SparkErrorKind::SparkNoSuchElement, + "SQL_CONF_NOT_FOUND", + &[("sqlConf", key)], + ) + }) + }; + let invalid = |key: &str| SparkError::connect_msg(format!("Invalid configuration: {key}")); + let integer = |key: &str, allow_zero: bool| { + required(key)? + .parse::() + .ok() + .filter(|value| allow_zero || *value > 0) + .ok_or_else(|| invalid(key)) + }; + let chunk_bytes = integer("spark.sql.session.localRelationChunkSizeBytes", false)?; + let upload_batch_bytes = integer( + "spark.sql.session.localRelationBatchOfChunksSizeBytes", + false, + )?; Ok(Self { - values: block_on(client.get_config_options(&Self::KEYS))?, + timestamp_type: required("spark.sql.timestampType")?.to_owned(), + timezone: required("spark.sql.session.timeZone")?.to_owned(), + use_large_var_types: required("spark.sql.execution.arrow.useLargeVarTypes")? == "true", + cache_threshold: integer("spark.sql.session.localRelationCacheThreshold", true)?, + upload: Options { + size_limit: integer("spark.sql.session.localRelationSizeLimit", false)?, + chunk_rows: integer("spark.sql.session.localRelationChunkSizeRows", false)?, + // Match upstream's min(chunk size, upload batch size). + chunk_bytes: chunk_bytes.min(upload_batch_bytes), + upload_batch_bytes, + }, }) } /// Server timestamp type, including its registered default. pub fn timestamp_type(&self) -> Option<&str> { - self.values[0].as_deref() + Some(&self.timestamp_type) } /// Server session timezone; do not substitute a client-local timezone. pub fn timezone(&self) -> Option<&str> { - self.values[1].as_deref() + Some(&self.timezone) } /// Whether the server prefers large Arrow string and binary offsets. pub fn use_large_var_types(&self) -> Option<&str> { - self.values[2].as_deref() - } - - fn upload_value(&self, index: usize) -> Result { - let value = self.values[index + 3] - .as_deref() - .unwrap_or(Self::UPLOAD_DEFAULTS[index]); - // Spark 4.2 uses decimal bytes; older bytesConf values may include units. - let parsed = if index == 1 { - parse_byte_size(value) + Some(if self.use_large_var_types { + "true" } else { - value.parse::().ok() - }; - parsed - .filter(|value| index == 0 || *value > 0) - .ok_or_else(|| { - SparkError::connect_msg(format!("Invalid configuration: {}", Self::KEYS[index + 3])) - }) + "false" + }) } } -fn parse_byte_size(value: &str) -> Option { - let value = value.trim().to_ascii_lowercase(); - let number = value.trim_end_matches(|c: char| c.is_ascii_alphabetic()); - let exponent = match &value[number.len()..] { - "" | "b" => 0, - "k" | "kb" => 1, - "m" | "mb" => 2, - "g" | "gb" => 3, - "t" | "tb" => 4, - "p" | "pb" => 5, - _ => return None, - }; - number - .parse::() - .ok()? - .checked_mul(1024usize.checked_pow(exponent)?) -} - struct Options { size_limit: usize, chunk_rows: usize, @@ -95,21 +107,6 @@ struct Options { upload_batch_bytes: usize, } -impl Options { - fn from_input(options: &ArrowInputOptions) -> Result { - let parsed = (0..5) - .map(|index| options.upload_value(index)) - .collect::>>()?; - Ok(Self { - size_limit: parsed[1], - chunk_rows: parsed[2], - // Match upstream's min(chunk size, upload batch size). - chunk_bytes: parsed[3].min(parsed[4]), - upload_batch_bytes: parsed[4], - }) - } -} - fn arrow_error(error: arrow::error::ArrowError) -> SparkError { SparkError::connect_msg(error.to_string()) } @@ -123,7 +120,23 @@ fn validate(batches: &[RecordBatch]) -> Result<()> { Ok(()) } -pub(crate) fn inline_relation(batches: &[RecordBatch], schema: DataType) -> Result { +fn batch_data_size(batch: &RecordBatch) -> usize { + batch + .columns() + .iter() + .map(|array| { + array + .to_data() + .get_slice_memory_size() + .unwrap_or_else(|_| array.get_buffer_memory_size()) + }) + .sum() +} + +pub(crate) fn inline_relation( + batches: &[RecordBatch], + schema: Option, +) -> Result { validate(batches)?; let data = if let Some(first) = batches.first() { let mut writer = StreamWriter::try_new(Vec::new(), &first.schema()).map_err(arrow_error)?; @@ -140,7 +153,7 @@ pub(crate) fn inline_relation(batches: &[RecordBatch], schema: DataType) -> Resu pub(crate) fn create_relation( client: &SparkConnectClient, batches: Vec, - schema: DataType, + schema: Option, options: Option<&ArrowInputOptions>, ) -> Result { // Validate before any RPC, and preserve the offline empty-input path. @@ -157,20 +170,11 @@ pub(crate) fn create_relation( } }; // Count the logical slices rather than their potentially much larger backing buffers. - let bytes: usize = batches - .iter() - .flat_map(|b| b.columns()) - .map(|a| { - a.to_data() - .get_slice_memory_size() - .unwrap_or_else(|_| a.get_buffer_memory_size()) - }) - .sum(); - if bytes < options.upload_value(0)? { + let bytes: usize = batches.iter().map(batch_data_size).sum(); + if bytes < options.cache_threshold { return inline_relation(&batches, schema); } - // Chunk and batch settings are irrelevant to inline inputs. - let options = Options::from_input(options)?; + let options = &options.upload; let mut pending = Vec::new(); let mut pending_size = 0; @@ -198,14 +202,15 @@ pub(crate) fn create_relation( pending.push(chunk); Ok(()) }; - // Carry inferred schemas too, preserving the client's timestamp-type normalization. - enqueue(schema.json().into_bytes())?; - serialize_chunks(&batches, &options, &mut enqueue)?; + if let Some(schema) = &schema { + enqueue(schema.json().into_bytes())?; + } + serialize_chunks(&batches, options, &mut enqueue)?; hashes.extend(block_on(client.cache_artifacts(pending))?); - let schema_hash = hashes.remove(0); + let schema_hash = schema.map(|_| hashes.remove(0)); Ok(LogicalPlan::ChunkedCachedLocalRelation { data_hashes: hashes, - schema_hash: Some(schema_hash), + schema_hash, }) } @@ -213,35 +218,41 @@ fn new_writer(schema: &SchemaRef) -> Result>> { StreamWriter::try_new(Vec::new(), schema).map_err(arrow_error) } -/// Each chunk is an independently readable IPC stream. Check limits at most every -/// 1024 rows, as reference PySpark does; a single oversized row remains indivisible. +/// Match PySpark's pre-write estimate: stream schema plus each batch's data size. +/// The row setting caps individual record batches, not the complete IPC stream. fn serialize_chunks( batches: &[RecordBatch], options: &Options, emit: &mut impl FnMut(Vec) -> Result<()>, ) -> Result<()> { let schema = batches[0].schema(); + let schema_size = new_writer(&schema)? + .into_inner() + .map_err(arrow_error)? + .len(); let mut writer = new_writer(&schema)?; - let mut rows = 0; + let mut current_size = schema_size; + let mut has_data = false; for batch in batches { let mut offset = 0; while offset < batch.num_rows() { let length = (batch.num_rows() - offset) .min(1024) - .min(options.chunk_rows - rows); - writer - .write(&batch.slice(offset, length)) - .map_err(arrow_error)?; - offset += length; - rows += length; - if rows >= options.chunk_rows || writer.get_ref().len() >= options.chunk_bytes { + .min(options.chunk_rows); + let slice = batch.slice(offset, length); + let batch_size = batch_data_size(&slice); + if has_data && current_size + batch_size > options.chunk_bytes { emit(writer.into_inner().map_err(arrow_error)?)?; writer = new_writer(&schema)?; - rows = 0; + current_size = schema_size; } + writer.write(&slice).map_err(arrow_error)?; + current_size += batch_size; + has_data = true; + offset += length; } } - if rows > 0 { + if has_data { emit(writer.into_inner().map_err(arrow_error)?)?; } Ok(()) @@ -254,52 +265,124 @@ mod tests { use arrow::datatypes::{DataType as ArrowType, Field, Schema}; use arrow::ipc::reader::StreamReader; use std::io::Cursor; + use std::ops::Range; use std::sync::Arc; - #[test] - fn chunks_preserve_rows_schema_and_order() { + fn batch(values: Range) -> RecordBatch { let schema = Arc::new(Schema::new(vec![Field::new("id", ArrowType::Int64, true)])); - let batch = RecordBatch::try_new( - schema.clone(), - vec![Arc::new(Int64Array::from_iter_values(0..2500))], - ) + RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from_iter_values(values))]).unwrap() + } + + fn chunks(batch: &RecordBatch, chunk_rows: usize, chunk_bytes: usize) -> Vec> { + let options = Options { + size_limit: usize::MAX, + chunk_rows, + chunk_bytes, + upload_batch_bytes: usize::MAX, + }; + let mut chunks = Vec::new(); + serialize_chunks(std::slice::from_ref(batch), &options, &mut |chunk| { + chunks.push(chunk); + Ok(()) + }) .unwrap(); - for (chunk_rows, chunk_bytes) in [(700, usize::MAX), (10000, 1024)] { - let options = Options { - size_limit: usize::MAX, - chunk_rows, - chunk_bytes, - upload_batch_bytes: usize::MAX, - }; - let mut chunks = Vec::new(); - serialize_chunks(std::slice::from_ref(&batch), &options, &mut |c| { - chunks.push(c); - Ok(()) - }) - .unwrap(); - assert!(chunks.len() > 1); - let mut actual = Vec::new(); - for chunk in chunks { + chunks + .into_iter() + .map(|chunk| { let reader = StreamReader::try_new(Cursor::new(chunk), None).unwrap(); - assert_eq!(reader.schema(), schema); - let mut rows = 0; - for batch in reader { - let batch = batch.unwrap(); - rows += batch.num_rows(); - actual.extend( - batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap() - .values() - .iter() - .copied(), - ); - } - assert!(rows <= chunk_rows); - } - assert_eq!(actual, (0..2500).collect::>()); + assert_eq!(reader.schema(), batch.schema()); + reader.collect::, _>>().unwrap() + }) + .collect() + } + + fn chunk_row_counts(chunks: &[Vec]) -> Vec { + chunks + .iter() + .map(|chunk| chunk.iter().map(RecordBatch::num_rows).sum()) + .collect() + } + + #[test] + fn row_setting_limits_batches_without_splitting_the_stream() { + let chunks = chunks(&batch(0..2500), 700, usize::MAX); + assert_eq!(chunks.len(), 1); + assert_eq!( + chunks[0] + .iter() + .map(RecordBatch::num_rows) + .collect::>(), + vec![700, 700, 700, 400] + ); + } + + #[test] + fn chunks_flush_before_a_batch_exceeds_the_data_size_estimate() { + let batch = batch(0..25); + let schema_size = new_writer(&batch.schema()) + .unwrap() + .into_inner() + .unwrap() + .len(); + // Two four-row Int64 batches fit exactly, despite IPC message overhead. + let chunks = chunks(&batch, 4, schema_size + 8 * 8); + assert_eq!(chunk_row_counts(&chunks), vec![8, 8, 8, 1]); + let actual: Vec<_> = chunks + .iter() + .flatten() + .flat_map(|batch| { + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .iter() + .copied() + }) + .collect(); + assert_eq!(actual, (0..25).collect::>()); + } + + #[test] + fn an_oversized_batch_does_not_emit_an_empty_stream() { + let batch = batch(0..10); + let schema_size = new_writer(&batch.schema()) + .unwrap() + .into_inner() + .unwrap() + .len(); + let chunks = chunks(&batch, 4, schema_size + 4 * 8 - 1); + assert_eq!(chunk_row_counts(&chunks), vec![4, 4, 2]); + } + + #[test] + fn inline_schema_is_sent_only_when_explicit() { + let batch = batch(0..2); + let explicit = DataType::Struct { + fields: vec![crate::types::StructField { + name: "id".to_owned(), + data_type: DataType::Long, + nullable: true, + metadata: Default::default(), + }], + }; + for schema in [None, Some(explicit)] { + let relation = inline_relation(std::slice::from_ref(&batch), schema.clone()) + .unwrap() + .to_proto(); + let Some(spark_connect_proto::relation::RelType::LocalRelation(local)) = + relation.rel_type + else { + panic!("expected inline relation"); + }; + assert_eq!(local.schema, schema.as_ref().map(DataType::json)); + let data = local.data.unwrap(); + let decoded = StreamReader::try_new(Cursor::new(data), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(decoded, vec![batch.clone()]); } } } diff --git a/crates/spark-connect/src/dataframe.rs b/crates/spark-connect/src/dataframe.rs index 7baa8671..df69579e 100644 --- a/crates/spark-connect/src/dataframe.rs +++ b/crates/spark-connect/src/dataframe.rs @@ -1694,8 +1694,13 @@ impl DataFrame { /// Check if this DataFrame is local (collected). pub fn is_local(&self) -> bool { + let mut plan = self.plan.as_ref(); + // Arrow input uses toDF to apply inferred or supplied column names. + while let LogicalPlan::ToDF { input, .. } = plan { + plan = input.as_ref(); + } matches!( - self.plan.as_ref(), + plan, LogicalPlan::LocalRelation { .. } | LogicalPlan::ChunkedCachedLocalRelation { .. } ) } diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs index 86dacf27..e7c84254 100644 --- a/crates/spark-connect/src/plan.rs +++ b/crates/spark-connect/src/plan.rs @@ -225,7 +225,7 @@ pub enum LogicalPlan { }, /// LocalRelation: data supplied locally. LocalRelation { - schema: DataType, + schema: Option, data: Option>, }, /// Local Arrow data uploaded as session-scoped cache artifacts. @@ -965,9 +965,9 @@ impl LogicalPlan { if let Some(d) = data { local.data = Some(d.clone().into()); } - // Carry the schema (JSON) so the server applies the user's column - // names/types; required when no Arrow `data` is provided (emptyDataFrame). - local.schema = Some(schema.json()); + // Explicit schemas apply server-side casts. Inferred Arrow input + // omits this to avoid a redundant projection on every query. + local.schema = schema.as_ref().map(DataType::json); relation.rel_type = Some(proto::relation::RelType::LocalRelation(local)); } @@ -1757,7 +1757,10 @@ pub fn subquery_alias(input: impl Into>, alias: impl Into>) -> LogicalPlan { - LogicalPlan::LocalRelation { schema, data } + LogicalPlan::LocalRelation { + schema: Some(schema), + data, + } } /// Create a CachedRemoteRelation plan. diff --git a/crates/spark-connect/src/session.rs b/crates/spark-connect/src/session.rs index 9f5a8cc9..d08d0ea4 100644 --- a/crates/spark-connect/src/session.rs +++ b/crates/spark-connect/src/session.rs @@ -286,22 +286,26 @@ impl SparkSession { } else { Some(rows_to_arrow_ipc(&rows, &schema)?) }; - let plan = LogicalPlan::LocalRelation { schema, data }; + let plan = LogicalPlan::LocalRelation { + schema: Some(schema), + data, + }; Ok(DataFrame::new(self.clone(), plan)) } /// Create a local DataFrame from Arrow batches without materializing rows. /// - /// `schema` is the Spark schema applied by the server. All batches must share - /// the same Arrow schema; IPC serialization is still required for transport. + /// `schema` optionally supplies the Spark schema applied by the server. With + /// `None`, Spark infers it from Arrow. All batches must share the same Arrow + /// schema; IPC serialization is still required for transport. /// Non-empty inputs read the server's local-relation configuration. Large inputs /// are uploaded as cache artifacts and referenced by hash in the resulting plan. pub fn create_dataframe_from_arrow( &self, batches: Vec, - schema: DataType, + schema: impl Into>, ) -> Result { - let plan = crate::arrow_input::create_relation(&self.client, batches, schema, None)?; + let plan = crate::arrow_input::create_relation(&self.client, batches, schema.into(), None)?; Ok(DataFrame::new(self.clone(), plan)) } @@ -316,11 +320,15 @@ impl SparkSession { pub fn create_dataframe_from_arrow_with_options( &self, batches: Vec, - schema: DataType, + schema: impl Into>, options: &crate::ArrowInputOptions, ) -> Result { - let plan = - crate::arrow_input::create_relation(&self.client, batches, schema, Some(options))?; + let plan = crate::arrow_input::create_relation( + &self.client, + batches, + schema.into(), + Some(options), + )?; Ok(DataFrame::new(self.clone(), plan)) } @@ -437,7 +445,7 @@ impl SparkSession { /// `SparkSession.emptyDataFrame`. pub fn empty_data_frame(&self) -> Result { let plan = LogicalPlan::LocalRelation { - schema: DataType::Struct { fields: vec![] }, + schema: Some(DataType::Struct { fields: vec![] }), data: None, }; Ok(DataFrame::new(self.clone(), plan)) @@ -1272,9 +1280,11 @@ mod tests { fields: vec![struct_field("x", DataType::Integer)], }; let spark = session(); - let plan = - crate::arrow_input::inline_relation(&[batch.clone(), batch.clone()], schema.clone()) - .unwrap(); + let plan = crate::arrow_input::inline_relation( + &[batch.clone(), batch.clone()], + Some(schema.clone()), + ) + .unwrap(); let LogicalPlan::LocalRelation { data: Some(data), .. } = &plan diff --git a/dev/design/arrow-input-benchmark.md b/dev/design/arrow-input-benchmark.md deleted file mode 100644 index f63fa9bf..00000000 --- a/dev/design/arrow-input-benchmark.md +++ /dev/null @@ -1,83 +0,0 @@ -# Arrow input upload benchmarking - -## Current decision - -Keep sequential cache uploads with 1 MiB artifact frames, accelerated SHA-256, -and shared payload buffers. Generic artifact uploads retain 32 KiB frames. -Keep hash order, CRC validation, acknowledgements, and server-side deduplication. - -Two concurrent upload streams did not improve the localhost experiment. -Overlapping serialization and upload reduced latency slightly, but increased -peak client RSS growth from approximately 93 MiB to 145 MiB for a 136 MB input. -Neither concurrency prototype is retained. - -The selected framing change reduced the 136 MB creation-plus-query median from -230.58 ms to 138.31 ms; official PySpark measured 307.30 ms in the same experiment. -These are **localhost fixture measurements without Spark query execution**, -using ten fresh-process samples per client. They do not establish remote Spark -performance. A separate live localhost Spark check verified the uploaded values. -These measurements preceded the follow-up that batches normalization and upload -settings into one configuration RPC; they are not a rerun of that follow-up. - -## Required before merging: real remote Spark measurements - -Remote performance validation is **pending**. Benchmark against an actual Spark -Connect server on another machine over the deployment's normal network and TLS -path. A local JVM, a scripted fixture, or injected latency alone does not satisfy -this requirement. Obtain the remote endpoint and configured authentication from -the operator; keep credentials out of commands, reports, and checked-in files. - -Compare all three clients against that same server: - -- Official PySpark, matching the server version. -- The previous optimized sequential client with 32 KiB frames, including the - accelerated hash and shared-buffer changes. Comparing only against the earlier - software-hash implementation would overstate the framing improvement. -- The proposed sequential client with 1 MiB cache frames. - -Use the same Python interpreter and dependency versions. Record exact native -binary hashes and source revisions or patches. Run samples serially, rotating -client order, with at least ten fresh-process samples per client and case. - -### Workloads and measurement boundaries - -Use eight int64 columns at these sizes: - -| Rows | Unencoded input size | -| ---: | ---: | -| 12,500 | 0.8 MB | -| 250,000 | 16 MB | -| 2,125,000 | 136 MB | - -Create the input and establish a fresh session outside the measurement. Warm -server execution with an unrelated small query, without uploading the measured -input. Do not reuse a fixed session ID across cold-input samples: a cached hash -would conceal the upload cost. - -Measure creation plus a materializing query as the primary comparison, including -all uploads. Aggregate all eight columns, checking row count and column sums -against the generated data, so server execution consumes the complete relation. -Also measure repeated queries on a prepared DataFrame after one untimed query; -report these separately from the initial upload. Never warm up cold-input timing -by creating that same input in the same session. - -Keep real server configuration unchanged for the primary comparison. Record -the cache threshold, local relation size limit, chunk row/byte settings, and -upload-batch cap. The fixture used a 64 MiB upload-batch cap; the real server's -default is 1 GiB, so fixture memory and RPC counts cannot be extrapolated directly. - -### Report and decision - -Record client/server versions and hardware, client/server locations, TLS mode, -small-query round-trip latency, driver heap, and concurrent server activity. -Report raw samples, medians and ranges, client CPU, sampled peak RSS above the -prepared-input baseline, failures, and upload/network counters where available. -Keep server execution time visible rather than attributing the entire query -latency to the client. Record sampling intervals and any unavailable counters. - -The 0.8 MB input is a control: the framing change does not optimize its inline -path, which was slower than PySpark in these localhost results. Preserve this -result alongside larger-input gains. - -Reassess frame size if the remote workload regresses. Do not enable parallel -uploads or claim an end-to-end remote speedup solely from the localhost results. diff --git a/docs/architecture.md b/docs/architecture.md index 93107c3e..3286afe8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -120,7 +120,9 @@ Empty tables require an explicit schema, matching PySpark Connect. Inputs at or above `spark.sql.session.localRelationCacheThreshold` are serialized into independent IPC streams and uploaded as session-scoped cache artifacts. The plan carries `ChunkedCachedLocalRelation` hashes instead of embedding the -entire payload. Chunk sizes, upload batch sizes, and the total relation size +entire payload. Spark infers omitted schemas from the Arrow streams; explicit +schemas travel in the local relation or a separate schema artifact. +Chunk sizes, upload batch sizes, and the total relation size limit follow the server's local-relation settings; artifact RPC messages use 32 KiB pieces, with SHA-256 deduplication and CRC checks. diff --git a/python/pyspark/sql/_arrow_input.py b/python/pyspark/sql/_arrow_input.py index 3da01f70..0c5b337f 100644 --- a/python/pyspark/sql/_arrow_input.py +++ b/python/pyspark/sql/_arrow_input.py @@ -86,4 +86,8 @@ def prepare_table(table, schema, timestamp_type, timezone, large_types): "actual_length": str(normalized.num_columns), }, ) - return normalized, schema if explicit else inferred, names + # Match Connect: inferred Arrow schemas travel in IPC, without an extra + # Spark schema that would trigger server-side casts on every query. + if not explicit and names is None: + names = table.column_names + return normalized, schema if explicit else None, names diff --git a/python/tests/test_arrow_e2e.py b/python/tests/test_arrow_e2e.py index 1b7b998d..ced28b11 100644 --- a/python/tests/test_arrow_e2e.py +++ b/python/tests/test_arrow_e2e.py @@ -68,7 +68,7 @@ def test_to_arrow_empty_result(session): def test_create_dataframe_arrow_cached_upload(session): - """Exercise real-server config values and cached input above the default 1 MiB.""" + """Materialize cached Arrow input larger than 1 MiB on a real server.""" import numpy as np rows = 262144 @@ -76,15 +76,7 @@ def test_create_dataframe_arrow_cached_upload(session): assert table.nbytes == 2 * 1024**2 isolated = session.newSession() try: - # Clear inherited session overrides so CI exercises registered server defaults. - isolated.conf.unset("spark.sql.session.localRelationCacheThreshold") - isolated.conf.unset("spark.sql.session.localRelationSizeLimit") - threshold = int(isolated.conf.get("spark.sql.session.localRelationCacheThreshold")) - assert threshold == 1024**2 - assert table.nbytes >= threshold - # Spark 4.2 declares this setting as longConf and returns decimal bytes. - assert isolated.conf.get("spark.sql.session.localRelationSizeLimit") == "3221225472" - + isolated.conf.set("spark.sql.session.localRelationCacheThreshold", str(1024**2)) frame = isolated.createDataFrame(table) actual = frame.selectExpr("count(*) AS rows", "sum(id) AS checksum").collect()[0] assert actual[0] == rows @@ -94,39 +86,32 @@ def test_create_dataframe_arrow_cached_upload(session): isolated.stop() -@pytest.mark.parametrize("schema,expected_width", [ - ("x long", "1"), - ("x long, y long, z long", "3"), - ("long", "1"), -]) -def test_create_dataframe_arrow_explicit_schema_width(session, schema, expected_width): - from pyspark.errors import PySparkValueError - - with pytest.raises(PySparkValueError) as error: - session.createDataFrame(pa.table({"x": [1], "y": [2]}), schema) - assert error.value.getCondition() == "AXIS_LENGTH_MISMATCH" - assert error.value.getMessageParameters() == { - "expected_length": expected_width, "actual_length": "2", - } - - -@pytest.mark.parametrize("schema_kind", ["array", "map"]) -@pytest.mark.parametrize("ddl", [False, True], ids=["datatype", "ddl"]) -@pytest.mark.parametrize("empty", [False, True], ids=["nonempty", "empty"]) -def test_create_dataframe_arrow_unsupported_schema(session, schema_kind, ddl, empty): - from pyspark.errors import PySparkTypeError - from pyspark.sql import types as T - - schema, expected_type = { - "array": (("array" if ddl else T.ArrayType(T.LongType())), "ArrayType"), - "map": (("map" if ddl else T.MapType(T.StringType(), T.LongType())), - "MapType"), - }[schema_kind] - table = pa.table({"x": pa.array([] if empty else [1], type=pa.int64())}) - with pytest.raises(PySparkTypeError) as error: - session.createDataFrame(table, schema) - assert error.value.getCondition() == "NOT_EXPECTED_TYPE" - assert error.value.getMessageParameters() == { - "arg_name": "schema", "expected_type": "list, None or StructType", - "arg_type": expected_type, - } +@pytest.mark.parametrize("cached", [False, True], ids=["inline", "cached"]) +@pytest.mark.parametrize("timestamp_type", ["TIMESTAMP_LTZ", "TIMESTAMP_NTZ"]) +def test_create_dataframe_arrow_inferred_timestamp(session, cached, timestamp_type): + value = datetime(2020, 1, 2, 3, 4, 5) + table = pa.table({"t": pa.array([value], type=pa.timestamp("ns"))}) + isolated = session.newSession() + try: + isolated.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") + isolated.conf.set("spark.sql.timestampType", timestamp_type) + isolated.conf.set("spark.sql.session.localRelationCacheThreshold", + "0" if cached else str(1024**2)) + frame = isolated.createDataFrame(table) + spark_type = "timestamp" if timestamp_type == "TIMESTAMP_LTZ" else "timestamp_ntz" + assert frame.schema.simpleString() == f"struct" + assert frame.selectExpr("CAST(t AS STRING)").collect()[0][0] == "2020-01-02 03:04:05" + + result = frame.toArrow() + if timestamp_type == "TIMESTAMP_LTZ": + assert result.column("t").type.unit == "us" + assert result.column("t").type.tz is not None + # The native Arrow result carries the session timezone; compare instants. + assert result.column("t").cast(pa.timestamp("us", tz="UTC")).to_pylist() == [ + datetime(2020, 1, 2, 11, 4, 5, tzinfo=timezone.utc) + ] + else: + assert result.column("t").type == pa.timestamp("us") + assert result.column("t").to_pylist() == [value] + finally: + isolated.stop() diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index c83ef9fc..4e24ab3b 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -420,7 +420,6 @@ def _arrow_input_service(protocol, *, upload_error=None, **overrides): def configure(raw, context): request = protocol("ConfigRequest").FromString(raw) - state["user_context"] = request.user_context operation = request.operation.WhichOneof("op_type") keys = getattr(request.operation, operation).keys state["config_requests"].append((operation, list(keys))) @@ -453,8 +452,6 @@ def checked(chunk): for raw in requests: state["message_sizes"].append(len(raw)) request = protocol("AddArtifactsRequest").FromString(raw) - assert request.HasField("user_context") - assert request.user_context == state["user_context"] session_id = request.session_id if request.HasField("batch"): for artifact in request.batch.artifacts: @@ -480,16 +477,32 @@ def checked(chunk): def execute(raw, context): request = protocol("ExecutePlanRequest").FromString(raw) relation = request.plan.root + while relation.HasField("to_df"): + state["column_names"] = list(relation.to_df.column_names) + relation = relation.to_df.input state["relations"].append(relation) if relation.HasField("chunked_cached_local_relation"): cached = relation.chunked_cached_local_relation payloads = [state["cache"]["cache/" + h] for h in cached.dataHashes] - state["spark_schema"] = state["cache"]["cache/" + cached.schemaHash].decode() + state["spark_schema"] = ( + state["cache"]["cache/" + cached.schemaHash].decode() + if cached.HasField("schemaHash") else None + ) else: payloads = [relation.local_relation.data] + state["spark_schema"] = ( + relation.local_relation.schema + if relation.local_relation.HasField("schema") else None + ) rows, first, last = 0, None, None + state["arrow_schemas"], state["batch_rows"] = [], [] for payload in payloads: - for batch in pa.ipc.open_stream(payload): + reader = pa.ipc.open_stream(payload) + state["arrow_schemas"].append(reader.schema) + stream_rows = [] + state["batch_rows"].append(stream_rows) + for batch in reader: + stream_rows.append(batch.num_rows) rows += batch.num_rows if batch.num_rows: if first is None: @@ -540,35 +553,96 @@ def stop(raw, context): server.stop(0).wait(timeout=5) -@pytest.mark.parametrize("rows", [2, 17000000]) -@pytest.mark.parametrize("batch_bytes", [64 * 1024**2, 1024**3]) -def test_arrow_input_upload_above_grpc_limit(protocol, rows, batch_bytes): - import numpy as np +@pytest.mark.parametrize("cached", [False, True], ids=["inline", "cached"]) +@pytest.mark.parametrize("explicit", [False, True], ids=["inferred", "explicit"]) +def test_arrow_input_schema_handoff(protocol, cached, explicit): import json + from pyspark.sql import types as T - table = pa.table({"id": np.arange(rows, dtype=np.int64)}) + table = pa.table({"id": [1, 2]}) + schema = T.StructType([T.StructField("id", T.LongType())]) if explicit else None with _arrow_input_service(protocol, **{ - "spark.sql.session.localRelationBatchOfChunksSizeBytes": str(batch_bytes) + "spark.sql.session.localRelationCacheThreshold": "0" if cached else "1048576", }) as (spark, state): - df = spark.createDataFrame(table) + frame = spark.createDataFrame(table, schema) + assert frame.toArrow().column(0).to_pylist() == [2] + assert state["result"] == (2, 1, 2) + assert state["arrow_schemas"] == [table.schema] assert len(state["config_requests"]) == 1 - assert state["config_requests"][0][0] == "get_option" + assert state["config_requests"][0][0] == "get" assert len(state["config_requests"][0][1]) == 8 + relation = state["relations"][0] + if cached: + local = relation.chunked_cached_local_relation + assert local.HasField("schemaHash") == explicit + assert len(state["cache"]) == (2 if explicit else 1) + uploads = state["uploads"] + spark.createDataFrame(table, schema).toArrow() + assert state["uploads"] == uploads + else: + assert state["uploads"] == 0 + assert relation.local_relation.HasField("schema") == explicit + if explicit: + assert json.loads(state["spark_schema"]) == json.loads(schema.json()) + else: + assert state["spark_schema"] is None + assert state["column_names"] == ["id"] + + +def test_arrow_input_upload_above_grpc_limit(protocol): + import numpy as np + + rows = 128 * 1024**2 // 8 + 1 + table = pa.table({"id": np.arange(rows, dtype=np.int64)}) + assert table.nbytes > 128 * 1024**2 + with _arrow_input_service(protocol) as (spark, state): + df = spark.createDataFrame(table) assert df.isLocal() assert df.toArrow().column(0).to_pylist() == [rows] assert state["result"] == (rows, 0, rows - 1) - if rows == 2: - assert state["uploads"] == 0 - assert state["relations"][0].HasField("local_relation") - else: - assert table.nbytes > 128 * 1024**2 - assert state["relations"][0].HasField("chunked_cached_local_relation") - assert state["relations"][0].ByteSize() < 1024**2 - assert max(state["message_sizes"]) < 1025 * 1024 - assert json.loads(state["spark_schema"])["fields"][0]["name"] == "id" - uploads = state["uploads"] - spark.createDataFrame(table).toArrow() - assert state["uploads"] == uploads + assert state["relations"][0].HasField("chunked_cached_local_relation") + assert state["relations"][0].ByteSize() < 1024**2 + assert max(state["message_sizes"]) < 33 * 1024 + assert state["spark_schema"] is None + assert all(schema == table.schema for schema in state["arrow_schemas"]) + + +@pytest.mark.parametrize("schema,expected_width", [ + ("x long", "1"), + ("x long, y long, z long", "3"), + ("long", "1"), +]) +def test_arrow_input_ddl_width_errors(protocol, schema, expected_width): + from pyspark.errors import PySparkValueError + + with _arrow_input_service(protocol) as (spark, state): + with pytest.raises(PySparkValueError) as error: + spark.createDataFrame(pa.table({"x": [1], "y": [2]}), schema) + assert error.value.getCondition() == "AXIS_LENGTH_MISMATCH" + assert error.value.getMessageParameters() == { + "expected_length": expected_width, "actual_length": "2", + } + assert not state["relations"] + assert state["uploads"] == 0 + + +@pytest.mark.parametrize("schema,expected_type", [ + ("array", "ArrayType"), + ("map", "MapType"), +]) +def test_arrow_input_unsupported_ddl_schema(protocol, schema, expected_type): + from pyspark.errors import PySparkTypeError + + with _arrow_input_service(protocol) as (spark, state): + with pytest.raises(PySparkTypeError) as error: + spark.createDataFrame(pa.table({"x": [1]}), schema) + assert error.value.getCondition() == "NOT_EXPECTED_TYPE" + assert error.value.getMessageParameters() == { + "arg_name": "schema", "expected_type": "list, None or StructType", + "arg_type": expected_type, + } + assert not state["relations"] + assert state["uploads"] == 0 def test_arrow_input_upload_limits_and_chunk_order(protocol): @@ -578,7 +652,7 @@ def test_arrow_input_upload_limits_and_chunk_order(protocol): options = { "spark.sql.session.localRelationCacheThreshold": "0", "spark.sql.session.localRelationChunkSizeRows": "7", - "spark.sql.session.localRelationChunkSizeBytes": "1000", + "spark.sql.session.localRelationChunkSizeBytes": "500", "spark.sql.session.localRelationBatchOfChunksSizeBytes": "1000", } with _arrow_input_service(protocol, **options) as (spark, state): @@ -588,8 +662,15 @@ def test_arrow_input_upload_limits_and_chunk_order(protocol): assert state["uploads"] > 1 cached = state["relations"][0].chunked_cached_local_relation assert len(cached.dataHashes) > 1 - for hash in cached.dataHashes: - assert pa.ipc.open_stream(state["cache"]["cache/" + hash]).read_all().num_rows <= 7 + assert all(rows <= 7 for stream in state["batch_rows"] for rows in stream) + assert any(sum(stream) > 7 for stream in state["batch_rows"]) + actual = [ + value + for hash in cached.dataHashes + for batch in pa.ipc.open_stream(state["cache"]["cache/" + hash]) + for value in batch.column(0).to_pylist() + ] + assert actual == list(range(100)) options["spark.sql.session.localRelationSizeLimit"] = "10" with _arrow_input_service(protocol, **options) as (spark, state): @@ -599,43 +680,11 @@ def test_arrow_input_upload_limits_and_chunk_order(protocol): assert state["uploads"] == 0 -@pytest.mark.parametrize("size_limit", ["3221225472", "3221225472b", "3GB", " 3gb "]) -def test_arrow_input_upload_accepts_byte_size_limits(protocol, size_limit): - import numpy as np - - rows = 262144 - table = pa.table({"id": np.arange(rows, dtype=np.int64)}) - with _arrow_input_service(protocol, **{ - "spark.sql.session.localRelationSizeLimit": size_limit, - }) as (spark, state): - spark.createDataFrame(table).toArrow() - assert state["uploads"] > 0 - assert state["relations"][0].HasField("chunked_cached_local_relation") - assert state["result"] == (rows, 0, rows - 1) - - -@pytest.mark.parametrize("size_limit", ["1024", "1024b", "1KB"]) -def test_arrow_input_byte_size_limits_are_enforced(protocol, size_limit): - from pyspark.errors import AnalysisException - - with _arrow_input_service(protocol, **{ - "spark.sql.session.localRelationCacheThreshold": "0", - "spark.sql.session.localRelationSizeLimit": size_limit, - }) as (spark, state): - with pytest.raises(AnalysisException, match="LOCAL_RELATION_SIZE_LIMIT_EXCEEDED") as error: - spark.createDataFrame(pa.table({"id": list(range(1000))})) - assert "limit (1024 bytes)" in str(error.value) - assert state["uploads"] == 0 - - @pytest.mark.parametrize("setting,value", [ ("SizeLimit", "0"), - ("SizeLimit", "-1GB"), - ("SizeLimit", "1.5GB"), - ("SizeLimit", "3XB"), + ("SizeLimit", "-1"), ("SizeLimit", "18446744073709551616"), - ("SizeLimit", "18446744073709551615GB"), - ("ChunkSizeRows", "1KB"), + ("ChunkSizeRows", "0"), ]) def test_arrow_input_rejects_invalid_upload_sizes(protocol, setting, value): key = f"spark.sql.session.localRelation{setting}" @@ -667,13 +716,15 @@ def test_arrow_input_upload_failure(protocol, failure, match, rows): def test_arrow_input_duplicate_chunks_preserve_multiplicity(protocol, batch_bytes): options = {"spark.sql.session.localRelationCacheThreshold": "0", "spark.sql.session.localRelationChunkSizeRows": "5", + "spark.sql.session.localRelationChunkSizeBytes": "1", "spark.sql.session.localRelationBatchOfChunksSizeBytes": str(batch_bytes)} with _arrow_input_service(protocol, **options) as (spark, state): spark.createDataFrame(pa.table({"id": [1] * 20})).toArrow() cached = state["relations"][0].chunked_cached_local_relation assert len(cached.dataHashes) == 4 assert len(set(cached.dataHashes)) == 1 - assert len(state["cache"]) == 2 # One schema and one unique data chunk. + assert not cached.HasField("schemaHash") + assert len(state["cache"]) == 1 assert state["result"] == (20, 1, 1) @@ -697,55 +748,3 @@ def import_module(name, *args, **kwargs): spark.createDataFrame(data) assert imported == [] assert state["config_requests"] == [] - - -@pytest.mark.parametrize("cached", [False, True]) -@pytest.mark.parametrize("missing", [ - "spark.sql.session.localRelationSizeLimit", - "spark.sql.session.localRelationChunkSizeRows", - "spark.sql.session.localRelationChunkSizeBytes", - "spark.sql.session.localRelationBatchOfChunksSizeBytes", - "all", -]) -def test_arrow_input_missing_upload_settings_use_defaults(protocol, missing, cached): - options = {missing: None} if missing != "all" else { - f"spark.sql.session.localRelation{suffix}": None - for suffix in ["SizeLimit", "ChunkSizeRows", "ChunkSizeBytes", "BatchOfChunksSizeBytes"] - } - options["spark.sql.session.localRelationCacheThreshold"] = "0" if cached else "1048576" - with _arrow_input_service(protocol, **options) as (spark, state): - spark.createDataFrame(pa.table({"id": [1, 2]})).toArrow() - assert state["result"] == (2, 1, 2) - assert len(state["config_requests"]) == 1 - assert bool(state["uploads"]) == cached - - -def test_arrow_input_ignores_chunk_settings_for_inline_input(protocol): - options = { - "spark.sql.session.localRelationChunkSizeRows": "invalid", - "spark.sql.session.localRelationChunkSizeBytes": "0", - } - with _arrow_input_service(protocol, **options) as (spark, state): - spark.createDataFrame(pa.table({"id": [1, 2]})).toArrow() - assert state["result"] == (2, 1, 2) - assert state["uploads"] == 0 - assert len(state["config_requests"]) == 1 - options["spark.sql.session.localRelationCacheThreshold"] = "0" - with _arrow_input_service(protocol, **options) as (spark, state): - with pytest.raises(Exception, match="Invalid configuration"): - spark.createDataFrame(pa.table({"id": [1, 2]})) - assert state["uploads"] == 0 - - -@pytest.mark.parametrize("rows", [2, 200000]) -def test_arrow_input_missing_cache_threshold_uses_default(protocol, rows): - import numpy as np - - table = pa.table({"id": np.arange(rows, dtype=np.int64)}) - with _arrow_input_service(protocol, **{ - "spark.sql.session.localRelationCacheThreshold": None - }) as (spark, state): - spark.createDataFrame(table).toArrow() - assert state["result"] == (rows, 0, rows - 1) - assert bool(state["uploads"]) == (table.nbytes >= 1048576) - assert len(state["config_requests"]) == 1 diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 8041221d..23213dbc 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -3087,7 +3087,7 @@ def allow_response_after_request(): assert service.calls == 1 -@pytest.mark.parametrize("names,expected", [(None, None), (["a"], ["a", "_2"]), ([], ["_1", "_2"])]) +@pytest.mark.parametrize("names,expected", [(None, ["x", "y"]), (["a"], ["a", "_2"]), ([], ["_1", "_2"])]) def test_arrow_input_normalization(names, expected): pa = pytest.importorskip("pyarrow") from pyspark.sql._arrow_input import prepare_table @@ -3095,7 +3095,7 @@ def test_arrow_input_normalization(names, expected): table = pa.table({"x": pa.chunked_array([[1], [2]]), "y": ["a", None]}) normalized, schema, columns = prepare_table(table, names, "TIMESTAMP_LTZ", "UTC", "false") assert normalized.equals(table) - assert schema.names == ["x", "y"] + assert schema is None assert columns == expected @@ -3146,8 +3146,8 @@ def test_arrow_input_supported_schema_types(schema): if isinstance(schema, (T.AtomicType, T.StructType)): assert result is schema else: - assert result.names == ["x"] - assert names == (["x"] if isinstance(schema, (list, tuple)) else None) + assert result is None + assert names == (None if isinstance(schema, (T.AtomicType, T.StructType)) else ["x"]) def test_arrow_input_array_and_map_struct_fields_allowed(): @@ -3234,7 +3234,8 @@ def test_arrow_input_timestamps(timestamp_type, expected): table = pa.table({"t": pa.array([1001], type=pa.timestamp("ns"))}) normalized, schema, _ = prepare_table(table, None, timestamp_type, "UTC", "false") - assert isinstance(schema.fields[0].dataType, expected) + assert schema is None + assert normalized.column(0).type.tz == ("UTC" if expected is T.TimestampType else None) assert normalized.column(0).type.unit == "us" assert normalized.column(0).cast(pa.int64()).to_pylist() == [1] From 06a1426ec17189845b67c6a00aa7a35010379a32 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Thu, 5 Nov 2026 07:26:08 +0900 Subject: [PATCH 08/10] [SPARK-60675][CONNECT] Match PySpark isLocal() for Arrow input renamed with toDF The server resolves toDF into a Project over the local relation, and Dataset.isLocal is false for a Project, so official PySpark returns False for a pa.Table with inferred or listed column names, and for local_df.toDF(...). Stop unwrapping ToDF in DataFrame::is_local and keep only the ChunkedCachedLocalRelation arm, matching PySpark 4.3.0 for inline and cached input with inferred, listed, StructType, and DDL schemas. Update the Arrow input tests to match. Co-authored-by: Isaac --- crates/spark-connect/src/dataframe.rs | 30 +++++++++++++++++++++------ python/tests/test_arrow_e2e.py | 4 +++- python/tests/test_arrow_handoff.py | 4 +++- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/crates/spark-connect/src/dataframe.rs b/crates/spark-connect/src/dataframe.rs index c35900f3..3c5b0305 100644 --- a/crates/spark-connect/src/dataframe.rs +++ b/crates/spark-connect/src/dataframe.rs @@ -1693,14 +1693,13 @@ impl DataFrame { } /// Check if this DataFrame is local (collected). + /// + /// Mirrors the server's `Dataset.isLocal`: only a local relation itself is local. The + /// server resolves `toDF`, which Arrow input with inferred or listed column names + /// applies, into a `Project`, which is not. pub fn is_local(&self) -> bool { - let mut plan = self.plan.as_ref(); - // Arrow input uses toDF to apply inferred or supplied column names. - while let LogicalPlan::ToDF { input, .. } = plan { - plan = input.as_ref(); - } matches!( - plan, + self.plan.as_ref(), LogicalPlan::LocalRelation { .. } | LogicalPlan::ChunkedCachedLocalRelation { .. } ) } @@ -3329,6 +3328,25 @@ mod plan_construction_tests { .expect("failed to build session") } + #[test] + fn is_local_only_for_a_local_relation_itself() { + let spark = session(); + let local = spark.empty_data_frame().unwrap(); + let cached = DataFrame::new( + spark.clone(), + LogicalPlan::ChunkedCachedLocalRelation { + data_hashes: vec!["hash".to_string()], + schema_hash: None, + }, + ); + assert!(local.is_local()); + assert!(cached.is_local()); + // As in PySpark, where the server resolves toDF into a Project. + assert!(!local.to_df(vec![]).is_local()); + assert!(!cached.to_df(vec!["x"]).is_local()); + assert!(!spark.range(1).unwrap().is_local()); + } + #[test] fn with_watermark_plan() { let spark = session(); diff --git a/python/tests/test_arrow_e2e.py b/python/tests/test_arrow_e2e.py index ced28b11..3a58d246 100644 --- a/python/tests/test_arrow_e2e.py +++ b/python/tests/test_arrow_e2e.py @@ -81,7 +81,9 @@ def test_create_dataframe_arrow_cached_upload(session): actual = frame.selectExpr("count(*) AS rows", "sum(id) AS checksum").collect()[0] assert actual[0] == rows assert actual[1] == rows * (rows - 1) // 2 - assert frame.isLocal() + # As in PySpark, inferred column names are applied with toDF, which the server + # resolves into a Project, so the frame is not local. + assert not frame.isLocal() finally: isolated.stop() diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index 4e24ab3b..8002cbf2 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -565,6 +565,8 @@ def test_arrow_input_schema_handoff(protocol, cached, explicit): "spark.sql.session.localRelationCacheThreshold": "0" if cached else "1048576", }) as (spark, state): frame = spark.createDataFrame(table, schema) + # As in PySpark: inferred column names are applied with toDF, which is not local. + assert frame.isLocal() == explicit assert frame.toArrow().column(0).to_pylist() == [2] assert state["result"] == (2, 1, 2) assert state["arrow_schemas"] == [table.schema] @@ -597,7 +599,7 @@ def test_arrow_input_upload_above_grpc_limit(protocol): assert table.nbytes > 128 * 1024**2 with _arrow_input_service(protocol) as (spark, state): df = spark.createDataFrame(table) - assert df.isLocal() + assert not df.isLocal() assert df.toArrow().column(0).to_pylist() == [rows] assert state["result"] == (rows, 0, rows - 1) assert state["relations"][0].HasField("chunked_cached_local_relation") From 10f35a4459ad6ec6e4b790c58fdf83d32fcc4f51 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Thu, 5 Nov 2026 08:05:05 +0900 Subject: [PATCH 09/10] [SPARK-60675][CONNECT] Skip toDF without column names and keep zero-column Arrow schemas As in PySpark (and the implementation from #141 that this replaced), apply toDF only when there are column names, so a zero-column table with rows stays local. PyArrow's input cast also drops the rows of a zero-column table, so the C Stream importer received no batches and the relation carried neither data nor a schema, which the server rejects with SCHEMA_REQUIRED_FOR_LOCAL_RELATION. Keep the imported Arrow schema as an empty batch instead, as PySpark's IPC stream does. Co-authored-by: Isaac --- crates/pyspark-rs/src/pyarrow_import.rs | 19 ++++++++++++++----- python/tests/test_arrow_e2e.py | 12 ++++++++++++ python/tests/test_arrow_handoff.py | 19 +++++++++++++++++++ 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/crates/pyspark-rs/src/pyarrow_import.rs b/crates/pyspark-rs/src/pyarrow_import.rs index 2ef7e528..eb39cfbf 100644 --- a/crates/pyspark-rs/src/pyarrow_import.rs +++ b/crates/pyspark-rs/src/pyarrow_import.rs @@ -18,6 +18,7 @@ //! PyArrow Table input through arrow-rs's owned C Stream importer. use arrow::pyarrow::{FromPyArrow, Table}; +use arrow::record_batch::RecordBatch; use pyo3::prelude::*; use spark_connect::session::SparkSession; @@ -67,9 +68,14 @@ pub(crate) fn create_dataframe( Vec::new() } else { // PyArrow 10 readers support _export_to_c, even without PyCapsules. - Table::from_pyarrow_bound(&normalized.call_method0("to_reader")?)? - .into_inner() - .0 + let (mut batches, arrow_schema) = + Table::from_pyarrow_bound(&normalized.call_method0("to_reader")?)?.into_inner(); + // The cast drops the rows of a zero-column table, leaving no batches. Keep its + // schema, as PySpark's IPC stream does, so the relation still carries one. + if batches.is_empty() { + batches.push(RecordBatch::new_empty(arrow_schema)); + } + batches }; let dataframe = py.detach(|| { match options.as_ref() { @@ -80,9 +86,12 @@ pub(crate) fn create_dataframe( } .to_pyerr() })?; + // As in PySpark, column names are applied with toDF only when there are any. let dataframe = match names { - Some(names) => dataframe.to_df(names.iter().map(String::as_str).collect()), - None => dataframe, + Some(names) if !names.is_empty() => { + dataframe.to_df(names.iter().map(String::as_str).collect()) + } + _ => dataframe, }; Ok(PyDataFrame::new(dataframe)) } diff --git a/python/tests/test_arrow_e2e.py b/python/tests/test_arrow_e2e.py index 3a58d246..eeec709e 100644 --- a/python/tests/test_arrow_e2e.py +++ b/python/tests/test_arrow_e2e.py @@ -88,6 +88,18 @@ def test_create_dataframe_arrow_cached_upload(session): isolated.stop() +@pytest.mark.parametrize("schema", [None, []], ids=["inferred", "empty-list"]) +def test_create_dataframe_arrow_zero_columns(session, schema): + table = pa.table({"a": [1, 2]}).select([]) + frame = session.createDataFrame(table, schema) + # As in PySpark, no toDF is applied without column names, so the frame is local. + assert frame.isLocal() + assert frame.columns == [] + # Both clients first cast the input to its Spark schema, and for a zero-column table + # that cast decides the row count (PyArrow drops the rows). + assert frame.count() == table.cast(pa.schema([])).num_rows + + @pytest.mark.parametrize("cached", [False, True], ids=["inline", "cached"]) @pytest.mark.parametrize("timestamp_type", ["TIMESTAMP_LTZ", "TIMESTAMP_NTZ"]) def test_create_dataframe_arrow_inferred_timestamp(session, cached, timestamp_type): diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index 8002cbf2..f38e3e55 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -609,6 +609,25 @@ def test_arrow_input_upload_above_grpc_limit(protocol): assert all(schema == table.schema for schema in state["arrow_schemas"]) +@pytest.mark.parametrize("kind", ["inferred", "empty-list", "struct"]) +def test_arrow_input_zero_column_table(protocol, kind): + from pyspark.sql import types as T + + # The input cast drops the rows of a zero-column table, leaving no batches. + table = pa.table({"a": [1, 2]}).select([]) + schema = {"inferred": None, "empty-list": [], "struct": T.StructType([])}[kind] + explicit = kind == "struct" + with _arrow_input_service(protocol) as (spark, state): + frame = spark.createDataFrame(table, schema) + # As in PySpark, no toDF is applied without column names, so the frame is local. + assert frame.isLocal() + assert frame.toArrow().column(0).to_pylist() == [0] + assert "column_names" not in state + # The relation still carries the Arrow schema, as PySpark's IPC stream does. + assert state["arrow_schemas"] == [pa.schema([])] + assert state["relations"][0].local_relation.HasField("schema") == explicit + + @pytest.mark.parametrize("schema,expected_width", [ ("x long", "1"), ("x long, y long, z long", "3"), From 7e3137030334efaece92b805c0e799b41094cfeb Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Thu, 5 Nov 2026 08:58:43 +0900 Subject: [PATCH 10/10] [SPARK-60675][CONNECT] Remove RuntimeConf::get_many, used only by the replaced Arrow path SPARK-60739 (#141) added RuntimeConf::get_many for its createDataFrame(pyarrow.Table) path, which this PR replaces. Its only caller is gone, it is untested, and it has not been released, so remove it rather than keep an unused public method. Co-authored-by: Isaac --- crates/spark-connect/src/conf.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/crates/spark-connect/src/conf.rs b/crates/spark-connect/src/conf.rs index d90907cf..d43d611f 100644 --- a/crates/spark-connect/src/conf.rs +++ b/crates/spark-connect/src/conf.rs @@ -70,12 +70,6 @@ impl RuntimeConf { Ok(results.into_iter().next().flatten()) } - /// Get several configuration values in one request, in the order of `keys`; an unknown - /// key errors as in `get`. Mirrors the reference client's `get_config_dict`. - pub fn get_many(&self, keys: &[&str]) -> Result>> { - block_on(self.client.get_configs(keys)) - } - /// Get all configuration values as a HashMap. /// /// # Example