Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 16 additions & 6 deletions crates/pyspark-rs/src/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1039,9 +1039,14 @@ impl PySparkSession {
let schema = DataType::Struct {
fields: fields.clone(),
};
let frame = self
.session
.create_dataframe_from_inline_arrow_ipc(ipc.extract()?, schema);
let ipc: Vec<u8> = ipc.extract()?;
let frame = py
.detach(|| {
self.session
.create_dataframe_from_inline_arrow_ipc(ipc, schema)
.map_err(Box::new)
})
.to_pyerr()?;
// Match pandas row input: explicit nonempty field names travel
// through toDF as well as the local relation's logical schema.
return Ok(PyDataFrame::new(
Expand All @@ -1064,9 +1069,14 @@ impl PySparkSession {
let schema = DataType::Struct {
fields: fields.clone(),
};
let frame = self
.session
.create_dataframe_from_inline_arrow_ipc(ipc.extract()?, schema);
let ipc: Vec<u8> = ipc.extract()?;
let frame = py
.detach(|| {
self.session
.create_dataframe_from_inline_arrow_ipc(ipc, schema)
.map_err(Box::new)
})
.to_pyerr()?;
// Match pandas row input: explicit nonempty field names travel
// through toDF as well as the local relation's logical schema.
return Ok(PyDataFrame::new(
Expand Down
2 changes: 1 addition & 1 deletion crates/spark-connect/src/arrow_input.rs
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ fn validate(batches: &[RecordBatch]) -> Result<()> {
Ok(())
}

fn batch_data_size(batch: &RecordBatch) -> usize {
pub(crate) fn batch_data_size(batch: &RecordBatch) -> usize {
batch
.columns()
.iter()
Expand Down
117 changes: 85 additions & 32 deletions crates/spark-connect/src/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -418,16 +418,42 @@ impl SparkSession {
// Empty data -> send the schema only (no Arrow `data`); the server builds an empty
// relation from the schema. A zero-row Arrow payload would be empty bytes, which the
// server cannot parse ("Unexpected end of input. Missing schema.").
let data = if rows.is_empty() {
None
} else {
Some(rows_to_arrow_ipc(&rows, &schema)?)
};
let plan = LogicalPlan::LocalRelation {
if rows.is_empty() {
let plan = LogicalPlan::LocalRelation {
schema: Some(schema),
data: None,
};
return Ok(DataFrame::new(self.clone(), plan));
}
let batch = rows_to_arrow_batch(&rows, &schema)?;
Ok(DataFrame::new(
self.clone(),
self.local_relation(batch, schema)?,
))
}

/// The local relation of `batch`: inline, without reading the server's configuration, when
/// it is smaller than [`LOCAL_RELATION_SETTINGS_BYTES`]; otherwise as the Arrow importer
/// sends one, which reads the server's local-relation settings and uploads the data as
/// cache artifacts at `spark.sql.session.localRelationCacheThreshold`, as PySpark does. A
/// single gRPC message cannot carry a relation beyond the server's message size limit.
fn local_relation(
&self,
batch: arrow::record_batch::RecordBatch,
schema: DataType,
) -> Result<LogicalPlan> {
if crate::arrow_input::batch_data_size(&batch) >= LOCAL_RELATION_SETTINGS_BYTES {
return crate::arrow_input::create_relation(
&self.client,
vec![batch],
Some(schema),
None,
);
}
Ok(LogicalPlan::LocalRelation {
schema: Some(schema),
data,
};
Ok(DataFrame::new(self.clone(), plan))
data: Some(batch_to_arrow_ipc(&batch)?),
})
}

/// Create an inline local relation from columns with the same encoding as local rows.
Expand All @@ -441,29 +467,39 @@ impl SparkSession {
) -> Result<DataFrame> {
let arrow_schema = arrow::datatypes::Schema::new(schema_to_arrow_fields(&schema)?);
let row_count = columns.first().map_or(0, |column| column.len());
let data = Some(columns_to_arrow_ipc(columns, arrow_schema, row_count)?);
let plan = LogicalPlan::LocalRelation {
schema: Some(schema),
data,
};
Ok(DataFrame::new(self.clone(), plan))
let batch = columns_to_arrow_batch(columns, arrow_schema, row_count)?;
Ok(DataFrame::new(
self.clone(),
self.local_relation(batch, schema)?,
))
}

/// Internal binding bridge for Arrow IPC already validated and encoded from
/// local rows or columns. Retains the row path's inline transport policy.
/// local rows or columns. Retains the row path's transport policy: inline, unless the
/// IPC is large enough to consult the server's local-relation settings.
#[doc(hidden)]
pub fn create_dataframe_from_inline_arrow_ipc(
&self,
data: Vec<u8>,
schema: DataType,
) -> DataFrame {
DataFrame::new(
self.clone(),
) -> Result<DataFrame> {
let plan = if data.len() >= LOCAL_RELATION_SETTINGS_BYTES {
let reader =
arrow::ipc::reader::StreamReader::try_new(std::io::Cursor::new(data), None)
.map_err(|e| {
SparkError::connect_msg(format!("Failed to read Arrow IPC: {e}"))
})?;
let batches = reader
.collect::<std::result::Result<Vec<_>, _>>()
.map_err(|e| SparkError::connect_msg(format!("Failed to read Arrow IPC: {e}")))?;
crate::arrow_input::create_relation(&self.client, batches, Some(schema), None)?
} else {
LogicalPlan::LocalRelation {
schema: Some(schema),
data: Some(data),
},
)
}
};
Ok(DataFrame::new(self.clone(), plan))
}

/// Create a local DataFrame from Arrow batches without materializing rows.
Expand Down Expand Up @@ -895,13 +931,18 @@ fn row_width_mismatch(row_index: usize, actual: usize, expected: usize) -> Spark

/// Convert Arrow RecordBatch data to IPC bytes.
/// Used by createDataFrame to package local data.
fn rows_to_arrow_ipc(rows: &[Row], schema: &DataType) -> Result<Vec<u8>> {
/// The size of a local relation, in Arrow data, from which `createDataFrame` reads the
/// server's local-relation settings to decide whether to upload it: half of the default
/// `spark.sql.session.localRelationCacheThreshold`, and well below the default gRPC message
/// size limit of the server (128 MiB). Smaller relations are sent inline without that RPC.
const LOCAL_RELATION_SETTINGS_BYTES: usize = 32 << 20;

fn rows_to_arrow_batch(
rows: &[Row],
schema: &DataType,
) -> Result<arrow::record_batch::RecordBatch> {
use arrow::datatypes::Schema as ArrowSchema;

if rows.is_empty() {
return Ok(vec![]);
}

// Convert our schema to Arrow schema
let arrow_schema_vec = schema_to_arrow_fields(schema)?;
let arrow_schema = ArrowSchema::new(arrow_schema_vec);
Expand All @@ -925,21 +966,33 @@ fn rows_to_arrow_ipc(rows: &[Row], schema: &DataType) -> Result<Vec<u8>> {
columns.push(values_to_arrow(&column, &field.data_type)?);
}

columns_to_arrow_ipc(columns, arrow_schema, rows.len())
columns_to_arrow_batch(columns, arrow_schema, rows.len())
}

/// The Arrow IPC stream of local rows, as the server reads a local relation.
#[cfg(test)]
fn rows_to_arrow_ipc(rows: &[Row], schema: &DataType) -> Result<Vec<u8>> {
if rows.is_empty() {
return Ok(vec![]);
}
batch_to_arrow_ipc(&rows_to_arrow_batch(rows, schema)?)
}

fn columns_to_arrow_ipc(
fn columns_to_arrow_batch(
columns: Vec<arrow::array::ArrayRef>,
arrow_schema: arrow::datatypes::Schema,
row_count: usize,
) -> Result<Vec<u8>> {
use arrow::ipc::writer::StreamWriter;
) -> Result<arrow::record_batch::RecordBatch> {
use arrow::record_batch::RecordBatch;

// Preserve the row count even for a batch without columns.
let options = arrow::record_batch::RecordBatchOptions::new().with_row_count(Some(row_count));
let batch = RecordBatch::try_new_with_options(Arc::new(arrow_schema), columns, &options)
.map_err(|e| SparkError::connect_msg(format!("Failed to create Arrow batch: {}", e)))?;
RecordBatch::try_new_with_options(Arc::new(arrow_schema), columns, &options)
.map_err(|e| SparkError::connect_msg(format!("Failed to create Arrow batch: {}", e)))
}

fn batch_to_arrow_ipc(batch: &arrow::record_batch::RecordBatch) -> Result<Vec<u8>> {
use arrow::ipc::writer::StreamWriter;

// Write to IPC stream
let mut buffer = Vec::new();
Expand Down
48 changes: 48 additions & 0 deletions python/tests/test_arrow_handoff.py
Original file line number Diff line number Diff line change
Expand Up @@ -1177,6 +1177,54 @@ def test_add_artifacts_names_each_artifact_as_pyspark(protocol, tmp_path):
}


# Local relations of at least 32 MiB read the server's local-relation settings, and are uploaded
# as cache artifacts at its threshold, whichever path encoded them; a single gRPC message
# cannot carry a relation beyond the server's message size limit.

@pytest.mark.parametrize("path", ["local_rows", "pandas_rows", "float64", "strings"])
def test_large_local_relations_follow_the_cache_threshold(protocol, path):
# (The mock service keeps gRPC's default 4 MiB message limit, so the inline variant of
# these inputs, below a higher threshold, can only be checked against a real server.)
import numpy as np

if path == "local_rows":
data = [("y" * 2000, i) for i in range(20_000)]
schema = T.StructType([T.StructField("v", T.StringType()), T.StructField("i", T.LongType())])
expected_rows = 20_000
elif path == "pandas_rows":
# float32 columns into DoubleType keep the row path.
data = pd.DataFrame(np.ones((100_000, 45), dtype=np.float32),
columns=[f"c{i}" for i in range(45)])
schema = T.StructType([T.StructField(c, T.DoubleType()) for c in data.columns])
expected_rows = 100_000
elif path == "float64":
data = pd.DataFrame(np.ones((100_000, 45)), columns=[f"c{i}" for i in range(45)])
schema = T.StructType([T.StructField(c, T.DoubleType()) for c in data.columns])
expected_rows = 100_000
else:
data = pd.DataFrame({"v": ["x" * 3500] * 10_000}, dtype=object)
schema = T.StructType([T.StructField("v", T.StringType())])
expected_rows = 10_000
overrides = {"spark.sql.session.localRelationCacheThreshold": "1"}
with _arrow_input_service(protocol, **overrides) as (spark, state):
df = spark.createDataFrame(data, schema)
df.toArrow()
assert sum(table.num_rows for table in state["input_tables"]) == expected_rows
assert state["uploads"]
assert any("spark.sql.session.localRelationCacheThreshold" in keys
for _, keys in state["config_requests"])


def test_small_local_relations_stay_inline_without_reading_settings(protocol):
with _arrow_input_service(protocol) as (spark, state):
schema = T.StructType([T.StructField("v", T.StringType()), T.StructField("i", T.LongType())])
spark.createDataFrame([("y" * 100, i) for i in range(100)], schema).toArrow()
assert not state["uploads"]
assert not any("spark.sql.session.localRelationCacheThreshold" in keys
for _, keys in state["config_requests"])
assert state["input_tables"][0].num_rows == 100


# Signed-int64 pandas input uses the same Arrow importer and mock service.

@pytest.mark.parametrize("storage", ["object", "default", "python", "pyarrow", "python_nan", "pyarrow_nan"])
Expand Down
Loading