Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/pyspark-rs/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ mod observation;
mod pipelines;
mod profiler;
mod pyarrow_export;
mod pyarrow_import;
mod readwriter;
mod resource;
mod row;
Expand Down
97 changes: 97 additions & 0 deletions crates/pyspark-rs/src/pyarrow_import.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

//! PyArrow Table input through arrow-rs's owned C Stream importer.

use arrow::pyarrow::{FromPyArrow, Table};
use arrow::record_batch::RecordBatch;
use pyo3::prelude::*;
use spark_connect::session::SparkSession;

use crate::dataframe::PyDataFrame;
use crate::errors::ResultExt;

pub(crate) fn create_dataframe(
py: Python<'_>,
session: &SparkSession,
table: &Bound<'_, PyAny>,
schema: Option<&Bound<'_, PyAny>>,
) -> PyResult<PyDataFrame> {
let helper = py.import("pyspark.sql._arrow_input")?;
// Resolve DDL with the same converter as the existing local-data path.
let schema = match schema {
Some(s) if s.extract::<String>().is_ok() => Some(crate::types::data_type_to_py(
py,
&crate::types::py_to_data_type(s)?,
)?),
Some(s) => Some(s.clone().unbind()),
None => None,
};
let schema = schema.unwrap_or_else(|| py.None());
// Empty input needs only the explicit schema, and no server configuration.
let empty: bool = table.call_method0("__len__")?.extract::<usize>()? == 0;
let options = if empty {
None
} else {
Some(py.detach(|| session.arrow_input_options().to_pyerr())?)
};
let result = helper.getattr("prepare_table")?.call1((
table,
schema,
options.as_ref().and_then(|o| o.timestamp_type()),
options.as_ref().and_then(|o| o.timezone()),
options.as_ref().and_then(|o| o.use_large_var_types()),
))?;
let normalized = result.get_item(0)?;
let schema = result.get_item(1)?;
let spark_schema = if schema.is_none() {
None
} else {
Some(crate::types::py_to_data_type(&schema)?)
};
let names: Option<Vec<String>> = result.get_item(2)?.extract()?;
let batches = if empty {
Vec::new()
} else {
// PyArrow 10 readers support _export_to_c, even without PyCapsules.
let (mut batches, arrow_schema) =
Table::from_pyarrow_bound(&normalized.call_method0("to_reader")?)?.into_inner();
// The cast drops the rows of a zero-column table, leaving no batches. Keep its
// schema, as PySpark's IPC stream does, so the relation still carries one.
if batches.is_empty() {
batches.push(RecordBatch::new_empty(arrow_schema));
}
batches
};
let dataframe = py.detach(|| {
match options.as_ref() {
Some(options) => {
session.create_dataframe_from_arrow_with_options(batches, spark_schema, options)
}
None => session.create_dataframe_from_arrow(batches, spark_schema),
}
.to_pyerr()
})?;
// As in PySpark, column names are applied with toDF only when there are any.
let dataframe = match names {
Some(names) if !names.is_empty() => {
dataframe.to_df(names.iter().map(String::as_str).collect())
}
_ => dataframe,
};
Ok(PyDataFrame::new(dataframe))
}
118 changes: 6 additions & 112 deletions crates/pyspark-rs/src/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -278,118 +278,11 @@ impl PySparkSession {
pub fn new(session: SparkSession) -> Self {
PySparkSession { session }
}

/// `createDataFrame` of a pyarrow Table, as the reference client does it: the column
/// names come from a list `schema` (extra columns are named `_N`), or else from the table;
/// the Spark types come from a `StructType` (or DDL) `schema`, or else from the Arrow
/// types. The table is cast to the Arrow form of those types, with its timestamps
/// localized to the session time zone, and sent as an Arrow IPC stream.
fn create_dataframe_from_arrow(
&self,
py: Python<'_>,
table: &Bound<'_, PyAny>,
schema: Option<&Bound<'_, PyAny>>,
) -> PyResult<PyDataFrame> {
let mut names: Option<Vec<String>> = None;
let mut user_schema: Option<DataType> = None;
match schema {
None => names = Some(table.getattr("column_names")?.extract()?),
Some(s) => {
if let Ok(mut given) = s.extract::<Vec<String>>() {
let num_columns: usize = table.getattr("num_columns")?.extract()?;
given.extend((given.len()..num_columns).map(|i| format!("_{}", i + 1)));
names = Some(given);
} else {
user_schema = Some(crate::types::py_to_data_type(s)?);
}
}
}
// As the reference does for any empty input: a relation of the user's schema, which
// a list of names does not give.
if table.len()? == 0 {
return match user_schema {
Some(dt) => Ok(PyDataFrame::new(
py.detach(|| self.session.create_dataframe(vec![], dt))
.to_pyerr()?,
)),
None => Err(crate::column::raise_pyspark(
py,
"PySparkValueError",
"CANNOT_INFER_EMPTY_SCHEMA",
&[],
)),
};
}

let conf = self.session.conf();
let configs = py
.detach(|| {
conf.get_many(&[
"spark.sql.timestampType",
"spark.sql.session.timeZone",
"spark.sql.execution.arrow.useLargeVarTypes",
])
})
.to_pyerr()?;
let config = |i: usize| configs.get(i).cloned().flatten();
let prefer_timestamp_ntz = config(0).as_deref() == Some("TIMESTAMP_NTZ");
let timezone = config(1);
let prefers_large_types = config(2).as_deref() == Some("true");

let types = py.import("pyspark.sql.pandas.types")?;
let spark_schema = match &user_schema {
Some(dt @ DataType::Struct { .. }) => {
crate::types::data_type_to_py(py, dt)?.into_bound(py)
}
_ => types.call_method1(
"from_arrow_schema",
(table.getattr("schema")?, prefer_timestamp_ntz),
)?,
};
let kwargs = PyDict::new(py);
kwargs.set_item("error_on_duplicated_field_names_in_struct", true)?;
kwargs.set_item("timezone", "UTC")?;
kwargs.set_item("prefers_large_types", prefers_large_types)?;
let arrow_schema = types.call_method("to_arrow_schema", (&spark_schema,), Some(&kwargs))?;
let converted = types
.call_method1(
"_check_arrow_table_timestamps_localize",
(table, &spark_schema, true, timezone),
)?
.call_method1("cast", (arrow_schema,))?
.call_method1("rename_columns", (spark_schema.getattr("names")?,))?;

let pa = py.import("pyarrow")?;
let sink = pa.call_method0("BufferOutputStream")?;
let writer = pa
.getattr("ipc")?
.call_method1("new_stream", (&sink, converted.getattr("schema")?))?;
writer.call_method1("write_table", (&converted,))?;
writer.call_method0("close")?;
let stream = sink.call_method0("getvalue")?.call_method0("to_pybytes")?;
let data = stream.cast::<pyo3::types::PyBytes>()?.as_bytes().to_vec();

// The reference sends the user's schema, or none for the server to read the Arrow
// one; the plan always carries a schema, so the Arrow one is sent explicitly.
let relation_schema = match user_schema {
Some(dt) => dt,
None => crate::types::py_to_data_type(&spark_schema)?,
};
let df = self
.session
.create_dataframe_from_arrow(data, relation_schema);
// Like the reference, apply the column names with toDF.
Ok(PyDataFrame::new(match names {
Some(names) if !names.is_empty() => {
df.to_df(names.iter().map(String::as_str).collect())
}
_ => df,
}))
}
}

/// Whether `obj` is an instance of `module.class`, without importing `module` when it is not
/// loaded yet: an instance can only exist once its module was imported.
/// loaded yet: an instance can only exist once its module was imported. A `None` entry in
/// `sys.modules` blocks the import, so it counts as not loaded.
fn is_loaded_instance(
py: Python<'_>,
obj: &Bound<'_, PyAny>,
Expand All @@ -398,8 +291,8 @@ fn is_loaded_instance(
) -> PyResult<bool> {
let modules = py.import("sys")?.getattr("modules")?;
match modules.cast::<PyDict>()?.get_item(module)? {
Some(m) => obj.is_instance(&m.getattr(class)?),
None => Ok(false),
Some(m) if !m.is_none() => obj.is_instance(&m.getattr(class)?),
_ => Ok(false),
}
}

Expand Down Expand Up @@ -508,8 +401,9 @@ impl PySparkSession {
// signature parity. See PARITY_GAP_ANALYSIS.md.
let _ = (samplingRatio, verifySchema);

// A caller holding a Table has already imported PyArrow.
if is_loaded_instance(py, data, "pyarrow", "Table")? {
return self.create_dataframe_from_arrow(py, data, schema);
return crate::pyarrow_import::create_dataframe(py, &self.session, data, schema);
}

// Accept a pandas DataFrame (as pyspark does): convert it to a list of row lists
Expand Down
1 change: 1 addition & 0 deletions crates/spark-connect-core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ serde_json = "1"
regex = "1"
prost-types.workspace = true
crc32fast = "1"
sha2 = "0.10"

[dev-dependencies]
tempfile = "3"
96 changes: 67 additions & 29 deletions crates/spark-connect-core/src/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -744,33 +744,7 @@ impl SparkConnectClient {

// Build request stream with proper chunking/batching
let requests = build_artifact_request_stream(self.session_id.clone(), artifacts)?;

// Create a futures stream from the requests
let request_stream = futures::stream::iter(requests);

// Call the AddArtifacts streaming RPC
let mut req = Request::new(request_stream);
self._attach_metadata(&mut req);

let response = self
.stub
.clone()
.add_artifacts(req)
.await
.map_err(SparkError::from_grpc_status)?
.into_inner();

// Check that all artifacts were successfully received
for summary in response.artifacts {
if !summary.is_crc_successful {
return Err(SparkError::connect_msg(format!(
"CRC check failed for artifact: {}",
summary.name
)));
}
}

Ok(())
self.upload_artifact_requests(requests, None).await
}

/// Upload a single local file as an artifact under an explicit artifact name
Expand All @@ -784,8 +758,61 @@ impl SparkConnectClient {
Box::new(FileArtifact::new(local_path)),
)];
let requests = build_artifact_request_stream(self.session_id.clone(), artifacts)?;
let request_stream = futures::stream::iter(requests);
let mut req = Request::new(request_stream);
self.upload_artifact_requests(requests, None).await
}

/// Upload session-scoped cache blobs, returning SHA-256 hashes in input order.
/// Existing artifacts and duplicate blobs are uploaded only once.
pub async fn cache_artifacts(&self, blobs: Vec<Vec<u8>>) -> Result<Vec<String>> {
use crate::artifact::InMemoryArtifact;
use sha2::{Digest, Sha256};
use std::collections::HashSet;

let hashes: Vec<String> = blobs
.iter()
.map(|blob| {
Sha256::digest(blob)
.iter()
.map(|byte| format!("{byte:02x}"))
.collect::<String>()
})
.collect();
let names: Vec<String> = hashes.iter().map(|hash| format!("cache/{hash}")).collect();
let unique: HashSet<&str> = names.iter().map(String::as_str).collect();
if unique.is_empty() {
return Ok(hashes);
}
let cached = self
.artifact_status(&unique.into_iter().collect::<Vec<_>>())
.await?;
let mut seen = HashSet::new();
let mut artifacts = Vec::new();
for (blob, name) in blobs.into_iter().zip(names) {
if !cached.get(&name).copied().unwrap_or(false) && seen.insert(name.clone()) {
artifacts.push(NamedArtifact::new(
name,
Box::new(InMemoryArtifact::new(blob)),
));
}
}
if artifacts.is_empty() {
return Ok(hashes);
}
let expected: HashSet<String> = artifacts.iter().map(|a| a.name.clone()).collect();
let requests = build_artifact_request_stream(self.session_id.clone(), artifacts)?;
self.upload_artifact_requests(requests, Some(&expected))
.await?;
Ok(hashes)
}

/// Send artifact frames and validate the response.
/// Cache uploads also require an acknowledgement for every requested artifact.
async fn upload_artifact_requests(
&self,
requests: Vec<spark_connect_proto::AddArtifactsRequest>,
expected_cache_names: Option<&std::collections::HashSet<String>>,
) -> Result<()> {
let mut req = Request::new(futures::stream::iter(requests));
self._attach_metadata(&mut req);
let response = self
.stub
Expand All @@ -794,13 +821,24 @@ impl SparkConnectClient {
.await
.map_err(SparkError::from_grpc_status)?
.into_inner();
let mut received = std::collections::HashSet::new();
for summary in response.artifacts {
if !summary.is_crc_successful {
return Err(SparkError::connect_msg(format!(
"CRC check failed for artifact: {}",
summary.name
)));
}
if expected_cache_names.is_some() {
received.insert(summary.name);
}
}
if let Some(expected) = expected_cache_names {
if !expected.is_subset(&received) {
return Err(SparkError::connect_msg(
"Missing cache artifact upload acknowledgement",
));
}
}
Ok(())
}
Expand Down
Loading
Loading