From a09480f33c843e59fb98928724a8d6aa1f6c95c5 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 13:45:00 -0800 Subject: [PATCH 01/17] Add an experimental Rust Spark Connect protocol frontend --- .github/workflows/server-frontend.yml | 92 + Cargo.lock | 18 + Cargo.toml | 3 + README.md | 11 + SECURITY.md | 19 + crates/spark-connect-server/Cargo.toml | 46 + crates/spark-connect-server/README.md | 98 ++ crates/spark-connect-server/build.rs | 145 ++ crates/spark-connect-server/src/engine/mod.rs | 126 ++ crates/spark-connect-server/src/lib.rs | 63 + crates/spark-connect-server/src/schema.rs | 224 +++ crates/spark-connect-server/src/service.rs | 1232 +++++++++++++ .../tests/engine_injection.rs | 1528 +++++++++++++++++ crates/spark-connect-server/tests/protocol.rs | 903 ++++++++++ .../spark-connect-server/tests/support/mod.rs | 265 +++ dev/design/ci.md | 15 +- 16 files changed, 4786 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/server-frontend.yml create mode 100644 crates/spark-connect-server/Cargo.toml create mode 100644 crates/spark-connect-server/README.md create mode 100644 crates/spark-connect-server/build.rs create mode 100644 crates/spark-connect-server/src/engine/mod.rs create mode 100644 crates/spark-connect-server/src/lib.rs create mode 100644 crates/spark-connect-server/src/schema.rs create mode 100644 crates/spark-connect-server/src/service.rs create mode 100644 crates/spark-connect-server/tests/engine_injection.rs create mode 100644 crates/spark-connect-server/tests/protocol.rs create mode 100644 crates/spark-connect-server/tests/support/mod.rs diff --git a/.github/workflows/server-frontend.yml b/.github/workflows/server-frontend.yml new file mode 100644 index 00000000..7540c9ee --- /dev/null +++ b/.github/workflows/server-frontend.yml @@ -0,0 +1,92 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Server frontend CI + +on: + pull_request: + branches: [master, "branch-*"] + paths: &frontend-paths + - ".github/workflows/server-frontend.yml" + - "Cargo.toml" + - "Cargo.lock" + - ".cargo/**" + - "rust-toolchain*" + - "crates/spark-connect-server/**" + - "crates/spark-connect-proto/**" + - "crates/spark-connect-core/**" + - "crates/spark-connect/**" + push: + branches: [master, "branch-*"] + paths: *frontend-paths + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CARGO_BUILD_JOBS: "2" + CARGO_PROFILE_TEST_DEBUG: "0" + RUSTFLAGS: "-D warnings" + +jobs: + frontend: + name: Frontend (Rust ${{ matrix.rust-version }}) + runs-on: ubuntu-24.04 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + rust-version: [stable, "1.88.0"] + steps: + - uses: actions/checkout@v4 + + - name: Install Rust toolchain + # Use runner-provided rustup to keep actions limited to actions/*. + run: | + rustup toolchain install ${{ matrix.rust-version }} --profile minimal --component clippy + rustup default ${{ matrix.rust-version }} + + - name: Cache cargo and target + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-${{ runner.arch }}-cargo-server-frontend-${{ matrix.rust-version }}-${{ hashFiles('Cargo.lock') }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-cargo-server-frontend-${{ matrix.rust-version }}- + + - name: Install protobuf compiler + # Ubuntu 24.04 provides protoc with proto3 optional support (3.15+). + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + protoc --version + + - name: Test frontend + # Protocol and injected-engine tests use local fixtures without an execution backend. + run: cargo test --locked -p spark-connect-server + + - name: Strict Clippy for frontend + run: cargo clippy --locked -p spark-connect-server --all-targets --no-deps -- -D warnings diff --git a/Cargo.lock b/Cargo.lock index 40cf5dab..edfae584 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4048,6 +4048,24 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spark-connect-server" +version = "0.1.0" +dependencies = [ + "apache-spark-connect", + "apache-spark-connect-proto", + "arrow", + "futures", + "h2", + "prost", + "tokio", + "tokio-stream", + "tonic", + "tonic-build", + "tonic-prost", + "uuid", +] + [[package]] name = "sqlparser" version = "0.60.0" diff --git a/Cargo.toml b/Cargo.toml index fb6261a1..473aebeb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ members = [ "crates/spark-connect-build", "crates/pyspark-rs", "examples", + "crates/spark-connect-server", ] exclude = [ "examples/wasm-udfs", @@ -44,6 +45,8 @@ exclude = [ # (so `cargo test --workspace` stays green without it). Build them explicitly with # `cargo build --manifest-path examples/wasm-udfs/Cargo.toml` (etc) in CI or when # the target is installed locally. +# The experimental protocol frontend is opt-in; build or test it with +# `-p spark-connect-server`. It is included in workspace lint checks. default-members = [ "crates/spark-connect-proto", "crates/spark-connect-core", diff --git a/README.md b/README.md index d75af986..fcc755db 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,17 @@ fn main() -> Result<(), Box> { See the [WASM UDF guide](https://apache.github.io/spark-connect-rust/udfs/) for the full setup, SQL registration, supported types, and more. +## Experimental server frontend + +The unpublished [`spark-connect-server`](crates/spark-connect-server/README.md) +library provides a Rust Spark Connect protocol frontend. It manages protocol sessions, +operations and Arrow IPC responses, delegating planning and execution to an +application-provided `Engine`. The library includes no execution backend or standalone +server executable; its protocol tests use injected engine fixtures. + +The frontend is intended for local, trusted experiments. Read its +[security model](SECURITY.md#experimental-server-frontend) before embedding it. + ## Contributing Issues are tracked in ASF JIRA under diff --git a/SECURITY.md b/SECURITY.md index 80e53253..d80df8b4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -25,3 +25,22 @@ material, TLS validation) inherit from that model. Automated security scanning agents should consult that document for the project's in-scope / out-of-scope declarations before reporting issues. + +## Experimental server frontend + +The optional `spark-connect-server` library is an experimental protocol +frontend for local, trusted workloads. It does not provide authentication, +authorization or TLS. Session IDs, operation IDs and client-provided user +context identify protocol state; they do not establish a caller's identity +or access rights. + +Bind its listener only to loopback. The embedding application supplies the +listener and the `Engine` implementation; the library does not enforce the +listener's network scope. Any process that can reach it can submit requests +to that engine. Protocol limits and session isolation are resource and +correctness guards, not a security boundary between mutually untrusted users. + +This frontend does not include an execution backend or a standalone server. +Its injected-engine tests establish frontend protocol behavior, not safe +exposure of an execution engine to untrusted clients. Apache Spark's operator +security guidance remains relevant to any application that connects an engine. diff --git a/crates/spark-connect-server/Cargo.toml b/crates/spark-connect-server/Cargo.toml new file mode 100644 index 00000000..68def7fa --- /dev/null +++ b/crates/spark-connect-server/Cargo.toml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "spark-connect-server" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false +description = "Experimental Spark Connect server frontend with an injectable execution engine" + +[lints] +workspace = true + +[dependencies] +spark-connect-proto = { package = "apache-spark-connect-proto", path = "../spark-connect-proto" } +arrow.workspace = true +futures.workspace = true +prost.workspace = true +tokio = { workspace = true, features = ["net"] } +tokio-stream = { version = "0.1", features = ["net"] } +tonic = { workspace = true, features = ["server", "router"] } +tonic-prost.workspace = true +uuid.workspace = true + +[build-dependencies] +tonic-build = "0.14" + +[dev-dependencies] +h2 = "0.4" +spark-connect = { package = "apache-spark-connect", path = "../spark-connect" } diff --git a/crates/spark-connect-server/README.md b/crates/spark-connect-server/README.md new file mode 100644 index 00000000..f236fcc8 --- /dev/null +++ b/crates/spark-connect-server/README.md @@ -0,0 +1,98 @@ + + +# Experimental Spark Connect server frontend + +This unpublished library serves a subset of the Spark Connect protocol using the +workspace's `apache-spark-connect-proto` messages. The caller supplies an +`Engine` implementation and a bound Tokio TCP listener to `serve_with_engine`. +No execution engine or standalone server executable is included. + +Rust owns protocol sessions, operation IDs, response framing and result-stream +lifetimes. Each protocol session receives an `EngineSession` that owns its +planning, catalog and configuration state. Its `prepare` method returns a +`PreparedQuery` or command response payloads; the frontend adds protocol IDs and +`ResultComplete`. The engine decides which plans, commands and metadata requests +it can execute and reports unsupported inputs explicitly. + +## Protocol subset + +The frontend routes `ExecutePlan`, `AnalyzePlan` and `Config` to the engine. It +implements `ReleaseExecute` and `ReleaseSession`, including operation cancellation +through `ReleaseAll` and session recreation when `allow_reconnect` permits it. +Completed responses are not retained. Reattachment, artifact upload/status, +`Interrupt`, enriched error details, session cloning and status queries return +`UNIMPLEMENTED`. + +The frontend validates UUID session and operation IDs, rejects duplicate active +operation IDs within a session, and checks client-observed server session IDs. +Client user IDs distinguish session keys but are not authenticated identities. + +## Arrow results + +A prepared query declares an Arrow schema before execution. The frontend converts +supported types to the Spark schema representation and checks each batch against +that declaration. Equivalent Arrow representations and differences in batch +nullability or metadata are accepted. Declared field metadata and unsupported +types fail explicitly. + +Engines may yield Arrow record batches or complete Arrow IPC streams containing +one batch and its declared row count. The frontend encodes record batches and +forwards IPC that fits the response limit; oversized batches are split by rows. +Result envelopes, including schema and batch responses, are limited to 3 MiB. +A single row or schema that cannot fit fails with `RESOURCE_EXHAUSTED`. Each batch +carries a cumulative row offset, and empty results still include an empty batch. + +## Lifetime and trust requirements + +Engine methods and streams must cancel their owned work when dropped. The +frontend cancels them when a client disconnects, an operation is released, a +session is released or server shutdown begins. Results pass through a bounded +channel, so unread responses eventually stop polling the engine. Blocking Arrow +encoding retains operation ownership until its worker finishes. + +`ReleaseAll` waits up to two seconds for teardown and reports `DEADLINE_EXCEEDED` +if it is still draining. Shutdown cancels sessions and waits up to two seconds +for transport drain; returning from `serve_with_engine` does not prove that an +engine's independently scheduled or blocking work has finished. The caller must +honor the engine's cleanup contract before releasing its resources. + +This is an experiment for trusted local clients and trusted engine outputs. +Keep the supplied listener on loopback: this frontend supplies no authentication, +authorization or TLS. Requests can be up to 64 MiB. Active sessions and operations +have no admission limit or idle expiry; only released-session tombstones are +bounded, to the most recent 4096 distinct IDs. An evicted ID can create a new +session. These limits do not provide isolation or complete memory and CPU quotas. +Execution permissions and resource access are the engine's responsibility. + +## Build and test + +Use the workspace Rust toolchain and a `protoc` compiler with proto3 optional +support. From the repository root: + +```bash +cargo test --locked -p spark-connect-server +cargo clippy --locked -p spark-connect-server --all-targets --no-deps -- -D warnings +``` + +The tests run a real gRPC frontend against deterministic injected engines. They +cover wire framing, schema and IPC handling, backend error propagation, +backpressure, operation ownership, cancellation and session lifetime. Fixture +relations select predefined outputs; they do not implement Spark operators or +establish query semantics or Spark server compatibility. diff --git a/crates/spark-connect-server/build.rs b/crates/spark-connect-server/build.rs new file mode 100644 index 00000000..3ffc7807 --- /dev/null +++ b/crates/spark-connect-server/build.rs @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use tonic_build::manual::{Builder, Method, Service}; + +fn main() { + // The official protocol crate already generates messages and client bindings. + // Generate only the standard service routes, using those exact message types. + // Keep this RPC list and its streaming signatures synchronized with + // spark-connect-proto/proto/spark/connect/base.proto on each protocol bump. + let methods = [ + ( + "execute_plan", + "ExecutePlan", + "ExecutePlanRequest", + "ExecutePlanResponse", + false, + true, + ), + ( + "analyze_plan", + "AnalyzePlan", + "AnalyzePlanRequest", + "AnalyzePlanResponse", + false, + false, + ), + ( + "config", + "Config", + "ConfigRequest", + "ConfigResponse", + false, + false, + ), + ( + "add_artifacts", + "AddArtifacts", + "AddArtifactsRequest", + "AddArtifactsResponse", + true, + false, + ), + ( + "artifact_status", + "ArtifactStatus", + "ArtifactStatusesRequest", + "ArtifactStatusesResponse", + false, + false, + ), + ( + "interrupt", + "Interrupt", + "InterruptRequest", + "InterruptResponse", + false, + false, + ), + ( + "reattach_execute", + "ReattachExecute", + "ReattachExecuteRequest", + "ExecutePlanResponse", + false, + true, + ), + ( + "release_execute", + "ReleaseExecute", + "ReleaseExecuteRequest", + "ReleaseExecuteResponse", + false, + false, + ), + ( + "release_session", + "ReleaseSession", + "ReleaseSessionRequest", + "ReleaseSessionResponse", + false, + false, + ), + ( + "fetch_error_details", + "FetchErrorDetails", + "FetchErrorDetailsRequest", + "FetchErrorDetailsResponse", + false, + false, + ), + ( + "clone_session", + "CloneSession", + "CloneSessionRequest", + "CloneSessionResponse", + false, + false, + ), + ( + "get_status", + "GetStatus", + "GetStatusRequest", + "GetStatusResponse", + false, + false, + ), + ]; + let mut service = Service::builder() + .name("SparkConnectService") + .package("spark.connect"); + for (name, route, input, output, client_stream, server_stream) in methods { + let mut method = Method::builder() + .name(name) + .route_name(route) + .input_type(format!("spark_connect_proto::{input}")) + .output_type(format!("spark_connect_proto::{output}")) + .codec_path("tonic_prost::ProstCodec"); + if client_stream { + method = method.client_streaming(); + } + if server_stream { + method = method.server_streaming(); + } + service = service.method(method.build()); + } + Builder::new() + .build_client(false) + .compile(&[service.build()]); + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/crates/spark-connect-server/src/engine/mod.rs b/crates/spark-connect-server/src/engine/mod.rs new file mode 100644 index 00000000..c4a337da --- /dev/null +++ b/crates/spark-connect-server/src/engine/mod.rs @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Native engine boundary for Spark Connect planning and execution. +//! +//! The frontend owns protocol IDs, operation lifetime, framing and backpressure. +//! Engines own their catalog, configuration and native plans. This Rust API is +//! an initial in-process contract for an injected execution engine. + +use std::pin::Pin; +use std::sync::Arc; + +use arrow::datatypes::SchemaRef; +use arrow::record_batch::RecordBatch; +use futures::Stream; +use spark_connect_proto as proto; +use tonic::Status; + +/// Convert an engine Arrow schema into the Spark Connect schema representation. +pub use crate::schema::spark_schema; + +/// Protocol context for a new backend session. +#[derive(Clone, Debug)] +pub struct SessionContext { + pub session_id: String, + /// Client-supplied metadata, not an authenticated identity. + pub user_context: Option, +} + +/// Factory for sessions bound to one native engine. +pub trait Engine: Send + Sync { + /// Allocate local session state without blocking or performing I/O. + /// Defer asynchronous initialization to the session methods. The frontend + /// serializes creation to ensure one backend session per protocol session. + fn create_session(&self, context: SessionContext) -> Result, Status>; +} + +/// Backend-owned catalog, configuration and planning state. +/// +/// Dropping a method future must cancel its work. Dropping the final session +/// reference releases its resources after in-flight operations have finished. +// async_trait repeats Future's must_use annotation on generated trait methods. +#[allow(clippy::double_must_use)] +#[tonic::async_trait] +pub trait EngineSession: Send + Sync { + /// Prepare a query, or perform an eager protocol command. The frontend + /// reserves the operation ID before calling this method. + /// If cancelled, a command must leave its state change fully committed or + /// rolled back, with no partially applied changes. + async fn prepare(&self, operation: proto::plan::OpType) -> Result; + + async fn analyze( + &self, + input: proto::analyze_plan_request::Analyze, + ) -> Result; + + async fn config( + &self, + input: proto::config_request::operation::OpType, + ) -> Result; +} + +/// Prepared native query or command-specific response payloads. +pub enum PreparedOperation { + Query(Box), + /// The frontend adds envelope IDs and the final ResultComplete message. + Command(Vec), +} + +/// Query execution starts after the frontend validates and sends its schema. +// async_trait repeats Future's must_use annotation on generated trait methods. +#[allow(clippy::double_must_use)] +#[tonic::async_trait] +pub trait PreparedQuery: Send { + /// The declared Spark schema. Batches must have the same field names and + /// Spark-visible types; equivalent Arrow representations are accepted. + /// Batch nullability and metadata may differ and remain in each IPC batch, + /// while the protocol schema header uses this declaration. The declared + /// fields must not contain metadata, which Spark schema conversion rejects. + fn schema(&self) -> SchemaRef; + + /// Dropping this future or its returned stream must cancel backend work; + /// implementations must not leave detached execution tasks running. + async fn execute(self: Box) -> Result; +} + +/// Engine output, optionally already serialized for Spark Connect. +pub enum ResultBatch { + Arrow(RecordBatch), + /// A complete IPC stream containing its schema, exactly one record batch, + /// and EOS. The producer must supply that batch's actual nonnegative row count. + Ipc { + data: Vec, + row_count: i64, + }, +} + +impl From for ResultBatch { + fn from(batch: RecordBatch) -> Self { + Self::Arrow(batch) + } +} + +/// Pull-based results: the frontend polls only as client backpressure allows. +pub type BatchStream = Pin> + Send>>; + +/// Engine configuration output without frontend-owned protocol IDs. +#[derive(Default)] +pub struct ConfigResult { + pub pairs: Vec, + pub warnings: Vec, +} diff --git a/crates/spark-connect-server/src/lib.rs b/crates/spark-connect-server/src/lib.rs new file mode 100644 index 00000000..92f19de6 --- /dev/null +++ b/crates/spark-connect-server/src/lib.rs @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod engine; +mod schema; +mod service; + +// tonic's generated async-trait methods duplicate Future's must_use annotation. +#[allow(clippy::double_must_use)] +pub mod grpc { + include!(concat!( + env!("OUT_DIR"), + "/spark.connect.SparkConnectService.rs" + )); +} + +/// Serve with an injected execution engine and a fresh session registry. +/// +/// The frontend binds each protocol session to the engine session created for it. +/// The caller owns the engine and signals shutdown before releasing its resources. +pub async fn serve_with_engine( + listener: tokio::net::TcpListener, + engine: std::sync::Arc, + shutdown: impl std::future::Future + Send + 'static, +) -> Result<(), tonic::transport::Error> { + let service = std::sync::Arc::new(service::ConnectService::new(engine)); + let router = tonic::transport::Server::builder().add_service( + grpc::spark_connect_service_server::SparkConnectServiceServer::from_arc(service.clone()) + .max_decoding_message_size(64 * 1024 * 1024), + ); + let (draining, drain_started) = tokio::sync::oneshot::channel(); + let server = router.serve_with_incoming_shutdown( + tokio_stream::wrappers::TcpListenerStream::new(listener), + async move { + shutdown.await; + service.cancel_all(); + let _ = draining.send(()); + }, + ); + tokio::pin!(server); + tokio::select! { + result = &mut server => result, + _ = drain_started => { + // A peer that stops reading must not prevent local process shutdown. + tokio::time::timeout(std::time::Duration::from_secs(2), &mut server) + .await.unwrap_or(Ok(())) + } + } +} diff --git a/crates/spark-connect-server/src/schema.rs b/crates/spark-connect-server/src/schema.rs new file mode 100644 index 00000000..a6a37a49 --- /dev/null +++ b/crates/spark-connect-server/src/schema.rs @@ -0,0 +1,224 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! The supported Spark schema boundary. Unsupported Arrow types fail explicitly. + +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, FieldRef, Schema, TimeUnit}; +use prost::Message; +use spark_connect_proto as proto; +use tonic::Status; + +pub(crate) const MAX_RESULT_BYTES: usize = 3 * 1024 * 1024; + +pub(crate) fn result_schema(schema: &Schema) -> Result { + let schema = spark_schema(schema)?; + validate_schema_size(&schema)?; + Ok(schema) +} + +pub(crate) fn validate_schema_size(schema: &proto::DataType) -> Result<(), Status> { + if schema.encoded_len() > MAX_RESULT_BYTES { + return Err(Status::resource_exhausted( + "Spark schema exceeds the 3 MiB result limit", + )); + } + Ok(()) +} + +/// Compare Spark-visible fields and types, allowing equivalent Arrow physical +/// representations and differences in nullability or field/schema metadata. +pub(crate) fn comparable_schema(schema: &Schema) -> Result { + fn field(input: &FieldRef) -> FieldRef { + Arc::new(Field::new(input.name(), data_type(input.data_type()), true)) + } + + fn data_type(input: &DataType) -> DataType { + match input { + DataType::List(child) => DataType::List(field(child)), + DataType::LargeList(child) => DataType::LargeList(field(child)), + DataType::Struct(fields) => DataType::Struct(fields.iter().map(field).collect()), + DataType::Map(entries, sorted) => DataType::Map(field(entries), *sorted), + other => other.clone(), + } + } + + let fields: Vec<_> = schema.fields().iter().map(field).collect(); + spark_schema(&Schema::new(fields)) +} + +pub fn spark_schema(schema: &Schema) -> Result { + spark_type(&DataType::Struct(schema.fields().clone())) +} + +fn validate_field_metadata(field: &Field) -> Result<(), Status> { + if !field.metadata().is_empty() { + return Err(Status::unimplemented( + "Arrow field metadata in Spark schemas", + )); + } + Ok(()) +} + +fn spark_field(field: &Field) -> Result { + validate_field_metadata(field)?; + Ok(proto::data_type::StructField { + name: field.name().clone(), + data_type: Some(spark_type(field.data_type())?), + nullable: field.is_nullable(), + metadata: None, + }) +} + +fn spark_type(data_type: &DataType) -> Result { + use proto::data_type::{self as pb, Kind}; + let kind = match data_type { + DataType::Null => Kind::Null(pb::Null::default()), + DataType::Boolean => Kind::Boolean(pb::Boolean::default()), + DataType::Int8 => Kind::Byte(pb::Byte::default()), + DataType::Int16 => Kind::Short(pb::Short::default()), + DataType::Int32 => Kind::Integer(pb::Integer::default()), + DataType::Int64 => Kind::Long(pb::Long::default()), + DataType::Float32 => Kind::Float(pb::Float::default()), + DataType::Float64 => Kind::Double(pb::Double::default()), + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Kind::String(pb::String { + collation: "UTF8_BINARY".into(), + ..Default::default() + }), + DataType::Binary | DataType::LargeBinary | DataType::BinaryView => { + Kind::Binary(pb::Binary::default()) + } + DataType::Date32 => Kind::Date(pb::Date::default()), + DataType::Timestamp(TimeUnit::Microsecond, None) => { + Kind::TimestampNtz(pb::TimestampNtz::default()) + } + DataType::Timestamp(TimeUnit::Microsecond, Some(_)) => { + Kind::Timestamp(pb::Timestamp::default()) + } + DataType::Decimal128(precision, scale) => Kind::Decimal(pb::Decimal { + precision: Some(i32::from(*precision)), + scale: Some(i32::from(*scale)), + ..Default::default() + }), + DataType::List(field) | DataType::LargeList(field) => { + validate_field_metadata(field)?; + Kind::Array(Box::new(pb::Array { + element_type: Some(Box::new(spark_type(field.data_type())?)), + contains_null: field.is_nullable(), + ..Default::default() + })) + } + DataType::Struct(fields) => Kind::Struct(pb::Struct { + fields: fields + .iter() + .map(|field| spark_field(field)) + .collect::>()?, + ..Default::default() + }), + DataType::Map(entries, _) => { + validate_field_metadata(entries)?; + let DataType::Struct(fields) = entries.data_type() else { + return Err(Status::invalid_argument( + "Arrow map entries must be a struct", + )); + }; + if fields.len() != 2 { + return Err(Status::invalid_argument( + "Arrow map entries need key and value fields", + )); + } + validate_field_metadata(&fields[0])?; + validate_field_metadata(&fields[1])?; + Kind::Map(Box::new(pb::Map { + key_type: Some(Box::new(spark_type(fields[0].data_type())?)), + value_type: Some(Box::new(spark_type(fields[1].data_type())?)), + value_contains_null: fields[1].is_nullable(), + ..Default::default() + })) + } + other => { + return Err(Status::unimplemented(format!( + "Arrow result type {other:?}" + ))) + } + }; + Ok(proto::DataType { kind: Some(kind) }) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + fn annotated(name: &str, data_type: DataType, nullable: bool) -> Field { + Field::new(name, data_type, nullable).with_metadata(HashMap::from([( + "producer.note".into(), + "cannot be represented in Spark schemas".into(), + )])) + } + + #[test] + fn declared_metadata_is_rejected_on_list_and_every_map_child() { + let entries = |key: Field, value: Field| { + Field::new("entries", DataType::Struct(vec![key, value].into()), false) + }; + let types = [ + DataType::List(Arc::new(annotated("item", DataType::Int64, true))), + DataType::LargeList(Arc::new(annotated("item", DataType::Int64, true))), + DataType::Map( + Arc::new(annotated( + "entries", + DataType::Struct( + vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Int64, true), + ] + .into(), + ), + false, + )), + false, + ), + DataType::Map( + Arc::new(entries( + annotated("key", DataType::Utf8, false), + Field::new("value", DataType::Int64, true), + )), + false, + ), + DataType::Map( + Arc::new(entries( + Field::new("key", DataType::Utf8, false), + annotated("value", DataType::Int64, true), + )), + false, + ), + ]; + for data_type in types { + let schema = Schema::new(vec![Field::new("nested", data_type, true)]); + assert_eq!( + result_schema(&schema).unwrap_err().code(), + tonic::Code::Unimplemented + ); + // Batch annotations remain acceptable when comparing the declared + // Spark-visible shape, including all nested child fields. + comparable_schema(&schema).unwrap(); + } + } +} diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs new file mode 100644 index 00000000..b4f951da --- /dev/null +++ b/crates/spark-connect-server/src/service.rs @@ -0,0 +1,1232 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::{HashMap, VecDeque}; +use std::panic::AssertUnwindSafe; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; + +use arrow::ipc::{reader::StreamReader, writer::StreamWriter}; +use arrow::record_batch::RecordBatch; +use futures::{FutureExt, Stream, StreamExt}; +use prost::Message; +use spark_connect_proto as proto; +use tokio::sync::{mpsc, watch}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{Request, Response, Status}; +use uuid::Uuid; + +use crate::engine::{Engine, EngineSession, PreparedOperation, ResultBatch, SessionContext}; +use crate::grpc::spark_connect_service_server::SparkConnectService; +use crate::schema::{self, result_schema}; + +type Output = Pin> + Send>>; +type SessionKey = (String, String); +const MAX_ARROW_IPC_BYTES: usize = crate::schema::MAX_RESULT_BYTES; +const MAX_RELEASED_SESSIONS: usize = 4096; +const RELEASE_WAIT: std::time::Duration = std::time::Duration::from_secs(2); + +struct Session { + id: String, + backend: Arc, + cancelled: watch::Sender, + operations: Mutex>>, +} + +struct Operation { + cancelled: watch::Sender, + done: watch::Sender, +} + +impl Session { + fn new(backend: Arc) -> Self { + Self { + id: Uuid::new_v4().to_string(), + backend, + cancelled: watch::channel(false).0, + operations: Mutex::new(HashMap::new()), + } + } + + async fn while_open( + &self, + action: impl std::future::Future> + Send, + ) -> Result { + let mut cancelled = self.cancelled.subscribe(); + if *cancelled.borrow() { + return Err(Status::cancelled("session released")); + } + tokio::select! { + biased; + _ = cancelled.changed() => Err(Status::cancelled("session released")), + result = action => result, + } + } + + fn operation( + self: &Arc, + id: &str, + ) -> Result<(Arc, watch::Receiver), Status> { + // Subscribe before publishing: ReleaseAll can cancel as soon as the + // operation appears in the registry, before preparation is first polled. + let (cancelled, cancellation) = watch::channel(false); + let running = Arc::new(Operation { + cancelled, + done: watch::channel(false).0, + }); + let mut operations = self + .operations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if operations.contains_key(id) { + return Err(Status::already_exists("operation is already running")); + } + operations.insert(id.to_owned(), running.clone()); + Ok(( + Arc::new(OperationGuard { + session: self.clone(), + id: id.to_owned(), + running, + }), + cancellation, + )) + } +} + +// This guard spans preparation and execution, including eager commands. Drop the +// backend future/stream before the guard so ReleaseAll observes completed teardown. +struct OperationGuard { + session: Arc, + id: String, + running: Arc, +} + +impl Drop for OperationGuard { + fn drop(&mut self) { + self.session + .operations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&self.id); + self.running.done.send_replace(true); + } +} + +#[derive(Default)] +struct SessionRegistry { + // None is a recently released-session tombstone, retained until eviction. + entries: HashMap>>, + // Remember the most recent distinct closed IDs in release order. Repeated + // releases do not refresh retention. After eviction, an old ID may create a + // fresh session; active sessions never count toward this finite budget. + released: VecDeque, +} + +impl SessionRegistry { + fn release(&mut self, key: SessionKey, allow_reconnect: bool) -> Option> { + // Releasing a missing or already closed session is a no-op. In + // particular, a later reconnecting release cannot erase a tombstone. + let session = self.entries.get_mut(&key)?.take()?; + if allow_reconnect { + self.entries.remove(&key); + } else { + self.released.push_back(key); + if self.released.len() > MAX_RELEASED_SESSIONS { + if let Some(oldest) = self.released.pop_front() { + self.entries.remove(&oldest); + } + } + } + Some(session) + } +} + +pub(crate) struct ConnectService { + engine: Arc, + sessions: Mutex, + closing: std::sync::atomic::AtomicBool, +} + +impl ConnectService { + pub(crate) fn new(engine: Arc) -> Self { + Self { + engine, + sessions: Mutex::new(SessionRegistry::default()), + closing: std::sync::atomic::AtomicBool::new(false), + } + } + + fn key(id: &str, user: Option<&proto::UserContext>) -> Result { + Uuid::parse_str(id).map_err(|_| Status::invalid_argument("session_id must be a UUID"))?; + Ok(( + user.map(|u| u.user_id.clone()).unwrap_or_default(), + id.into(), + )) + } + + fn session( + &self, + id: &str, + user: Option<&proto::UserContext>, + observed: Option<&str>, + ) -> Result, Status> { + self.find_session(id, user, observed, true) + } + + fn find_session( + &self, + id: &str, + user: Option<&proto::UserContext>, + observed: Option<&str>, + create: bool, + ) -> Result, Status> { + let key = Self::key(id, user)?; + let mut sessions = self + .sessions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.closing.load(std::sync::atomic::Ordering::SeqCst) { + return Err(Status::unavailable("server is shutting down")); + } + let session = match sessions.entries.get(&key) { + Some(Some(session)) => session.clone(), + Some(None) => return Err(Status::failed_precondition("session was released")), + None if !create => return Err(Status::not_found("session does not exist")), + // An observed ID refers to a previous incarnation; a new session + // cannot satisfy it. Reject before allocating backend resources. + None if observed.is_some() => { + return Err(Status::failed_precondition( + "server-side session ID changed", + )); + } + None => { + let backend = self.engine.create_session(SessionContext { + session_id: id.to_owned(), + user_context: user.cloned(), + })?; + let session = Arc::new(Session::new(backend)); + sessions.entries.insert(key, Some(session.clone())); + session + } + }; + if observed.is_some_and(|id| id != session.id) { + return Err(Status::failed_precondition( + "server-side session ID changed", + )); + } + Ok(session) + } + + pub(crate) fn cancel_all(&self) { + self.closing + .store(true, std::sync::atomic::Ordering::SeqCst); + // Stalled HTTP/2 connections can retain the service after the bounded + // transport drain. Remove its registry ownership now so backend sessions + // can close once their cancelled operations finish. + let sessions = std::mem::take( + &mut *self + .sessions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + ); + for session in sessions.entries.into_values().flatten() { + session.cancelled.send_replace(true); + } + } +} + +fn response( + session_id: &str, + session: &Session, + operation_id: &str, + response_type: Option, +) -> proto::ExecutePlanResponse { + proto::ExecutePlanResponse { + session_id: session_id.into(), + server_side_session_id: session.id.clone(), + operation_id: operation_id.into(), + response_id: Uuid::new_v4().to_string(), + response_type, + ..Default::default() + } +} + +fn checked_response( + message: proto::ExecutePlanResponse, +) -> Result { + if message.encoded_len() > MAX_ARROW_IPC_BYTES { + return Err(Status::resource_exhausted( + "ExecutePlan response exceeds the 3 MiB result limit", + )); + } + Ok(message) +} + +fn complete(id: &str, session: &Session, operation: &str) -> proto::ExecutePlanResponse { + response( + id, + session, + operation, + Some(proto::execute_plan_response::ResponseType::ResultComplete( + proto::execute_plan_response::ResultComplete {}, + )), + ) +} + +fn encode_batch(batch: &RecordBatch) -> Result, Status> { + use arrow::datatypes::{DataType, FieldRef, Schema}; + use arrow::record_batch::RecordBatchOptions; + + fn materialized_field(field: &FieldRef) -> FieldRef { + let data_type = materialized_type(field.data_type()); + if data_type == *field.data_type() { + field.clone() + } else { + Arc::new(field.as_ref().clone().with_data_type(data_type)) + } + } + + fn materialized_type(data_type: &DataType) -> DataType { + match data_type { + DataType::Utf8View => DataType::LargeUtf8, + DataType::BinaryView => DataType::LargeBinary, + DataType::List(field) => DataType::List(materialized_field(field)), + DataType::LargeList(field) => DataType::LargeList(materialized_field(field)), + DataType::Struct(fields) => { + DataType::Struct(fields.iter().map(materialized_field).collect()) + } + DataType::Map(field, sorted) => DataType::Map(materialized_field(field), *sorted), + other => other.clone(), + } + } + + // IPC writes all backing buffers of view arrays even after a row slice. Materialize + // their bytes, including nested children, so splitting measures only retained rows. + // Use 64-bit offsets so materializing a batch cannot overflow 32-bit byte offsets + // before the result-size check has a chance to split it. + let source_schema = batch.schema(); + let fields: Vec<_> = source_schema + .fields() + .iter() + .map(materialized_field) + .collect(); + let changed = fields + .iter() + .zip(source_schema.fields()) + .any(|(target, source)| target.data_type() != source.data_type()); + let materialized = if changed { + let columns = batch + .columns() + .iter() + .zip(&fields) + .map(|(column, field)| { + if column.data_type() == field.data_type() { + Ok(column.clone()) + } else { + arrow::compute::cast(column.as_ref(), field.data_type()) + } + }) + .collect::, _>>() + .map_err(|error| Status::internal(error.to_string()))?; + let target_schema = Arc::new(Schema::new_with_metadata( + fields, + source_schema.metadata().clone(), + )); + Some( + RecordBatch::try_new_with_options( + target_schema, + columns, + &RecordBatchOptions::new().with_row_count(Some(batch.num_rows())), + ) + .map_err(|error| Status::internal(error.to_string()))?, + ) + } else { + None + }; + let batch = materialized.as_ref().unwrap_or(batch); + let mut bytes = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut bytes, batch.schema().as_ref()) + .map_err(|e| Status::internal(e.to_string()))?; + writer + .write(batch) + .map_err(|e| Status::internal(e.to_string()))?; + writer + .finish() + .map_err(|e| Status::internal(e.to_string()))?; + } + Ok(bytes) +} + +fn estimated_rows_per_batch(batch: &RecordBatch) -> usize { + let bytes = batch.get_array_memory_size(); + // Start below the exact IPC limit to allow schema/offset overhead. This is + // only an estimate: skewed rows and annotations are checked after encoding. + // Include retained backing allocations so even sliced/view inputs begin + // conservatively without materializing an entire oversized engine batch. + rows_for_bytes(batch.num_rows(), bytes) +} + +fn rows_for_bytes(rows: usize, bytes: usize) -> usize { + (rows as u128 * (MAX_ARROW_IPC_BYTES / 2) as u128 / bytes.max(1) as u128) + .clamp(1, rows.max(1) as u128) as usize +} + +// Fields drop in declaration order, including when encoding panics or a +// cancelled caller drops a completed JoinHandle without observing its output. +struct EncodingBatch { + batch: RecordBatch, + _guard: Arc, +} + +impl EncodingBatch { + fn encode(self) -> Result<(Self, Vec), Status> { + let data = encode_batch(&self.batch)?; + Ok((self, data)) + } +} + +async fn send_batch( + sender: &mpsc::Sender>, + id: &str, + operation: &str, + batch: ResultBatch, + offset: &mut i64, + guard: &Arc, +) -> Result { + let (batch, rows_per_batch) = match batch { + ResultBatch::Arrow(batch) => { + let rows = estimated_rows_per_batch(&batch); + (batch, rows) + } + ResultBatch::Ipc { data, row_count } => { + if row_count < 0 { + return Err(Status::internal("negative engine IPC row count")); + } + let batch = proto::execute_plan_response::ArrowBatch { + row_count, + data: data.into(), + start_offset: Some(*offset), + ..Default::default() + }; + let result = response( + id, + &guard.session, + operation, + Some(proto::execute_plan_response::ResponseType::ArrowBatch( + batch.clone(), + )), + ); + if result.encoded_len() <= MAX_ARROW_IPC_BYTES { + *offset = offset + .checked_add(row_count) + .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; + return Ok(sender.send(Ok(result)).await.is_ok()); + } + drop(result); + // Ordinary IPC is forwarded unchanged. Only oversized payloads need + // vectors so the existing row splitter can enforce the envelope limit. + let guard = guard.clone(); + tokio::task::spawn_blocking(move || { + let _guard = guard; + let ipc_bytes = batch.data.len(); + let mut reader = StreamReader::try_new(&batch.data[..], None) + .map_err(|e| Status::internal(format!("invalid engine IPC: {e}")))?; + let batch = reader + .next() + .transpose() + .map_err(|e| Status::internal(format!("invalid engine IPC: {e}")))? + .ok_or_else(|| Status::internal("engine IPC contains no record batch"))?; + if i64::try_from(batch.num_rows()).ok() != Some(row_count) + || reader.next().is_some() + { + return Err(Status::internal( + "engine IPC batch count differs from its declaration", + )); + } + // Decoded columns can share the full IPC allocation. Use its + // serialized size instead of counting that allocation per column. + let rows = rows_for_bytes(batch.num_rows(), ipc_bytes); + Ok((batch, rows)) + }) + .await + .map_err(|_| Status::internal("Arrow IPC decoding task failed"))?? + } + }; + send_arrow_batch(sender, id, operation, batch, rows_per_batch, offset, guard).await +} + +async fn send_arrow_batch( + sender: &mpsc::Sender>, + id: &str, + operation: &str, + batch: RecordBatch, + rows_per_batch: usize, + offset: &mut i64, + guard: &Arc, +) -> Result { + let mut pending = vec![(batch, rows_per_batch)]; + while let Some((batch, rows_per_batch)) = pending.pop() { + // Keep the remainder as one slice instead of allocating a pending entry + // per row in a large engine batch. + let batch = if batch.num_rows() > rows_per_batch { + pending.push(( + batch.slice(rows_per_batch, batch.num_rows() - rows_per_batch), + rows_per_batch, + )); + batch.slice(0, rows_per_batch) + } else { + batch + }; + // Arrow materialization/IPC encoding is CPU work. Retain operation + // ownership if cancellation drops the JoinHandle, so ReleaseAll cannot + // claim drain while a result batch is still being encoded. + let encoding = EncodingBatch { + batch, + _guard: guard.clone(), + }; + let (encoded, data) = tokio::task::spawn_blocking(move || encoding.encode()) + .await + .map_err(|_| Status::internal("Arrow result encoding task failed"))??; + let batch = &encoded.batch; + let result = response( + id, + &guard.session, + operation, + Some(proto::execute_plan_response::ResponseType::ArrowBatch( + proto::execute_plan_response::ArrowBatch { + row_count: batch.num_rows() as i64, + data: data.into(), + start_offset: Some(*offset), + ..Default::default() + }, + )), + ); + let result_bytes = result.encoded_len(); + if result_bytes > MAX_ARROW_IPC_BYTES { + if batch.num_rows() <= 1 { + return Err(Status::resource_exhausted( + "one Arrow row/schema exceeds the 3 MiB result limit", + )); + } + // Recheck every slice including the envelope: skew and schema + // overhead can exceed the initial memory estimate. + let rows_per_batch = ((batch.num_rows() as u128 * MAX_ARROW_IPC_BYTES as u128 * 9 + / (result_bytes as u128 * 10)) as usize) + .clamp(1, batch.num_rows() - 1); + drop(result); + pending.push((batch.clone(), rows_per_batch)); + continue; + } + *offset = offset + .checked_add(batch.num_rows() as i64) + .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; + if sender.send(Ok(result)).await.is_err() { + return Ok(false); + } + } + Ok(true) +} + +#[tonic::async_trait] +impl SparkConnectService for ConnectService { + type ExecutePlanStream = Output; + type ReattachExecuteStream = Output; + + async fn execute_plan( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let operation = req + .operation_id + .unwrap_or_else(|| Uuid::new_v4().to_string()); + Uuid::parse_str(&operation) + .map_err(|_| Status::invalid_argument("operation_id must be a UUID"))?; + let op = req + .plan + .and_then(|p| p.op_type) + .ok_or_else(|| Status::invalid_argument("missing plan"))?; + let session = self.session( + &req.session_id, + req.user_context.as_ref(), + req.client_observed_server_side_session_id.as_deref(), + )?; + let mut cancelled = session.cancelled.subscribe(); + if *cancelled.borrow() { + return Err(Status::cancelled("session released")); + } + // Reserve before calling the backend: preparing a command can mutate state. + let (guard, mut operation_cancelled) = session.operation(&operation)?; + let prepared = tokio::select! { + biased; + _ = cancelled.changed() => return Err(Status::cancelled("session released")), + _ = operation_cancelled.changed() => return Err(Status::cancelled("operation released")), + prepared = session.backend.prepare(op) => prepared?, + }; + let query = match prepared { + PreparedOperation::Query(query) => query, + PreparedOperation::Command(payloads) => { + let mut messages = payloads + .into_iter() + .map(|payload| { + let message = + response(&req.session_id, &session, &operation, Some(payload)); + checked_response(message) + }) + .collect::, Status>>()?; + messages.push(checked_response(complete( + &req.session_id, + &session, + &operation, + ))?); + return Ok(Response::new(Box::pin(tokio_stream::iter( + messages.into_iter().map(Ok), + )))); + } + }; + let arrow_schema = query.schema(); + let spark_schema = result_schema(&arrow_schema)?; + let expected_batch_schema = schema::comparable_schema(&arrow_schema)?; + let (sender, receiver) = mpsc::channel(2); + tokio::spawn(async move { + let run = async { + let mut header = response(&req.session_id, &session, &operation, None); + header.schema = Some(spark_schema); + let header = checked_response(header)?; + if sender.send(Ok(header)).await.is_err() { + return Ok(()); + } + let mut batches = query.execute().await?; + let mut offset = 0; + let mut sent = false; + while let Some(batch) = batches.next().await { + let batch = batch?; + let batch_schema = match &batch { + ResultBatch::Arrow(batch) => batch.schema(), + ResultBatch::Ipc { data, .. } => { + StreamReader::try_new(data.as_slice(), None) + .map_err(|e| Status::internal(format!("invalid engine IPC: {e}")))? + .schema() + } + }; + let actual_schema = + schema::comparable_schema(batch_schema.as_ref()).map_err(|error| { + Status::internal(format!( + "invalid engine batch schema: {}", + error.message() + )) + })?; + if actual_schema != expected_batch_schema { + return Err(Status::internal( + "engine batch schema differs from its declared schema", + )); + } + if !send_batch( + &sender, + &req.session_id, + &operation, + batch, + &mut offset, + &guard, + ) + .await? + { + return Ok(()); + } + sent = true; + } + // Teardown can itself panic. Complete only after the backend + // stream has been dropped within the panic boundary. + drop(batches); + if !sent { + let batch = RecordBatch::new_empty(arrow_schema); + if !send_batch( + &sender, + &req.session_id, + &operation, + batch.into(), + &mut offset, + &guard, + ) + .await? + { + return Ok(()); + } + } + let _ = sender + .send(Ok(checked_response(complete( + &req.session_id, + &session, + &operation, + ))?)) + .await; + Ok::<_, Status>(()) + }; + let result = AssertUnwindSafe(async { + tokio::select! { + biased; + _ = cancelled.changed() => Err(Status::cancelled("session released")), + _ = operation_cancelled.changed() => Err(Status::cancelled("operation released")), + _ = sender.closed() => Ok(()), + result = run => result, + } + }) + .catch_unwind() + .await + .unwrap_or_else(|_| Err(Status::internal("engine execution panicked"))); + drop(guard); + // A slow peer can keep this terminal send pending after cancellation. + // Backend lifetime must end before waiting for that response slot. + drop(session); + if let Err(error) = result { + let _ = sender.send(Err(error)).await; + } + }); + Ok(Response::new(Box::pin(ReceiverStream::new(receiver)))) + } + + async fn analyze_plan( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let input = req + .analyze + .ok_or_else(|| Status::invalid_argument("missing analysis"))?; + let session = self.session( + &req.session_id, + req.user_context.as_ref(), + req.client_observed_server_side_session_id.as_deref(), + )?; + let result = session.while_open(session.backend.analyze(input)).await?; + if let proto::analyze_plan_response::Result::Schema(result) = &result { + if let Some(schema) = &result.schema { + schema::validate_schema_size(schema)?; + } + } + Ok(Response::new(proto::AnalyzePlanResponse { + session_id: req.session_id, + server_side_session_id: session.id.clone(), + result: Some(result), + })) + } + + async fn config( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let input = req + .operation + .and_then(|operation| operation.op_type) + .ok_or_else(|| Status::invalid_argument("missing config operation"))?; + let session = self.session( + &req.session_id, + req.user_context.as_ref(), + req.client_observed_server_side_session_id.as_deref(), + )?; + let result = session.while_open(session.backend.config(input)).await?; + Ok(Response::new(proto::ConfigResponse { + session_id: req.session_id, + server_side_session_id: session.id.clone(), + pairs: result.pairs, + warnings: result.warnings, + })) + } + + async fn release_execute( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + Uuid::parse_str(&req.operation_id) + .map_err(|_| Status::invalid_argument("operation_id must be a UUID"))?; + let session = self.find_session( + &req.session_id, + req.user_context.as_ref(), + req.client_observed_server_side_session_id.as_deref(), + false, + )?; + // Completed responses are not retained; release is idempotent for absent operations. + let running = session + .operations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&req.operation_id) + .cloned(); + let operation_id = running.as_ref().map(|_| req.operation_id.clone()); + if matches!( + req.release, + Some(proto::release_execute_request::Release::ReleaseAll(_)) + ) { + if let Some(running) = running { + let mut done = running.done.subscribe(); + running.cancelled.send_replace(true); + tokio::time::timeout(RELEASE_WAIT, async { + while !*done.borrow_and_update() { + if done.changed().await.is_err() { + break; + } + } + }) + .await + .map_err(|_| { + Status::deadline_exceeded("operation cancellation is still draining") + })?; + } + } + Ok(Response::new(proto::ReleaseExecuteResponse { + session_id: req.session_id, + server_side_session_id: session.id.clone(), + operation_id, + })) + } + + async fn release_session( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let key = Self::key(&req.session_id, req.user_context.as_ref())?; + let session = { + let mut sessions = self + .sessions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + sessions.release(key, req.allow_reconnect) + }; + let server_id = session.as_ref().map(|s| s.id.clone()).unwrap_or_default(); + if let Some(session) = session { + session.cancelled.send_replace(true); + } + Ok(Response::new(proto::ReleaseSessionResponse { + session_id: req.session_id, + server_side_session_id: server_id, + })) + } + + async fn reattach_execute( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "execution reattachment is not supported", + )) + } + async fn add_artifacts( + &self, + _: Request>, + ) -> Result, Status> { + Err(Status::unimplemented("artifact uploads are not supported")) + } + async fn artifact_status( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("artifact status is not supported")) + } + async fn interrupt( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "operation interruption is not supported", + )) + } + async fn fetch_error_details( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "enriched error details are not supported", + )) + } + async fn clone_session( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "server-side session cloning is not supported", + )) + } + async fn get_status( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("status queries are not supported")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct RegistryEngine; + struct RegistrySession; + + impl Engine for RegistryEngine { + fn create_session( + &self, + _context: SessionContext, + ) -> Result, Status> { + Ok(Arc::new(RegistrySession)) + } + } + + #[tonic::async_trait] + impl EngineSession for RegistrySession { + async fn prepare( + &self, + _operation: proto::plan::OpType, + ) -> Result { + Err(Status::unimplemented( + "registry fixture does not execute queries", + )) + } + + async fn analyze( + &self, + _input: proto::analyze_plan_request::Analyze, + ) -> Result { + Err(Status::unimplemented( + "registry fixture does not analyze queries", + )) + } + + async fn config( + &self, + _input: proto::config_request::operation::OpType, + ) -> Result { + Err(Status::unimplemented( + "registry fixture has no configuration", + )) + } + } + + async fn release(service: &ConnectService, id: &str, allow_reconnect: bool) { + service + .release_session(Request::new(proto::ReleaseSessionRequest { + session_id: id.into(), + allow_reconnect, + ..Default::default() + })) + .await + .unwrap(); + } + + #[test] + fn schema_response_limit_includes_protocol_envelope() { + let session = Session::new(Arc::new(RegistrySession)); + let schema = proto::DataType { + kind: Some(proto::data_type::Kind::Struct(proto::data_type::Struct { + fields: vec![proto::data_type::StructField { + name: "x".repeat(MAX_ARROW_IPC_BYTES - 64), + ..Default::default() + }], + ..Default::default() + })), + }; + assert!(schema.encoded_len() < MAX_ARROW_IPC_BYTES); + let mut header = response( + &Uuid::new_v4().to_string(), + &session, + &Uuid::new_v4().to_string(), + None, + ); + header.schema = Some(schema); + assert_eq!( + checked_response(header).unwrap_err().code(), + tonic::Code::ResourceExhausted + ); + } + + #[test] + fn initial_row_budget_splits_large_batches_before_encoding() { + use arrow::array::Int64Array; + use arrow::datatypes::{DataType, Field, Schema}; + + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Int64, + false, + )])), + vec![Arc::new(Int64Array::from_iter_values(0..1_000_000))], + ) + .unwrap(); + let rows = estimated_rows_per_batch(&batch); + assert!(rows > 0 && rows < batch.num_rows()); + assert!(rows * std::mem::size_of::() <= MAX_ARROW_IPC_BYTES / 2); + assert_eq!( + estimated_rows_per_batch(&RecordBatch::new_empty(batch.schema())), + 1 + ); + } + + #[tokio::test] + async fn existing_ipc_is_forwarded_unchanged() { + use arrow::array::Int64Array; + use arrow::datatypes::{DataType, Field, Schema}; + use arrow::ipc::writer::IpcWriteOptions; + + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), + vec![Arc::new(Int64Array::from_iter_values(0..10))], + ) + .unwrap(); + // Legacy IPC framing differs from the frontend encoder's default, so + // byte equality also proves that the payload was not re-encoded. + let options = IpcWriteOptions::try_new(8, true, arrow::ipc::MetadataVersion::V4).unwrap(); + let mut data = Vec::new(); + let mut writer = + StreamWriter::try_new_with_options(&mut data, batch.schema().as_ref(), options) + .unwrap(); + writer.write(&batch).unwrap(); + writer.finish().unwrap(); + drop(writer); + assert_ne!(data, encode_batch(&batch).unwrap()); + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("ipc").unwrap(); + let (sender, mut receiver) = mpsc::channel(1); + let mut offset = 7; + assert!(send_batch( + &sender, + "session", + "ipc", + ResultBatch::Ipc { + data: data.clone(), + row_count: 10 + }, + &mut offset, + &guard, + ) + .await + .unwrap()); + let result = receiver.recv().await.unwrap().unwrap(); + let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + result.response_type + else { + panic!("expected Arrow response"); + }; + assert_eq!(&batch.data[..], data.as_slice()); + assert_eq!(batch.row_count, 10); + assert_eq!(batch.start_offset, Some(7)); + assert_eq!(offset, 17); + } + + #[tokio::test] + async fn oversized_ipc_is_split_by_rows_and_checks_its_declared_count() { + use arrow::array::Int64Array; + use arrow::datatypes::{DataType, Field, Schema}; + + let batch = RecordBatch::try_new( + Arc::new(Schema::new( + (0..10) + .map(|column| Field::new(format!("id{column}"), DataType::Int64, false)) + .collect::>(), + )), + (0..10) + .map(|column| { + Arc::new(Int64Array::from_iter_values( + (0..50_000).map(|row| row + column * 50_000), + )) as arrow::array::ArrayRef + }) + .collect(), + ) + .unwrap(); + let data = encode_batch(&batch).unwrap(); + assert!(data.len() > MAX_ARROW_IPC_BYTES); + let expected_batches = data.len().div_ceil(MAX_ARROW_IPC_BYTES / 2); + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("ipc").unwrap(); + let (sender, mut receiver) = mpsc::channel(64); + let mut offset = 0; + assert!(send_batch( + &sender, + "session", + "ipc", + ResultBatch::Ipc { + data: data.clone(), + row_count: 50_000 + }, + &mut offset, + &guard, + ) + .await + .unwrap()); + let (mut rows, mut batches) = (0, 0); + while let Ok(result) = receiver.try_recv() { + let result = result.unwrap(); + assert!(result.encoded_len() <= MAX_ARROW_IPC_BYTES); + let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + result.response_type + else { + panic!("expected Arrow response"); + }; + assert_eq!(batch.start_offset, Some(rows)); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + let decoded = reader.next().unwrap().unwrap(); + assert_eq!(decoded.num_columns(), 10); + assert_eq!(decoded.num_rows() as i64, batch.row_count); + for (column, values) in decoded.columns().iter().enumerate() { + let ids = values.as_any().downcast_ref::().unwrap(); + let start = rows + column as i64 * 50_000; + assert!(ids + .values() + .iter() + .copied() + .eq(start..start + batch.row_count)); + } + rows += batch.row_count; + batches += 1; + assert!(reader.next().is_none()); + } + assert_eq!(rows, 50_000); + assert_eq!(offset, rows); + assert!((expected_batches..=expected_batches + 1).contains(&batches)); + for row_count in [-1, 49_999] { + let error = send_batch( + &sender, + "session", + "ipc", + ResultBatch::Ipc { + data: data.clone(), + row_count, + }, + &mut offset, + &guard, + ) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::Internal); + assert_eq!(offset, rows); + } + assert!(receiver.try_recv().is_err()); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn cancelled_encoding_wait_retains_operation_until_blocking_task_finishes() { + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let operation = Uuid::new_v4().to_string(); + let (guard, _) = session.operation(&operation).unwrap(); + let mut done = guard.running.done.subscribe(); + let encoding = EncodingBatch { + batch: RecordBatch::new_empty(Arc::new(arrow::datatypes::Schema::empty())), + _guard: guard.clone(), + }; + let (started, ready) = tokio::sync::oneshot::channel(); + let (release, resume) = std::sync::mpsc::channel(); + let waiting = tokio::spawn(async move { + tokio::task::spawn_blocking(move || { + let _ = started.send(()); + resume + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + encoding.encode() + }) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(5), ready) + .await + .unwrap() + .unwrap(); + waiting.abort(); + let _ = waiting.await; + drop(guard); + assert!(!*done.borrow()); + assert!(session.operations.lock().unwrap().contains_key(&operation)); + release.send(()).unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while !*done.borrow_and_update() { + done.changed().await.unwrap(); + } + }) + .await + .unwrap(); + assert!(!session.operations.lock().unwrap().contains_key(&operation)); + } + + #[tokio::test] + async fn released_session_retention_is_bounded_without_evicting_active_sessions() { + let service = ConnectService::new(Arc::new(RegistryEngine)); + let active_id = Uuid::new_v4().to_string(); + let active = service.session(&active_id, None, None).unwrap(); + + let oldest = Uuid::new_v4().to_string(); + let previous = service.session(&oldest, None, None).unwrap(); + release(&service, &oldest, false).await; + assert!(*previous.cancelled.borrow()); + release(&service, &oldest, false).await; + release(&service, &oldest, true).await; + assert!(matches!(service.session(&oldest, None, None), + Err(status) if status.code() == tonic::Code::FailedPrecondition)); + { + let registry = service.sessions.lock().unwrap(); + assert_eq!(registry.entries.len(), 2); + assert_eq!(registry.released.len(), 1); + } + let mut newest = oldest.clone(); + for _ in 1..MAX_RELEASED_SESSIONS { + newest = Uuid::new_v4().to_string(); + service.session(&newest, None, None).unwrap(); + release(&service, &newest, false).await; + } + release(&service, &oldest, false).await; + assert!(matches!( + service.session(&oldest, None, None), + Err(status) if status.code() == tonic::Code::FailedPrecondition + )); + { + let registry = service.sessions.lock().unwrap(); + assert_eq!(registry.released.len(), MAX_RELEASED_SESSIONS); + assert_eq!(registry.entries.len(), MAX_RELEASED_SESSIONS + 1); + } + + // Unknown releases neither allocate tombstones nor evict closed IDs. + let unknown = Uuid::new_v4().to_string(); + release(&service, &unknown, false).await; + release(&service, &unknown, true).await; + assert!(matches!(service.session(&oldest, None, None), + Err(status) if status.code() == tonic::Code::FailedPrecondition)); + service.session(&unknown, None, None).unwrap(); + release(&service, &unknown, false).await; + assert!(service.session(&oldest, None, None).is_ok()); + assert!(matches!( + service.session(&newest, None, None), + Err(status) if status.code() == tonic::Code::FailedPrecondition + )); + assert!(Arc::ptr_eq( + &active, + &service.session(&active_id, None, None).unwrap() + )); + let registry = service.sessions.lock().unwrap(); + assert_eq!(registry.released.len(), MAX_RELEASED_SESSIONS); + assert_eq!(registry.entries.len(), MAX_RELEASED_SESSIONS + 2); + } + + #[tokio::test] + async fn reconnecting_release_of_live_session_allows_a_new_backend() { + let service = ConnectService::new(Arc::new(RegistryEngine)); + let id = Uuid::new_v4().to_string(); + let previous = service.session(&id, None, None).unwrap(); + release(&service, &id, true).await; + assert!(*previous.cancelled.borrow()); + let current = service.session(&id, None, None).unwrap(); + assert!(!Arc::ptr_eq(¤t, &previous)); + } +} diff --git a/crates/spark-connect-server/tests/engine_injection.rs b/crates/spark-connect-server/tests/engine_injection.rs new file mode 100644 index 00000000..a3a81bd3 --- /dev/null +++ b/crates/spark-connect-server/tests/engine_injection.rs @@ -0,0 +1,1528 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Real gRPC tests of the injected backend boundary. The fake is independent of Spark execution. + +use std::collections::HashMap; +use std::io::Cursor; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::task::{Context, Poll}; +use std::time::Duration; + +use arrow::array::{Int64Array, StringArray}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::ipc::reader::StreamReader; +use arrow::ipc::writer::StreamWriter; +use arrow::record_batch::RecordBatch; +use futures::{Stream, StreamExt}; +use prost::Message; +use proto::execute_plan_response::ResponseType; +use proto::spark_connect_service_client::SparkConnectServiceClient; +use spark_connect_proto as proto; +use spark_connect_server::engine::{ + BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, + ResultBatch, SessionContext, +}; +use tokio::sync::oneshot; +use tokio::task::JoinHandle; +use tonic::transport::Channel; +use tonic::{Code, Status}; +use uuid::Uuid; + +const LIVE: i64 = 1_000_000; +const BLOCKED_PREPARE: i64 = 2_000_000; +const MISMATCHED_BATCH: i64 = 3_000_000; +const ANNOTATED_BATCH: i64 = 4_000_000; +const FLOODED_RESULT: i64 = 5_000_000; +const PANICKING_EXECUTION: i64 = 6_000_000; +const PANICKING_STREAM: i64 = 7_000_000; +const PANICKING_STREAM_DROP: i64 = 8_000_000; +const MISMATCHED_IPC_BATCH: i64 = 9_000_000; +const INVALID_IPC_BATCH: i64 = 10_000_000; + +#[derive(Default)] +struct Spy { + factory_calls: Mutex>, + fail_next: AtomicBool, + sessions_dropped: AtomicUsize, + prepares: AtomicUsize, + commands: AtomicUsize, + analyses: AtomicUsize, + configs: AtomicUsize, + executions: AtomicUsize, + streams_dropped: AtomicUsize, + batches_polled: AtomicUsize, + blocked_prepares_dropped: AtomicUsize, + blocked_analyses_dropped: AtomicUsize, + blocked_configs_dropped: AtomicUsize, + metadata_drop_observations: Mutex>, + slow_drop: Mutex)>>, + stream_drop_gate: Mutex>>, +} + +#[derive(Default)] +struct DropGate { + released: Mutex, + condition: Condvar, + entered: AtomicUsize, + exited: AtomicBool, + timed_out: AtomicBool, +} + +impl DropGate { + fn wait(&self) { + let released = self.released.lock().unwrap(); + self.entered.fetch_add(1, Ordering::SeqCst); + let (released, timeout) = self + .condition + .wait_timeout_while(released, Duration::from_secs(10), |released| !*released) + .unwrap(); + self.timed_out + .store(timeout.timed_out() && !*released, Ordering::SeqCst); + self.exited.store(true, Ordering::SeqCst); + } + + fn release(&self) { + *self.released.lock().unwrap() = true; + self.condition.notify_all(); + } +} + +// Even a failing assertion must never leave a worker blocked in the fake Drop. +struct UnblockDrop(Arc); + +impl Drop for UnblockDrop { + fn drop(&mut self) { + self.0.release(); + } +} + +struct FakeEngine(Arc); + +impl Engine for FakeEngine { + fn create_session(&self, context: SessionContext) -> Result, Status> { + let drop_gate = self + .0 + .slow_drop + .lock() + .unwrap() + .as_ref() + .filter(|(id, _)| id == &context.session_id) + .map(|(_, gate)| gate.clone()); + let user = context + .user_context + .map(|user| user.user_id) + .unwrap_or_default(); + self.0 + .factory_calls + .lock() + .unwrap() + .push((user, context.session_id)); + if self.0.fail_next.swap(false, Ordering::SeqCst) { + return Err(Status::unavailable( + "fake factory is temporarily unavailable", + )); + } + Ok(Arc::new(FakeSession { + spy: self.0.clone(), + configs: Mutex::new(HashMap::new()), + views: Mutex::new(HashMap::new()), + drop_gate, + })) + } +} + +struct FakeSession { + spy: Arc, + configs: Mutex>, + views: Mutex>, + drop_gate: Option>, +} + +impl Drop for FakeSession { + fn drop(&mut self) { + if let Some(gate) = &self.drop_gate { + // Keep Drop synchronous, while allowing Tokio to replace this worker + // so the test measures registry contention rather than timer starvation. + tokio::task::block_in_place(|| gate.wait()); + } + self.spy.metadata_drop_observations.lock().unwrap().push(( + self.spy.blocked_analyses_dropped.load(Ordering::SeqCst), + self.spy.blocked_configs_dropped.load(Ordering::SeqCst), + )); + self.spy.sessions_dropped.fetch_add(1, Ordering::SeqCst); + } +} + +struct BlockedPrepare(Arc); + +impl Drop for BlockedPrepare { + fn drop(&mut self) { + self.0 + .blocked_prepares_dropped + .fetch_add(1, Ordering::SeqCst); + } +} + +enum MetadataCall { + Analysis, + Config, +} + +struct BlockedMetadata { + spy: Arc, + call: MetadataCall, +} + +impl Drop for BlockedMetadata { + fn drop(&mut self) { + let counter = match self.call { + MetadataCall::Analysis => &self.spy.blocked_analyses_dropped, + MetadataCall::Config => &self.spy.blocked_configs_dropped, + }; + counter.fetch_add(1, Ordering::SeqCst); + } +} + +#[tonic::async_trait] +impl EngineSession for FakeSession { + async fn prepare(&self, op: proto::plan::OpType) -> Result { + self.spy.prepares.fetch_add(1, Ordering::SeqCst); + let (value, live, batch_mode) = match op { + proto::plan::OpType::Root(relation) => match relation.rel_type { + Some(proto::relation::RelType::Range(range)) => { + if range.end == BLOCKED_PREPARE { + let _dropped = BlockedPrepare(self.spy.clone()); + std::future::pending::<()>().await; + } + if range.end == -1 { + return Err(Status::invalid_argument("fake rejected the plan")); + } + // Deliberately unlike range semantics: this proves the injected + // backend, rather than the default planner, executed the request. + let batch_mode = match range.end { + MISMATCHED_BATCH => BatchMode::WrongType, + MISMATCHED_IPC_BATCH => BatchMode::WrongTypeIpc, + INVALID_IPC_BATCH => BatchMode::InvalidIpc, + ANNOTATED_BATCH => BatchMode::DifferentAnnotations, + FLOODED_RESULT => BatchMode::Flooded, + PANICKING_EXECUTION => BatchMode::PanicExecute, + PANICKING_STREAM => BatchMode::PanicStream, + PANICKING_STREAM_DROP => BatchMode::PanicDrop, + _ => BatchMode::Declared, + }; + (999, range.end == LIVE, batch_mode) + } + Some(proto::relation::RelType::Read(read)) => { + let Some(proto::read::ReadType::NamedTable(table)) = read.read_type else { + return Err(Status::unimplemented("fake read")); + }; + let value = self + .views + .lock() + .unwrap() + .get(&table.unparsed_identifier) + .copied() + .ok_or_else(|| Status::not_found("fake view not found"))?; + (value, false, BatchMode::Declared) + } + _ => return Err(Status::unimplemented("fake relation")), + }, + proto::plan::OpType::Command(command) => { + self.spy.commands.fetch_add(1, Ordering::SeqCst); + match command.command_type { + Some(proto::command::CommandType::CreateDataframeView(view)) => { + let Some(proto::relation::RelType::Range(input)) = + view.input.and_then(|input| input.rel_type) + else { + return Err(Status::invalid_argument("fake view requires range input")); + }; + let mut views = self.views.lock().unwrap(); + if !view.replace && views.contains_key(&view.name) { + return Err(Status::already_exists("fake view already exists")); + } + views.insert(view.name, input.end); + return Ok(PreparedOperation::Command(Vec::new())); + } + Some(proto::command::CommandType::SqlCommand(command)) => { + return Ok(PreparedOperation::Command(vec![ + ResponseType::SqlCommandResult( + proto::execute_plan_response::SqlCommandResult { + relation: command.input, + }, + ), + ])); + } + _ => return Err(Status::unimplemented("fake command")), + } + } + _ => return Err(Status::unimplemented("fake operation")), + }; + Ok(PreparedOperation::Query(Box::new(FakeQuery { + spy: self.spy.clone(), + value, + live, + batch_mode, + }))) + } + + async fn analyze( + &self, + input: proto::analyze_plan_request::Analyze, + ) -> Result { + self.spy.analyses.fetch_add(1, Ordering::SeqCst); + match input { + proto::analyze_plan_request::Analyze::Schema(_) => { + Ok(proto::analyze_plan_response::Result::Schema( + proto::analyze_plan_response::Schema { + schema: Some(proto::DataType { + kind: Some(proto::data_type::Kind::Struct(proto::data_type::Struct { + fields: vec![proto::data_type::StructField { + name: "x".repeat(3 * 1024 * 1024 + 1), + data_type: Some(proto::DataType { + kind: Some(proto::data_type::Kind::Long( + proto::data_type::Long::default(), + )), + }), + nullable: false, + metadata: None, + }], + ..Default::default() + })), + }), + }, + )) + } + proto::analyze_plan_request::Analyze::IsStreaming(_) => { + let _dropped = BlockedMetadata { + spy: self.spy.clone(), + call: MetadataCall::Analysis, + }; + std::future::pending().await + } + proto::analyze_plan_request::Analyze::IsLocal(_) => { + Ok(proto::analyze_plan_response::Result::IsLocal( + proto::analyze_plan_response::IsLocal { is_local: true }, + )) + } + _ => Err(Status::unimplemented("fake analysis")), + } + } + + async fn config( + &self, + input: proto::config_request::operation::OpType, + ) -> Result { + use proto::config_request::operation::OpType; + self.spy.configs.fetch_add(1, Ordering::SeqCst); + if matches!(&input, OpType::Get(get) if get.keys.iter().any(|key| key == "fake.block")) { + let _dropped = BlockedMetadata { + spy: self.spy.clone(), + call: MetadataCall::Config, + }; + return std::future::pending().await; + } + let mut configs = self.configs.lock().unwrap(); + match input { + OpType::Set(set) => { + for pair in set.pairs { + configs.insert( + pair.key, + pair.value + .ok_or_else(|| Status::invalid_argument("fake config value"))?, + ); + } + Ok(ConfigResult { + pairs: Vec::new(), + warnings: vec!["handled by fake backend".into()], + }) + } + OpType::GetOption(get) => Ok(ConfigResult { + pairs: get + .keys + .into_iter() + .map(|key| { + let value = configs.get(&key).cloned(); + proto::KeyValue { key, value } + }) + .collect(), + warnings: Vec::new(), + }), + _ => Err(Status::unimplemented("fake config operation")), + } + } +} + +struct FakeQuery { + spy: Arc, + value: i64, + live: bool, + batch_mode: BatchMode, +} + +enum BatchMode { + Declared, + WrongType, + WrongTypeIpc, + InvalidIpc, + DifferentAnnotations, + Flooded, + PanicExecute, + PanicStream, + PanicDrop, +} + +#[tonic::async_trait] +impl PreparedQuery for FakeQuery { + fn schema(&self) -> SchemaRef { + Arc::new(Schema::new(vec![Field::new( + "backend_value", + DataType::Int64, + false, + )])) + } + + async fn execute(self: Box) -> Result { + self.spy.executions.fetch_add(1, Ordering::SeqCst); + if matches!(self.batch_mode, BatchMode::PanicExecute) { + panic!("fake query execution panic"); + } + let batch = match self.batch_mode { + BatchMode::Declared + | BatchMode::InvalidIpc + | BatchMode::PanicExecute + | BatchMode::PanicStream + | BatchMode::PanicDrop => RecordBatch::try_new( + self.schema(), + vec![Arc::new(Int64Array::from(vec![self.value]))], + ), + BatchMode::WrongType | BatchMode::WrongTypeIpc => RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "backend_value", + DataType::Utf8, + false, + )])), + vec![Arc::new(StringArray::from(vec!["wrong type"]))], + ), + BatchMode::DifferentAnnotations => RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "backend_value", + DataType::Int64, + true, + ) + .with_metadata(HashMap::from([( + "backend.note".into(), + "ignored annotation".into(), + )]))])), + vec![Arc::new(Int64Array::from(vec![self.value]))], + ), + BatchMode::Flooded => RecordBatch::try_new( + self.schema(), + vec![Arc::new(Int64Array::from(vec![self.value; 64 * 1024]))], + ), + } + .unwrap(); + if matches!(self.batch_mode, BatchMode::Flooded) { + let spy = self.spy.clone(); + return Ok(Box::pin(SpyStream { + inner: Box::pin(futures::stream::repeat_with(move || { + spy.batches_polled.fetch_add(1, Ordering::SeqCst); + Ok(batch.clone().into()) + })), + spy: self.spy.clone(), + panic_on_drop: false, + })); + } + let result = match self.batch_mode { + BatchMode::WrongTypeIpc => { + let mut data = Vec::new(); + { + let mut writer = + StreamWriter::try_new(&mut data, batch.schema().as_ref()).unwrap(); + writer.write(&batch).unwrap(); + writer.finish().unwrap(); + } + ResultBatch::Ipc { data, row_count: 1 } + } + BatchMode::InvalidIpc => ResultBatch::Ipc { + data: b"invalid IPC".to_vec(), + row_count: 1, + }, + _ => batch.into(), + }; + let first = futures::stream::iter(vec![Ok(result)]); + let inner: BatchStream = if matches!(self.batch_mode, BatchMode::PanicStream) { + Box::pin(first.chain(futures::stream::poll_fn(|_| { + panic!("fake stream panic after a partial result"); + }))) + } else if self.live { + Box::pin(first.chain(futures::stream::pending())) + } else { + Box::pin(first) + }; + Ok(Box::pin(SpyStream { + inner, + spy: self.spy.clone(), + panic_on_drop: matches!(self.batch_mode, BatchMode::PanicDrop), + })) + } +} + +struct SpyStream { + inner: BatchStream, + spy: Arc, + panic_on_drop: bool, +} + +impl Stream for SpyStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + self.get_mut().inner.as_mut().poll_next(context) + } +} + +impl Drop for SpyStream { + fn drop(&mut self) { + let gate = self.spy.stream_drop_gate.lock().unwrap().clone(); + if let Some(gate) = gate { + tokio::task::block_in_place(|| gate.wait()); + } + self.spy.streams_dropped.fetch_add(1, Ordering::SeqCst); + assert!(!self.panic_on_drop, "fake stream teardown panic"); + } +} + +struct Server { + address: std::net::SocketAddr, + client: SparkConnectServiceClient, + shutdown: Option>, + task: Option>, +} + +impl Server { + async fn start(spy: Arc) -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (shutdown, stopped) = oneshot::channel(); + let task = tokio::spawn(async move { + spark_connect_server::serve_with_engine(listener, Arc::new(FakeEngine(spy)), async { + let _ = stopped.await; + }) + .await + .unwrap(); + }); + let client = SparkConnectServiceClient::connect(format!("http://{address}")) + .await + .unwrap(); + Self { + address, + client, + shutdown: Some(shutdown), + task: Some(task), + } + } + + async fn stop(mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + tokio::time::timeout(Duration::from_secs(5), self.task.take().unwrap()) + .await + .unwrap() + .unwrap(); + } +} + +impl Drop for Server { + fn drop(&mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + if let Some(task) = self.task.take() { + task.abort(); + } + } +} + +fn user(id: &str) -> Option { + Some(proto::UserContext { + user_id: id.into(), + ..Default::default() + }) +} + +fn relation(end: i64) -> proto::Relation { + proto::Relation { + rel_type: Some(proto::relation::RelType::Range(proto::Range { + end, + step: 1, + ..Default::default() + })), + ..Default::default() + } +} + +fn request(session: &str, end: i64) -> proto::ExecutePlanRequest { + proto::ExecutePlanRequest { + session_id: session.into(), + user_context: user("alice"), + operation_id: Some(Uuid::new_v4().to_string()), + plan: Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(relation(end))), + }), + ..Default::default() + } +} + +fn command_request( + session: &str, + command_type: proto::command::CommandType, +) -> proto::ExecutePlanRequest { + let mut request = request(session, 0); + request.plan = Some(proto::Plan { + op_type: Some(proto::plan::OpType::Command(proto::Command { + command_type: Some(command_type), + })), + }); + request +} + +fn config_request( + session: &str, + input: proto::config_request::operation::OpType, +) -> proto::ConfigRequest { + proto::ConfigRequest { + session_id: session.into(), + user_context: user("alice"), + operation: Some(proto::config_request::Operation { + op_type: Some(input), + }), + ..Default::default() + } +} + +fn get_config(session: &str) -> proto::ConfigRequest { + config_request( + session, + proto::config_request::operation::OpType::GetOption(proto::config_request::GetOption { + keys: vec!["spark.fake.setting".into()], + }), + ) +} + +fn release(session: &str, operation: &str) -> proto::ReleaseExecuteRequest { + proto::ReleaseExecuteRequest { + session_id: session.into(), + user_context: user("alice"), + operation_id: operation.into(), + release: Some(proto::release_execute_request::Release::ReleaseAll( + proto::release_execute_request::ReleaseAll {}, + )), + ..Default::default() + } +} + +fn release_session(session: &str) -> proto::ReleaseSessionRequest { + proto::ReleaseSessionRequest { + session_id: session.into(), + user_context: user("alice"), + ..Default::default() + } +} + +async fn wait_for_count(counter: &AtomicUsize, expected: usize) { + tokio::time::timeout(Duration::from_secs(5), async { + while counter.load(Ordering::SeqCst) < expected { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("backend lifecycle event must arrive"); + assert_eq!(counter.load(Ordering::SeqCst), expected); +} + +async fn collect( + stream: &mut tonic::Streaming, +) -> Vec { + let mut responses = Vec::new(); + while let Some(response) = stream.message().await.unwrap() { + responses.push(response); + } + responses +} + +fn values(responses: &[proto::ExecutePlanResponse]) -> Vec { + let mut values = Vec::new(); + for response in responses { + if let Some(ResponseType::ArrowBatch(batch)) = &response.response_type { + for batch in StreamReader::try_new(Cursor::new(batch.data.clone()), None).unwrap() { + let batch = batch.unwrap(); + let array = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + values.extend((0..array.len()).map(|index| array.value(index))); + } + } + } + values +} + +async fn assert_cancelled(stream: &mut tonic::Streaming) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + match stream.message().await { + Err(status) => { + assert_eq!(status.code(), Code::Cancelled); + break; + } + Ok(Some(response)) => assert!(!matches!( + response.response_type, + Some(ResponseType::ResultComplete(_)) + )), + Ok(None) => panic!("cancelled fake execution must report cancellation"), + } + } + }) + .await + .expect("cancelled fake execution must terminate"); +} + +async fn start_live( + server: &mut Server, + request: proto::ExecutePlanRequest, +) -> tonic::Streaming { + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + assert!(matches!( + stream.message().await.unwrap().unwrap().response_type, + Some(ResponseType::ArrowBatch(_)) + )); + stream +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn injected_backend_owns_query_analysis_config_and_catalog_state() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let mut first = server.client.clone(); + let mut second = server.client.clone(); + let (left, right) = tokio::join!( + first.config(get_config(&session)), + second.config(get_config(&session)) + ); + left.unwrap(); + right.unwrap(); + assert_eq!( + spy.factory_calls.lock().unwrap().len(), + 1, + "concurrent requests must share one backend session" + ); + + let set = config_request( + &session, + proto::config_request::operation::OpType::Set(proto::config_request::Set { + pairs: vec![proto::KeyValue { + key: "spark.fake.setting".into(), + value: Some("backend-owned".into()), + }], + silent: Some(false), + }), + ); + let configured = server.client.config(set).await.unwrap().into_inner(); + assert_eq!(configured.warnings, vec!["handled by fake backend"]); + assert_eq!( + server + .client + .config(get_config(&session)) + .await + .unwrap() + .into_inner() + .pairs[0] + .value + .as_deref(), + Some("backend-owned") + ); + let analysis = server + .client + .analyze_plan(proto::AnalyzePlanRequest { + session_id: session.clone(), + user_context: user("alice"), + analyze: Some(proto::analyze_plan_request::Analyze::IsLocal( + proto::analyze_plan_request::IsLocal { + plan: request(&session, 99).plan, + }, + )), + ..Default::default() + }) + .await + .unwrap() + .into_inner(); + assert!(matches!( + analysis.result, + Some(proto::analyze_plan_response::Result::IsLocal( + proto::analyze_plan_response::IsLocal { is_local: true } + )) + )); + + let mut stream = server + .client + .execute_plan(request(&session, 99)) + .await + .unwrap() + .into_inner(); + let responses = collect(&mut stream).await; + assert_eq!(values(&responses), vec![999]); + assert!(matches!( + responses.last().unwrap().response_type, + Some(ResponseType::ResultComplete(_)) + )); + let create_view = command_request( + &session, + proto::command::CommandType::CreateDataframeView(proto::CreateDataFrameViewCommand { + input: Some(relation(77)), + name: "fake_view".into(), + is_global: false, + replace: false, + }), + ); + let mut created = server + .client + .execute_plan(create_view) + .await + .unwrap() + .into_inner(); + collect(&mut created).await; + let mut read_view = request(&session, 0); + read_view.plan = Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(proto::Relation { + rel_type: Some(proto::relation::RelType::Read(proto::Read { + read_type: Some(proto::read::ReadType::NamedTable(proto::read::NamedTable { + unparsed_identifier: "fake_view".into(), + ..Default::default() + })), + ..Default::default() + })), + ..Default::default() + })), + }); + let mut stream = server + .client + .execute_plan(read_view.clone()) + .await + .unwrap() + .into_inner(); + assert_eq!(values(&collect(&mut stream).await), vec![77]); + let sql = command_request( + &session, + proto::command::CommandType::SqlCommand(proto::SqlCommand { + input: Some(relation(13)), + ..Default::default() + }), + ); + let mut stream = server.client.execute_plan(sql).await.unwrap().into_inner(); + let responses = collect(&mut stream).await; + assert!(matches!( + responses[0].response_type, + Some(ResponseType::SqlCommandResult(_)) + )); + + let mut other_user = get_config(&session); + other_user.user_context = user("bob"); + assert_eq!( + server + .client + .config(other_user) + .await + .unwrap() + .into_inner() + .pairs[0] + .value, + None + ); + read_view.user_context = user("bob"); + assert_eq!( + server + .client + .execute_plan(read_view) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + let other_session = Uuid::new_v4().to_string(); + assert_eq!( + server + .client + .config(get_config(&other_session)) + .await + .unwrap() + .into_inner() + .pairs[0] + .value, + None + ); + assert_eq!( + *spy.factory_calls.lock().unwrap(), + vec![ + ("alice".into(), session.clone()), + ("bob".into(), session), + ("alice".into(), other_session), + ] + ); + assert_eq!(spy.analyses.load(Ordering::SeqCst), 1); + assert_eq!(spy.commands.load(Ordering::SeqCst), 2); + assert_eq!(spy.executions.load(Ordering::SeqCst), 2); + server.stop().await; + wait_for_count(&spy.sessions_dropped, 3).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn invalid_requests_do_not_open_backends_and_factory_failures_can_retry() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + assert_eq!( + server + .client + .execute_plan(request("invalid", 99)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + let mut invalid = request(&session, 99); + invalid.operation_id = Some("invalid".into()); + assert_eq!( + server + .client + .execute_plan(invalid) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_execute(release(&session, &Uuid::new_v4().to_string())) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + assert!(spy.factory_calls.lock().unwrap().is_empty()); + spy.fail_next.store(true, Ordering::SeqCst); + assert_eq!( + server + .client + .config(get_config(&session)) + .await + .unwrap_err() + .code(), + Code::Unavailable + ); + server.client.config(get_config(&session)).await.unwrap(); + assert_eq!(spy.factory_calls.lock().unwrap().len(), 2); + server + .client + .release_session(release_session(&session)) + .await + .unwrap(); + wait_for_count(&spy.sessions_dropped, 1).await; + server + .client + .release_session(release_session(&session)) + .await + .unwrap(); + assert_eq!(spy.factory_calls.lock().unwrap().len(), 2); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn native_streams_drop_on_release_disconnect_session_release_and_shutdown() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let live = request(&session, LIVE); + let operation = live.operation_id.clone().unwrap(); + let mut stream = start_live(&mut server, live.clone()).await; + let mut duplicate = command_request( + &session, + proto::command::CommandType::SqlCommand(proto::SqlCommand { + input: Some(relation(1)), + ..Default::default() + }), + ); + duplicate.operation_id = Some(operation.clone()); + assert_eq!( + server + .client + .execute_plan(duplicate) + .await + .unwrap_err() + .code(), + Code::AlreadyExists + ); + assert_eq!(spy.commands.load(Ordering::SeqCst), 0); + assert_eq!(spy.prepares.load(Ordering::SeqCst), 1); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_eq!( + spy.streams_dropped.load(Ordering::SeqCst), + 1, + "release must wait for backend stream teardown" + ); + assert_cancelled(&mut stream).await; + + let abandoned = start_live(&mut server, live.clone()).await; + drop(abandoned); + wait_for_count(&spy.streams_dropped, 2).await; + let mut restarted = tokio::time::timeout(Duration::from_secs(5), async { + loop { + match server.client.execute_plan(live.clone()).await { + Ok(response) => break response.into_inner(), + Err(status) if status.code() == Code::AlreadyExists => { + tokio::time::sleep(Duration::from_millis(10)).await + } + Err(status) => panic!("unexpected restart error: {status}"), + } + } + }) + .await + .expect("disconnected operation must be removed"); + assert!(restarted.message().await.unwrap().unwrap().schema.is_some()); + assert!(matches!( + restarted.message().await.unwrap().unwrap().response_type, + Some(ResponseType::ArrowBatch(_)) + )); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_cancelled(&mut restarted).await; + wait_for_count(&spy.streams_dropped, 3).await; + + let mut live = start_live(&mut server, request(&session, LIVE)).await; + server + .client + .release_session(release_session(&session)) + .await + .unwrap(); + assert_cancelled(&mut live).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert_eq!( + server + .client + .execute_plan(request(&session, 99)) + .await + .unwrap_err() + .code(), + Code::FailedPrecondition + ); + let other = Uuid::new_v4().to_string(); + let mut live = start_live(&mut server, request(&other, LIVE)).await; + server.shutdown.take().unwrap().send(()).unwrap(); + assert_cancelled(&mut live).await; + server.stop().await; + wait_for_count(&spy.streams_dropped, 5).await; + wait_for_count(&spy.sessions_dropped, 2).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shutdown_releases_backend_session_while_a_result_remains_unread() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy.clone()).await; + let (mut client, connection) = h2::client::Builder::new() + .initial_window_size(1024) + .initial_connection_window_size(1024) + .handshake::<_, tonic::codegen::Bytes>( + tokio::net::TcpStream::connect(server.address) + .await + .unwrap(), + ) + .await + .unwrap(); + let connection = tokio::spawn(async move { + let _ = connection.await; + }); + let request = request(&Uuid::new_v4().to_string(), FLOODED_RESULT).encode_to_vec(); + let mut payload = vec![0]; + payload.extend_from_slice(&(request.len() as u32).to_be_bytes()); + payload.extend_from_slice(&request); + let request = tonic::codegen::http::Request::builder() + .method("POST") + .uri(format!( + "http://{}/spark.connect.SparkConnectService/ExecutePlan", + server.address + )) + .header("content-type", "application/grpc") + .body(()) + .unwrap(); + let (response, mut upload) = client.send_request(request, false).unwrap(); + upload.send_data(payload.into(), true).unwrap(); + let response = tokio::time::timeout(Duration::from_secs(5), response) + .await + .expect("response headers must arrive") + .unwrap(); + // Keep the body and HTTP/2 connection alive without reading or releasing + // capacity. Each 512 KiB batch exceeds the stream's 1 KiB receive window. + tokio::time::timeout(Duration::from_secs(5), async { + while spy.batches_polled.load(Ordering::SeqCst) < 4 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the frontend result queue must fill behind the unread peer"); + assert_eq!(spy.sessions_dropped.load(Ordering::SeqCst), 0); + server.shutdown.take().unwrap().send(()).unwrap(); + // Shutdown must release the session registry and the producer's session + // reference before waiting to send a terminal error through the full queue. + wait_for_count(&spy.streams_dropped, 1).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert!(!response.body().is_end_stream()); + drop(response); + drop(upload); + drop(client); + connection.abort(); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_reservation_precedes_prepare_and_cleans_up_after_cancellation_and_errors() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let blocked = request(&session, BLOCKED_PREPARE); + let operation = blocked.operation_id.clone().unwrap(); + let mut pending_client = server.client.clone(); + let pending = tokio::spawn(async move { pending_client.execute_plan(blocked).await }); + wait_for_count(&spy.prepares, 1).await; + let mut duplicate = command_request( + &session, + proto::command::CommandType::SqlCommand(proto::SqlCommand { + input: Some(relation(1)), + ..Default::default() + }), + ); + duplicate.operation_id = Some(operation.clone()); + assert_eq!( + server + .client + .execute_plan(duplicate) + .await + .unwrap_err() + .code(), + Code::AlreadyExists + ); + assert_eq!(spy.commands.load(Ordering::SeqCst), 0); + assert_eq!(spy.prepares.load(Ordering::SeqCst), 1); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_eq!(spy.blocked_prepares_dropped.load(Ordering::SeqCst), 1); + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), pending) + .await + .unwrap() + .unwrap() + .unwrap_err() + .code(), + Code::Cancelled + ); + + let mut retry = request(&session, 99); + retry.operation_id = Some(operation); + let mut stream = server + .client + .execute_plan(retry) + .await + .unwrap() + .into_inner(); + assert_eq!(values(&collect(&mut stream).await), vec![999]); + let rejected = request(&session, -1); + let mut retry = request(&session, 99); + retry.operation_id.clone_from(&rejected.operation_id); + assert_eq!( + server + .client + .execute_plan(rejected) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + let mut stream = server + .client + .execute_plan(retry) + .await + .unwrap() + .into_inner(); + assert_eq!(values(&collect(&mut stream).await), vec![999]); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn release_and_shutdown_cancel_backend_metadata_futures_before_dropping_sessions() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let mut analysis_client = server.client.clone(); + let analysis_session = session.clone(); + let analysis = tokio::spawn(async move { + analysis_client + .analyze_plan(proto::AnalyzePlanRequest { + session_id: analysis_session.clone(), + user_context: user("alice"), + analyze: Some(proto::analyze_plan_request::Analyze::IsStreaming( + proto::analyze_plan_request::IsStreaming { + plan: request(&analysis_session, 99).plan, + }, + )), + ..Default::default() + }) + .await + }); + wait_for_count(&spy.analyses, 1).await; + server + .client + .release_session(release_session(&session)) + .await + .unwrap(); + let status = tokio::time::timeout(Duration::from_secs(5), analysis) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert_eq!(status.code(), Code::Cancelled); + wait_for_count(&spy.blocked_analyses_dropped, 1).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert_eq!( + *spy.metadata_drop_observations.lock().unwrap(), + vec![(1, 0)] + ); + + let session = Uuid::new_v4().to_string(); + let mut config_client = server.client.clone(); + let config = tokio::spawn(async move { + config_client + .config(config_request( + &session, + proto::config_request::operation::OpType::Get(proto::config_request::Get { + keys: vec!["fake.block".into()], + }), + )) + .await + }); + wait_for_count(&spy.configs, 1).await; + server.shutdown.take().unwrap().send(()).unwrap(); + let status = tokio::time::timeout(Duration::from_secs(5), config) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert_eq!(status.code(), Code::Cancelled); + wait_for_count(&spy.blocked_configs_dropped, 1).await; + server.stop().await; + wait_for_count(&spy.sessions_dropped, 2).await; + assert_eq!( + *spy.metadata_drop_observations.lock().unwrap(), + vec![(1, 0), (1, 1)] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn slow_backend_drop_does_not_hold_the_global_session_registry_lock() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy.clone()).await; + // Channel clones share an HTTP/2 connection. Establish a new TCP connection + // before blocking Drop so the probe isolates the global registry lock. + let mut independent = SparkConnectServiceClient::connect(format!("http://{}", server.address)) + .await + .unwrap(); + let slow_session = Uuid::new_v4().to_string(); + let gate = Arc::new(DropGate::default()); + let unblock = UnblockDrop(gate.clone()); + *spy.slow_drop.lock().unwrap() = Some((slow_session.clone(), gate.clone())); + server + .client + .config(get_config(&slow_session)) + .await + .unwrap(); + let mut release_client = server.client.clone(); + let released = tokio::spawn(async move { + release_client + .release_session(release_session(&slow_session)) + .await + }); + // Store outcomes without asserting while Drop is blocked. Its own ten-second + // deadline and the RAII unblock guard also cover failures in this test. + let entered = tokio::time::timeout(Duration::from_secs(2), async { + while gate.entered.load(Ordering::SeqCst) == 0 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await; + let other = tokio::time::timeout( + Duration::from_secs(1), + independent.config(get_config(&Uuid::new_v4().to_string())), + ) + .await; + let still_blocked = !gate.exited.load(Ordering::SeqCst); + drop(unblock); + let released = tokio::time::timeout(Duration::from_secs(5), released).await; + server.stop().await; + + assert!(entered.is_ok(), "slow Drop must start"); + other + .expect("another session must remain responsive while Drop is blocked") + .unwrap(); + assert!( + still_blocked, + "the independent request must complete before Drop exits" + ); + assert!( + !gate.timed_out.load(Ordering::SeqCst), + "Drop must be explicitly unblocked" + ); + released.unwrap().unwrap().unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn frontend_rejects_oversized_analysis_and_batch_type_mismatches() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let status = server + .client + .analyze_plan(proto::AnalyzePlanRequest { + session_id: session.clone(), + user_context: user("alice"), + analyze: Some(proto::analyze_plan_request::Analyze::Schema( + proto::analyze_plan_request::Schema { + plan: request(&session, 99).plan, + }, + )), + ..Default::default() + }) + .await + .unwrap_err(); + assert_eq!(status.code(), Code::ResourceExhausted); + + let mut mismatch = server + .client + .execute_plan(request(&session, MISMATCHED_BATCH)) + .await + .unwrap() + .into_inner(); + let header = mismatch.message().await.unwrap().unwrap(); + assert!(header.schema.is_some()); + assert!(header.response_type.is_none()); + // The very next event must be the error: no inconsistent Arrow payload or + // successful completion may escape after the declared Int64 schema. + assert_eq!(mismatch.message().await.unwrap_err().code(), Code::Internal); + + let mut compatible = server + .client + .execute_plan(request(&session, ANNOTATED_BATCH)) + .await + .unwrap() + .into_inner(); + let responses = collect(&mut compatible).await; + assert_eq!(values(&responses), vec![999]); + assert!(matches!( + responses.last().unwrap().response_type, + Some(ResponseType::ResultComplete(_)) + )); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn frontend_rejects_mismatched_and_invalid_ipc_before_forwarding() { + let mut server = Server::start(Arc::new(Spy::default())).await; + let session = Uuid::new_v4().to_string(); + for (selector, message) in [ + ( + MISMATCHED_IPC_BATCH, + "engine batch schema differs from its declared schema", + ), + (INVALID_IPC_BATCH, "invalid engine IPC:"), + ] { + let mut stream = server + .client + .execute_plan(request(&session, selector)) + .await + .unwrap() + .into_inner(); + let header = stream.message().await.unwrap().unwrap(); + assert!(header.schema.is_some()); + assert!(header.response_type.is_none()); + // The next event is the schema-validation error, before any IPC payload + // or successful completion can escape the declared schema header. + let status = stream.message().await.unwrap_err(); + assert_eq!(status.code(), Code::Internal); + assert!(status.message().starts_with(message), "{status}"); + } + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn panicking_execution_reports_internal_after_header_or_partial_result() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + for selector in [PANICKING_EXECUTION, PANICKING_STREAM, PANICKING_STREAM_DROP] { + let mut stream = server + .client + .execute_plan(request(&session, selector)) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + if selector != PANICKING_EXECUTION { + assert!(matches!( + stream.message().await.unwrap().unwrap().response_type, + Some(ResponseType::ArrowBatch(_)) + )); + } + let error = tokio::time::timeout(Duration::from_secs(5), stream.message()) + .await + .unwrap() + .unwrap_err(); + assert_eq!(error.code(), Code::Internal); + assert!(error.message().contains("panicked")); + } + let mut recovered = server + .client + .execute_plan(request(&session, 1)) + .await + .unwrap() + .into_inner(); + assert_eq!(values(&collect(&mut recovered).await), vec![999]); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn oversized_command_payload_fails_before_sending_any_success_message() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy).await; + let session = Uuid::new_v4().to_string(); + let command = command_request( + &session, + proto::command::CommandType::SqlCommand(proto::SqlCommand { + input: Some(proto::Relation { + rel_type: Some(proto::relation::RelType::Read(proto::Read { + read_type: Some(proto::read::ReadType::NamedTable(proto::read::NamedTable { + unparsed_identifier: "x".repeat(3 * 1024 * 1024 + 1), + ..Default::default() + })), + ..Default::default() + })), + ..Default::default() + }), + ..Default::default() + }), + ); + assert_eq!( + server + .client + .execute_plan(command) + .await + .unwrap_err() + .code(), + Code::ResourceExhausted + ); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn release_all_timeout_keeps_cancellation_and_operation_ownership_until_drain() { + let spy = Arc::new(Spy::default()); + let gate = Arc::new(DropGate::default()); + *spy.stream_drop_gate.lock().unwrap() = Some(gate.clone()); + let unblock = UnblockDrop(gate.clone()); + let mut server = Server::start(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let original = request(&session, LIVE); + let operation = original.operation_id.clone().unwrap(); + let mut stream = server + .client + .execute_plan(original.clone()) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + assert!(matches!( + stream.message().await.unwrap().unwrap().response_type, + Some(ResponseType::ArrowBatch(_)) + )); + let error = server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap_err(); + assert_eq!(error.code(), Code::DeadlineExceeded); + assert!(!gate.exited.load(Ordering::SeqCst)); + assert_eq!( + server + .client + .execute_plan(original.clone()) + .await + .unwrap_err() + .code(), + Code::AlreadyExists + ); + drop(unblock); + assert_cancelled(&mut stream).await; + assert!(!gate.timed_out.load(Ordering::SeqCst)); + *spy.stream_drop_gate.lock().unwrap() = None; + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + let mut recovered = server + .client + .execute_plan(original) + .await + .unwrap() + .into_inner(); + assert!(recovered.message().await.unwrap().unwrap().schema.is_some()); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_cancelled(&mut recovered).await; + server.stop().await; +} diff --git a/crates/spark-connect-server/tests/protocol.rs b/crates/spark-connect-server/tests/protocol.rs new file mode 100644 index 00000000..6a4ef03c --- /dev/null +++ b/crates/spark-connect-server/tests/protocol.rs @@ -0,0 +1,903 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Verify observable Spark Connect wire behavior independently of DataFrame conveniences. + +use std::collections::HashSet; +use std::io::Cursor; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::Int64Array; +use arrow::ipc::reader::StreamReader; +use prost::Message; +use proto::execute_plan_response::ResponseType; +use proto::spark_connect_service_client::SparkConnectServiceClient; +use spark_connect_proto as proto; +use spark_connect_server::engine::{Engine, EngineSession, SessionContext}; + +mod support; +use support::{FixtureEngine, LIVE_FIXTURE}; +use tokio::sync::oneshot; +use tokio::task::JoinHandle; +use tonic::transport::Channel; +use tonic::Code; +use uuid::Uuid; + +struct Server { + client: SparkConnectServiceClient, + shutdown: Option>, + task: Option>, +} + +impl Server { + async fn start() -> Self { + Self::start_with_engine(Arc::new(FixtureEngine::default())).await + } + + async fn start_with_engine(engine: Arc) -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (shutdown, stopped) = oneshot::channel(); + let task = tokio::spawn(async move { + spark_connect_server::serve_with_engine(listener, engine, async { + let _ = stopped.await; + }) + .await + .unwrap(); + }); + let client = SparkConnectServiceClient::connect(format!("http://{address}")) + .await + .unwrap(); + Self { + client, + shutdown: Some(shutdown), + task: Some(task), + } + } + + async fn stop(mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + tokio::time::timeout(Duration::from_secs(5), self.task.take().unwrap()) + .await + .unwrap() + .unwrap(); + } +} + +#[derive(Default)] +struct CountingEngine { + sessions_created: AtomicUsize, +} + +impl Engine for CountingEngine { + fn create_session( + &self, + context: SessionContext, + ) -> Result, tonic::Status> { + self.sessions_created.fetch_add(1, Ordering::SeqCst); + FixtureEngine::default().create_session(context) + } +} + +impl Drop for Server { + fn drop(&mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + if let Some(task) = self.task.take() { + task.abort(); + } + } +} + +fn range(session: &str, end: i64) -> proto::ExecutePlanRequest { + proto::ExecutePlanRequest { + session_id: session.into(), + user_context: Some(proto::UserContext::default()), + operation_id: Some(Uuid::new_v4().to_string()), + plan: Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(proto::Relation { + rel_type: Some(proto::relation::RelType::Range(proto::Range { + start: Some(0), + end, + step: 1, + num_partitions: None, + })), + ..Default::default() + })), + }), + ..Default::default() + } +} + +fn release(session: &str, operation: &str) -> proto::ReleaseExecuteRequest { + proto::ReleaseExecuteRequest { + session_id: session.into(), + user_context: Some(proto::UserContext::default()), + operation_id: operation.into(), + release: Some(proto::release_execute_request::Release::ReleaseAll( + proto::release_execute_request::ReleaseAll {}, + )), + ..Default::default() + } +} + +fn config_request( + session: &str, + operation: proto::config_request::operation::OpType, +) -> proto::ConfigRequest { + proto::ConfigRequest { + session_id: session.into(), + user_context: Some(proto::UserContext::default()), + operation: Some(proto::config_request::Operation { + op_type: Some(operation), + }), + ..Default::default() + } +} + +async fn assert_cancelled(stream: &mut tonic::Streaming) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + match stream.message().await { + Ok(Some(response)) => assert!(!matches!( + response.response_type, + Some(ResponseType::ResultComplete(_)) + )), + Ok(None) => panic!("cancelled execution must report cancellation"), + Err(status) => { + assert_eq!(status.code(), Code::Cancelled); + break; + } + } + } + }) + .await + .expect("cancelled stream must terminate"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn arrow_stream_preserves_schema_ids_offsets_and_empty_results() { + let mut server = Server::start().await; + let session = Uuid::new_v4().to_string(); + let mut server_session = None; + for end in [0, 25_000] { + let request = range(&session, end); + let operation = request.operation_id.clone().unwrap(); + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + let mut response_ids = HashSet::new(); + let (mut rows, mut batches, mut schemas, mut complete) = (0, 0, 0, false); + while let Some(response) = stream.message().await.unwrap() { + assert!(!complete, "ResultComplete must be the last response"); + assert_eq!(response.session_id, session); + assert_eq!(response.operation_id, operation); + Uuid::parse_str(&response.response_id).unwrap(); + assert!(response_ids.insert(response.response_id)); + Uuid::parse_str(&response.server_side_session_id).unwrap(); + let stable = server_session.get_or_insert(response.server_side_session_id.clone()); + assert_eq!(*stable, response.server_side_session_id); + if let Some(schema) = response.schema { + let Some(proto::data_type::Kind::Struct(schema)) = schema.kind else { + panic!("struct schema"); + }; + assert_eq!(schema.fields.len(), 1); + assert_eq!(schema.fields[0].name, "id"); + assert!(!schema.fields[0].nullable); + assert!(matches!( + schema.fields[0].data_type.as_ref().unwrap().kind, + Some(proto::data_type::Kind::Long(_)) + )); + schemas += 1; + } + match response.response_type { + Some(ResponseType::ArrowBatch(batch)) => { + assert_eq!(batch.start_offset, Some(rows)); + let reader = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); + let mut batch_rows = 0; + for record in reader { + let record = record.unwrap(); + let values = record + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for index in 0..values.len() { + assert_eq!(values.value(index), rows + batch_rows + index as i64); + } + batch_rows += record.num_rows() as i64; + } + assert_eq!(batch_rows, batch.row_count); + rows += batch_rows; + batches += 1; + } + Some(ResponseType::ResultComplete(_)) => complete = true, + None => {} + other => panic!("unexpected result: {other:?}"), + } + } + assert_eq!(rows, end); + assert_eq!(schemas, 1); + assert!(complete); + assert!(batches >= 1, "empty execution must still return Arrow data"); + if end > 0 { + // The fixture supplies multiple physical batches; all must reach the wire. + assert!(batches > 1, "large result should stream multiple batches"); + } + } + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn observed_session_mismatch_and_released_sessions_are_rejected() { + let mut server = Server::start().await; + let session = Uuid::new_v4().to_string(); + let mut first = server + .client + .execute_plan(range(&session, 0)) + .await + .unwrap() + .into_inner(); + let original = first + .message() + .await + .unwrap() + .unwrap() + .server_side_session_id; + while first.message().await.unwrap().is_some() {} + let mut mismatch = range(&session, 1); + mismatch.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); + let error = server.client.execute_plan(mismatch).await.unwrap_err(); + assert_eq!(error.code(), Code::FailedPrecondition); + let release = proto::ReleaseSessionRequest { + session_id: session.clone(), + user_context: Some(proto::UserContext::default()), + ..Default::default() + }; + let released = server + .client + .release_session(release.clone()) + .await + .unwrap() + .into_inner(); + assert_eq!(released.server_side_session_id, original); + server.client.release_session(release).await.unwrap(); + let error = server + .client + .execute_plan(range(&session, 1)) + .await + .unwrap_err(); + assert_eq!(error.code(), Code::FailedPrecondition); + // A separate session on the same transport remains usable. + let mut other = server + .client + .execute_plan(range(&Uuid::new_v4().to_string(), 0)) + .await + .unwrap() + .into_inner(); + while other.message().await.unwrap().is_some() {} + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn release_all_cancels_a_pending_backend_stream_and_waits_for_teardown() { + let engine = Arc::new(FixtureEngine::default()); + let mut server = Server::start_with_engine(engine.clone()).await; + let session = Uuid::new_v4().to_string(); + let request = range(&session, LIVE_FIXTURE); + let operation = request.operation_id.clone().unwrap(); + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + let header = stream.message().await.unwrap().unwrap(); + assert!(header.schema.is_some()); + let batch = stream.message().await.unwrap().unwrap(); + assert!(matches!( + batch.response_type, + Some(ResponseType::ArrowBatch(_)) + )); + let release = proto::ReleaseExecuteRequest { + session_id: session.clone(), + user_context: Some(proto::UserContext::default()), + operation_id: operation.clone(), + client_observed_server_side_session_id: Some(header.server_side_session_id), + release: Some(proto::release_execute_request::Release::ReleaseAll( + proto::release_execute_request::ReleaseAll {}, + )), + ..Default::default() + }; + let released = tokio::time::timeout( + Duration::from_secs(3), + server.client.release_execute(release.clone()), + ) + .await + .expect("ReleaseAll must finish without consuming the result") + .unwrap() + .into_inner(); + assert_eq!(released.operation_id.as_deref(), Some(operation.as_str())); + assert_eq!( + engine.streams_dropped.load(Ordering::SeqCst), + 1, + "ReleaseAll must wait until the backend stream is dropped" + ); + let absent = server + .client + .release_execute(release) + .await + .unwrap() + .into_inner(); + assert_eq!( + absent.operation_id, None, + "completed teardown must remove the operation" + ); + assert_cancelled(&mut stream).await; + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn arrow_payloads_are_byte_limited_and_oversized_rows_fail() { + use arrow::array::{LargeStringArray, StringArray, StringViewArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use arrow::record_batch::RecordBatch; + + let session = Uuid::new_v4().to_string(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("s", DataType::Utf8, false), + ])); + // Skewed string sizes make initial byte-ratio slices too large, forcing + // refinement while preserving every value and row offset. + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from_iter_values(0..8192)), + Arc::new(StringArray::from_iter_values( + (0..8192).map(|id| "x".repeat(if id % 64 == 0 { 200000 } else { 1000 })), + )), + ], + ) + .unwrap(); + let oversized_schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)])); + let oversized = RecordBatch::try_new( + oversized_schema.clone(), + vec![Arc::new(StringArray::from(vec!["x".repeat(4000000)]))], + ) + .unwrap(); + let engine = FixtureEngine::default() + .with_batches(2, schema, vec![batch]) + .with_batches(3, oversized_schema, vec![oversized]); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let mut stream = server + .client + .execute_plan(range(&session, 2)) + .await + .unwrap() + .into_inner(); + let (mut rows, mut batches, mut complete) = (0, 0, false); + while let Some(response) = stream.message().await.unwrap() { + assert!(response.encoded_len() <= 3 * 1024 * 1024); + match response.response_type { + Some(ResponseType::ArrowBatch(batch)) => { + assert_eq!(batch.start_offset, Some(rows)); + let records = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); + let mut decoded = 0; + for record in records { + let record = record.unwrap(); + let ids = record + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let strings = record.column(1); + for index in 0..record.num_rows() { + let id = rows + decoded + index as i64; + assert_eq!(ids.value(index), id); + let value = match strings.data_type() { + arrow::datatypes::DataType::Utf8 => strings + .as_any() + .downcast_ref::() + .unwrap() + .value(index), + arrow::datatypes::DataType::LargeUtf8 => strings + .as_any() + .downcast_ref::() + .unwrap() + .value(index), + arrow::datatypes::DataType::Utf8View => strings + .as_any() + .downcast_ref::() + .unwrap() + .value(index), + other => panic!("unexpected string type: {other:?}"), + }; + assert_eq!(value.len(), if id % 64 == 0 { 200000 } else { 1000 }); + assert!(value.bytes().all(|byte| byte == b'x')); + } + decoded += record.num_rows() as i64; + } + assert_eq!(decoded, batch.row_count); + rows += decoded; + batches += 1; + } + Some(ResponseType::ResultComplete(_)) => complete = true, + None => {} + other => panic!("unexpected result: {other:?}"), + } + } + assert_eq!(rows, 8192); + assert!(batches > 1); + assert!(complete); + + let mut oversized = server + .client + .execute_plan(range(&session, 3)) + .await + .unwrap() + .into_inner(); + assert!(oversized.message().await.unwrap().unwrap().schema.is_some()); + let status = oversized.message().await.unwrap_err(); + assert_eq!(status.code(), Code::ResourceExhausted); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn release_session_cancels_live_stream_and_live_operation_ids_are_unique() { + let mut server = Server::start().await; + let session = Uuid::new_v4().to_string(); + let request = range(&session, LIVE_FIXTURE); + let mut stream = server + .client + .execute_plan(request.clone()) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + let duplicate = server.client.execute_plan(request).await.unwrap_err(); + assert_eq!(duplicate.code(), Code::AlreadyExists); + server + .client + .release_session(proto::ReleaseSessionRequest { + session_id: session.clone(), + user_context: Some(proto::UserContext::default()), + ..Default::default() + }) + .await + .unwrap(); + assert_cancelled(&mut stream).await; + assert_eq!( + server + .client + .execute_plan(range(&session, 0)) + .await + .unwrap_err() + .code(), + Code::FailedPrecondition + ); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn invalid_requests_do_not_allocate_sessions_and_unknown_release_is_rejected() { + let engine = Arc::new(CountingEngine::default()); + let mut server = Server::start_with_engine(engine.clone()).await; + let valid = Uuid::new_v4().to_string(); + assert_eq!( + server + .client + .execute_plan(range("not-a-uuid", 0)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + let mut invalid_operation = range(&valid, 0); + invalid_operation.operation_id = Some("not-a-uuid".into()); + assert_eq!( + server + .client + .execute_plan(invalid_operation) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + for plan in [None, Some(proto::Plan { op_type: None })] { + let mut request = range(&valid, 0); + request.plan = plan; + assert_eq!( + server + .client + .execute_plan(request) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + } + assert_eq!( + server + .client + .analyze_plan(proto::AnalyzePlanRequest { + session_id: valid.clone(), + ..Default::default() + }) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + for operation in [ + None, + Some(proto::config_request::Operation { op_type: None }), + ] { + assert_eq!( + server + .client + .config(proto::ConfigRequest { + session_id: valid.clone(), + operation, + ..Default::default() + }) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + } + // A stale observed ID cannot match a session that does not exist. Repeated + // requests must not consume backend session capacity while being rejected. + for _ in 0..3 { + let mut stale = range(&valid, 0); + stale.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); + assert_eq!( + server.client.execute_plan(stale).await.unwrap_err().code(), + Code::FailedPrecondition + ); + } + let mut stale = config_request( + &valid, + proto::config_request::operation::OpType::GetOption(proto::config_request::GetOption { + keys: vec!["missing".into()], + }), + ); + stale.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); + assert_eq!( + server.client.config(stale).await.unwrap_err().code(), + Code::FailedPrecondition + ); + // Identifier and envelope validation must precede backend session allocation. + assert_eq!(engine.sessions_created.load(Ordering::SeqCst), 0); + assert_eq!( + server + .client + .release_execute(release(&valid, &Uuid::new_v4().to_string())) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + assert_eq!( + server + .client + .release_execute(release("not-a-uuid", &valid)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_execute(release(&valid, "not-a-uuid")) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + let unknown = Uuid::new_v4().to_string(); + assert_eq!( + server + .client + .release_execute(release(&unknown, &valid)) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + assert_eq!(engine.sessions_created.load(Ordering::SeqCst), 0); + server + .client + .config(config_request( + &valid, + proto::config_request::operation::OpType::GetOption(proto::config_request::GetOption { + keys: vec!["spark.sql.session.timeZone".into()], + }), + )) + .await + .unwrap(); + assert_eq!(engine.sessions_created.load(Ordering::SeqCst), 1); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn disconnect_removes_operation_and_shutdown_cancels_live_streams() { + let mut server = Server::start().await; + let session = Uuid::new_v4().to_string(); + let request = range(&session, LIVE_FIXTURE); + let operation = request.operation_id.clone().unwrap(); + let mut abandoned = server + .client + .execute_plan(request.clone()) + .await + .unwrap() + .into_inner(); + assert!(abandoned.message().await.unwrap().unwrap().schema.is_some()); + drop(abandoned); + let mut resumed = tokio::time::timeout(Duration::from_secs(5), async { + loop { + match server.client.execute_plan(request.clone()).await { + Ok(response) => break response.into_inner(), + Err(status) if status.code() == Code::AlreadyExists => { + tokio::time::sleep(Duration::from_millis(20)).await; + } + Err(status) => panic!("unexpected retry failure: {status}"), + } + } + }) + .await + .expect("disconnect must remove the operation registry entry"); + assert!(resumed.message().await.unwrap().unwrap().schema.is_some()); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_cancelled(&mut resumed).await; + + let mut live = server + .client + .execute_plan(range(&session, LIVE_FIXTURE)) + .await + .unwrap() + .into_inner(); + assert!(live.message().await.unwrap().unwrap().schema.is_some()); + server.shutdown.take().unwrap().send(()).unwrap(); + assert_cancelled(&mut live).await; + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn backend_status_codes_survive_prepare_and_stream_failures() { + let engine = FixtureEngine::default() + .reject(10, Code::Unimplemented) + .reject(11, Code::InvalidArgument) + .stream_error(12, Code::ResourceExhausted); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let session = Uuid::new_v4().to_string(); + for (selector, expected) in [(10, Code::Unimplemented), (11, Code::InvalidArgument)] { + let status = server + .client + .execute_plan(range(&session, selector)) + .await + .unwrap_err(); + assert_eq!(status.code(), expected); + } + let mut stream = server + .client + .execute_plan(range(&session, 12)) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + assert_eq!( + stream.message().await.unwrap_err().code(), + Code::ResourceExhausted + ); + // Backend failures must not poison the protocol session or its transport. + let mut recovered = server + .client + .execute_plan(range(&session, 1)) + .await + .unwrap() + .into_inner(); + let (mut rows, mut complete) = (0, false); + while let Some(response) = recovered.message().await.unwrap() { + match response.response_type { + Some(ResponseType::ArrowBatch(batch)) => rows += batch.row_count, + Some(ResponseType::ResultComplete(_)) => complete = true, + None => {} + other => panic!("unexpected result: {other:?}"), + } + } + assert_eq!(rows, 1); + assert!(complete); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn deeply_nested_requests_hit_the_standard_protobuf_decoder_limit() { + let mut server = Server::start().await; + let session = Uuid::new_v4().to_string(); + let mut request = range(&session, 1); + let Some(proto::plan::OpType::Root(mut relation)) = request.plan.take().unwrap().op_type else { + unreachable!() + }; + for _ in 0..60 { + relation = proto::Relation { + rel_type: Some(proto::relation::RelType::Filter(Box::new(proto::Filter { + input: Some(Box::new(relation)), + condition: Some(proto::Expression { + expr_type: Some(proto::expression::ExprType::Literal( + proto::expression::Literal { + literal_type: Some(proto::expression::literal::LiteralType::Boolean( + true, + )), + ..Default::default() + }, + )), + ..Default::default() + }), + }))), + ..Default::default() + }; + } + request.plan = Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(relation)), + }); + let status = server.client.execute_plan(request).await.unwrap_err(); + // Each filter introduces nested protobuf messages. The normal decoder rejects + // this request before the backend can inspect it. + assert_eq!(status.code(), Code::Internal); + assert!(status.message().contains("recursion"), "{status}"); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_backing_buffers_are_compacted_before_splitting_including_struct_children() { + use std::sync::Arc; + + use arrow::array::{BinaryArray, BinaryViewArray, StringArray, StringViewArray, StructArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use arrow::record_batch::RecordBatch; + + let session = Uuid::new_v4().to_string(); + let text = |id: i64| format!("{id:04}{}", "x".repeat(1020)); + let nested_text = |id: i64| format!("n{id:04}{}", "y".repeat(1019)); + let binary = |id: i64| { + let mut bytes = id.to_le_bytes().to_vec(); + bytes.extend(std::iter::repeat_n((id % 251) as u8, 1016)); + bytes + }; + let strings = StringViewArray::from_iter_values((0..4096).map(text)); + let binaries = BinaryViewArray::from_iter_values((0..4096).map(binary)); + let nested_strings = StringViewArray::from_iter_values((0..4096).map(nested_text)); + // A one-row slice of these view arrays still references backing buffers + // larger than the response limit, although each individual value is 1 KiB. + for size in [ + strings + .data_buffers() + .iter() + .map(|buffer| buffer.len()) + .sum::(), + binaries + .data_buffers() + .iter() + .map(|buffer| buffer.len()) + .sum::(), + nested_strings + .data_buffers() + .iter() + .map(|buffer| buffer.len()) + .sum::(), + ] { + assert!(size > 3 * 1024 * 1024); + } + let nested_fields: arrow::datatypes::Fields = + vec![Field::new("text", DataType::Utf8View, false)].into(); + let nested = StructArray::new(nested_fields.clone(), vec![Arc::new(nested_strings)], None); + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8View, false), + Field::new("binary", DataType::BinaryView, false), + Field::new("nested", DataType::Struct(nested_fields), false), + ])), + vec![ + Arc::new(Int64Array::from_iter_values(0..4096)), + Arc::new(strings), + Arc::new(binaries), + Arc::new(nested), + ], + ) + .unwrap(); + let engine = FixtureEngine::default().with_batches(42, batch.schema(), vec![batch]); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let request = range(&session, 42); + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + let (mut rows, mut batches, mut complete) = (0, 0, false); + while let Some(response) = stream.message().await.unwrap() { + assert!(response.encoded_len() <= 3 * 1024 * 1024); + match response.response_type { + Some(ResponseType::ArrowBatch(batch)) => { + assert_eq!(batch.start_offset, Some(rows)); + let records = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); + let mut decoded = 0; + for record in records { + let record = record.unwrap(); + let ids = record + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(record.column(1).data_type(), &DataType::LargeUtf8); + assert_eq!(record.column(2).data_type(), &DataType::LargeBinary); + let strings = arrow::compute::cast(record.column(1), &DataType::Utf8).unwrap(); + let strings = strings.as_any().downcast_ref::().unwrap(); + let binaries = + arrow::compute::cast(record.column(2), &DataType::Binary).unwrap(); + let binaries = binaries.as_any().downcast_ref::().unwrap(); + let nested = record + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(nested.column(0).data_type(), &DataType::LargeUtf8); + let nested_strings = + arrow::compute::cast(nested.column(0), &DataType::Utf8).unwrap(); + let nested_strings = nested_strings + .as_any() + .downcast_ref::() + .unwrap(); + for index in 0..record.num_rows() { + let id = rows + decoded + index as i64; + assert_eq!(ids.value(index), id); + assert_eq!(strings.value(index), text(id)); + assert_eq!(binaries.value(index), binary(id)); + assert_eq!(nested_strings.value(index), nested_text(id)); + } + decoded += record.num_rows() as i64; + } + assert_eq!(decoded, batch.row_count); + rows += decoded; + batches += 1; + } + Some(ResponseType::ResultComplete(_)) => complete = true, + None => {} + other => panic!("unexpected result: {other:?}"), + } + } + assert_eq!(rows, 4096); + assert!(batches > 1); + assert!(complete); + server.stop().await; +} diff --git a/crates/spark-connect-server/tests/support/mod.rs b/crates/spark-connect-server/tests/support/mod.rs new file mode 100644 index 00000000..0df3a439 --- /dev/null +++ b/crates/spark-connect-server/tests/support/mod.rs @@ -0,0 +1,265 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Explicit backend outputs for frontend protocol tests. +//! +//! Range.end is only a fixture lookup key. This backend neither interprets SQL +//! nor implements Spark operators; each case supplies Arrow batches, a pending +//! stream, or a chosen failure. These tests do not verify query execution semantics. + +use std::collections::HashMap; +use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use arrow::array::Int64Array; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::record_batch::RecordBatch; +use futures::{Stream, StreamExt}; +use spark_connect_proto as proto; +use spark_connect_server::engine::{ + BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, + ResultBatch, SessionContext, +}; +use tonic::{Code, Status}; + +pub const LIVE_FIXTURE: i64 = 1_000_000_000_000; + +#[derive(Clone)] +enum Ending { + Complete, + Pending, + Error(Code), +} + +#[derive(Clone)] +enum Output { + Query { + schema: SchemaRef, + batches: Vec, + ending: Ending, + }, + Rejected(Code), +} + +pub struct FixtureEngine { + outputs: HashMap, + pub streams_dropped: Arc, +} + +impl Default for FixtureEngine { + fn default() -> Self { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let batch = |start, end| { + RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int64Array::from_iter_values(start..end))], + ) + .unwrap() + }; + let mut engine = Self { + outputs: HashMap::new(), + streams_dropped: Arc::new(AtomicUsize::new(0)), + }; + engine.outputs.insert( + 0, + Output::Query { + schema: schema.clone(), + batches: Vec::new(), + ending: Ending::Complete, + }, + ); + engine.outputs.insert( + 1, + Output::Query { + schema: schema.clone(), + batches: vec![batch(0, 1)], + ending: Ending::Complete, + }, + ); + engine.outputs.insert( + 25_000, + Output::Query { + schema: schema.clone(), + batches: vec![batch(0, 8192), batch(8192, 16_384), batch(16_384, 25_000)], + ending: Ending::Complete, + }, + ); + engine.outputs.insert( + LIVE_FIXTURE, + Output::Query { + schema: schema.clone(), + batches: vec![batch(0, 8192)], + ending: Ending::Pending, + }, + ); + engine + } +} + +impl FixtureEngine { + pub fn with_batches( + mut self, + selector: i64, + schema: SchemaRef, + batches: Vec, + ) -> Self { + self.outputs.insert( + selector, + Output::Query { + schema, + batches, + ending: Ending::Complete, + }, + ); + self + } + + pub fn reject(mut self, selector: i64, code: Code) -> Self { + self.outputs.insert(selector, Output::Rejected(code)); + self + } + + pub fn stream_error(mut self, selector: i64, code: Code) -> Self { + self.outputs.insert( + selector, + Output::Query { + schema: Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), + batches: Vec::new(), + ending: Ending::Error(code), + }, + ); + self + } +} + +impl Engine for FixtureEngine { + fn create_session(&self, _context: SessionContext) -> Result, Status> { + Ok(Arc::new(FixtureSession { + outputs: self.outputs.clone(), + streams_dropped: self.streams_dropped.clone(), + })) + } +} + +struct FixtureSession { + outputs: HashMap, + streams_dropped: Arc, +} + +#[tonic::async_trait] +impl EngineSession for FixtureSession { + async fn prepare(&self, operation: proto::plan::OpType) -> Result { + let proto::plan::OpType::Root(relation) = operation else { + return Err(Status::unimplemented("fixture accepts root selectors only")); + }; + let Some(proto::relation::RelType::Range(selector)) = relation.rel_type else { + return Err(Status::unimplemented("fixture requires a range selector")); + }; + match self.outputs.get(&selector.end) { + Some(Output::Query { + schema, + batches, + ending, + }) => Ok(PreparedOperation::Query(Box::new(FixtureQuery { + schema: schema.clone(), + batches: batches.clone(), + ending: ending.clone(), + streams_dropped: self.streams_dropped.clone(), + }))), + Some(Output::Rejected(code)) => Err(Status::new(*code, "prepared fixture error")), + None => Err(Status::unimplemented("unknown fixture selector")), + } + } + + async fn analyze( + &self, + _input: proto::analyze_plan_request::Analyze, + ) -> Result { + Err(Status::unimplemented( + "fixture does not analyze query semantics", + )) + } + + async fn config( + &self, + input: proto::config_request::operation::OpType, + ) -> Result { + let proto::config_request::operation::OpType::GetOption(input) = input else { + return Err(Status::unimplemented( + "fixture supports optional config lookup only", + )); + }; + Ok(ConfigResult { + pairs: input + .keys + .into_iter() + .map(|key| proto::KeyValue { key, value: None }) + .collect(), + warnings: Vec::new(), + }) + } +} + +struct FixtureQuery { + schema: SchemaRef, + batches: Vec, + ending: Ending, + streams_dropped: Arc, +} + +#[tonic::async_trait] +impl PreparedQuery for FixtureQuery { + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + async fn execute(self: Box) -> Result { + let first = futures::stream::iter(self.batches.into_iter().map(|batch| Ok(batch.into()))); + let inner: BatchStream = match self.ending { + Ending::Complete => Box::pin(first), + Ending::Pending => Box::pin(first.chain(futures::stream::pending())), + Ending::Error(code) => Box::pin(first.chain(futures::stream::once(async move { + Err(Status::new(code, "streamed fixture error")) + }))), + }; + Ok(Box::pin(TrackedStream { + inner, + streams_dropped: self.streams_dropped, + })) + } +} + +struct TrackedStream { + inner: BatchStream, + streams_dropped: Arc, +} + +impl Stream for TrackedStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + self.inner.as_mut().poll_next(context) + } +} + +impl Drop for TrackedStream { + fn drop(&mut self) { + self.streams_dropped.fetch_add(1, Ordering::SeqCst); + } +} diff --git a/dev/design/ci.md b/dev/design/ci.md index 47ecc8a3..bd789c10 100644 --- a/dev/design/ci.md +++ b/dev/design/ci.md @@ -16,13 +16,15 @@ The CI consists of these main workflows: and macOS universal2 wheels using the release recipe 4. **Coverage** (`.github/workflows/coverage.yml`) - Rust and Python coverage, with regression floors +5. **Experimental server frontend** (`.github/workflows/server-frontend.yml`) - Test and lint + the optional Rust protocol frontend with stable Rust and MSRV 1.88 The other workflows are `lint.yml` (rustfmt, clippy, rustdoc, ruff), `generated-drift.yml`, `vendor-drift.yml`, `docs.yml`, `pandas-parity.yml`, `pipelines-parity.yml` and `release.yml`. Most workflows run on pushes and pull requests targeting `master` and `branch-*`. Several also -accept manual dispatch. `docs.yml` and `wheel-build-checks.yml` run only when files matching their -path filters change. The release wheel build checks cover changes to packaged Python sources, +accept manual dispatch. `docs.yml`, `wheel-build-checks.yml` and `server-frontend.yml` filter push +and pull-request runs to matching paths. The release wheel build checks cover packaged Python sources, protocol definitions, dependencies and build configuration. `release.yml` runs on a schedule and on manual dispatch. @@ -69,6 +71,15 @@ The same workflow has four more jobs: This job should always pass; failures indicate incomplete implementation or test regressions. +## Experimental server frontend (`server-frontend.yml`) + +This focused workflow runs on frontend, protocol/client dependency, workspace manifest, +lockfile, Cargo configuration, toolchain and workflow changes. Each stable Rust and Rust +1.88.0 job has a 45-minute limit and runs `cargo test --locked -p spark-connect-server` +and strict Clippy for all frontend targets. Ubuntu 24.04 supplies a protobuf compiler with +proto3 optional support. Tests use injected engine fixtures and require no execution backend; +they cover frontend protocol behavior rather than Spark execution compatibility. + ## Python Connect Parity (`build_python_connect.yml`) **Status**: Gating. The parity and drop-in jobs pass only when every test that runs passes, except From abe667a62b26cb2027435cc3d79d7adee3cc319a Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 14:05:12 -0800 Subject: [PATCH 02/17] Keep frontend checks independent of client feature selection --- .github/workflows/server-frontend.yml | 2 -- Cargo.lock | 1 - crates/spark-connect-server/Cargo.toml | 1 - crates/spark-connect-server/tests/protocol.rs | 21 ++++++++++++++----- dev/design/ci.md | 2 +- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/server-frontend.yml b/.github/workflows/server-frontend.yml index 7540c9ee..a043bb4d 100644 --- a/.github/workflows/server-frontend.yml +++ b/.github/workflows/server-frontend.yml @@ -30,8 +30,6 @@ on: - "rust-toolchain*" - "crates/spark-connect-server/**" - "crates/spark-connect-proto/**" - - "crates/spark-connect-core/**" - - "crates/spark-connect/**" push: branches: [master, "branch-*"] paths: *frontend-paths diff --git a/Cargo.lock b/Cargo.lock index edfae584..61cee6fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4052,7 +4052,6 @@ dependencies = [ name = "spark-connect-server" version = "0.1.0" dependencies = [ - "apache-spark-connect", "apache-spark-connect-proto", "arrow", "futures", diff --git a/crates/spark-connect-server/Cargo.toml b/crates/spark-connect-server/Cargo.toml index 68def7fa..a27d0413 100644 --- a/crates/spark-connect-server/Cargo.toml +++ b/crates/spark-connect-server/Cargo.toml @@ -43,4 +43,3 @@ tonic-build = "0.14" [dev-dependencies] h2 = "0.4" -spark-connect = { package = "apache-spark-connect", path = "../spark-connect" } diff --git a/crates/spark-connect-server/tests/protocol.rs b/crates/spark-connect-server/tests/protocol.rs index 6a4ef03c..f863426c 100644 --- a/crates/spark-connect-server/tests/protocol.rs +++ b/crates/spark-connect-server/tests/protocol.rs @@ -740,7 +740,7 @@ async fn backend_status_codes_survive_prepare_and_stream_failures() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn deeply_nested_requests_hit_the_standard_protobuf_decoder_limit() { +async fn deeply_nested_requests_follow_the_configured_protobuf_decoder_limit() { let mut server = Server::start().await; let session = Uuid::new_v4().to_string(); let mut request = range(&session, 1); @@ -769,11 +769,22 @@ async fn deeply_nested_requests_hit_the_standard_protobuf_decoder_limit() { request.plan = Some(proto::Plan { op_type: Some(proto::plan::OpType::Root(relation)), }); + let decoded = proto::ExecutePlanRequest::decode(request.encode_to_vec().as_slice()); let status = server.client.execute_plan(request).await.unwrap_err(); - // Each filter introduces nested protobuf messages. The normal decoder rejects - // this request before the backend can inspect it. - assert_eq!(status.code(), Code::Internal); - assert!(status.message().contains("recursion"), "{status}"); + match decoded { + Err(error) => { + // The default decoder rejects this before the backend can inspect it. + assert!(error.to_string().contains("recursion"), "{error}"); + assert_eq!(status.code(), Code::Internal); + assert!(status.message().contains("recursion"), "{status}"); + } + Ok(_) => { + // Other workspace crates enable prost/no-recursion-limit. In that + // configuration the unsupported Filter reaches the fixture backend. + assert_eq!(status.code(), Code::Unimplemented); + assert_eq!(status.message(), "fixture requires a range selector"); + } + } server.stop().await; } diff --git a/dev/design/ci.md b/dev/design/ci.md index bd789c10..e2f68161 100644 --- a/dev/design/ci.md +++ b/dev/design/ci.md @@ -73,7 +73,7 @@ This job should always pass; failures indicate incomplete implementation or test ## Experimental server frontend (`server-frontend.yml`) -This focused workflow runs on frontend, protocol/client dependency, workspace manifest, +This focused workflow runs on frontend, protocol, workspace manifest, lockfile, Cargo configuration, toolchain and workflow changes. Each stable Rust and Rust 1.88.0 job has a 45-minute limit and runs `cargo test --locked -p spark-connect-server` and strict Clippy for all frontend targets. Ubuntu 24.04 supplies a protobuf compiler with From d5cd54024dcafd7357958403457c77061c90f979 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 14:20:12 -0800 Subject: [PATCH 03/17] Enforce consistent Arrow output schemas in the frontend --- .github/workflows/server-frontend.yml | 8 +- crates/spark-connect-server/README.md | 17 +- crates/spark-connect-server/src/engine/mod.rs | 10 +- crates/spark-connect-server/src/lib.rs | 2 + crates/spark-connect-server/src/service.rs | 101 +++- .../tests/engine_injection.rs | 83 +-- crates/spark-connect-server/tests/protocol.rs | 82 +-- .../tests/schema_consistency.rs | 492 ++++++++++++++++++ .../tests/support/server.rs | 80 +++ dev/design/ci.md | 10 +- 10 files changed, 719 insertions(+), 166 deletions(-) create mode 100644 crates/spark-connect-server/tests/schema_consistency.rs create mode 100644 crates/spark-connect-server/tests/support/server.rs diff --git a/.github/workflows/server-frontend.yml b/.github/workflows/server-frontend.yml index a043bb4d..e088800f 100644 --- a/.github/workflows/server-frontend.yml +++ b/.github/workflows/server-frontend.yml @@ -45,7 +45,8 @@ concurrency: env: CARGO_BUILD_JOBS: "2" CARGO_PROFILE_TEST_DEBUG: "0" - RUSTFLAGS: "-D warnings" + # Strict linting uses the pinned toolchain in lint.yml. + RUSTFLAGS: "" jobs: frontend: @@ -62,7 +63,7 @@ jobs: - name: Install Rust toolchain # Use runner-provided rustup to keep actions limited to actions/*. run: | - rustup toolchain install ${{ matrix.rust-version }} --profile minimal --component clippy + rustup toolchain install ${{ matrix.rust-version }} --profile minimal rustup default ${{ matrix.rust-version }} - name: Cache cargo and target @@ -85,6 +86,3 @@ jobs: - name: Test frontend # Protocol and injected-engine tests use local fixtures without an execution backend. run: cargo test --locked -p spark-connect-server - - - name: Strict Clippy for frontend - run: cargo clippy --locked -p spark-connect-server --all-targets --no-deps -- -D warnings diff --git a/crates/spark-connect-server/README.md b/crates/spark-connect-server/README.md index f236fcc8..b00cdd0c 100644 --- a/crates/spark-connect-server/README.md +++ b/crates/spark-connect-server/README.md @@ -43,18 +43,29 @@ Completed responses are not retained. Reattachment, artifact upload/status, The frontend validates UUID session and operation IDs, rejects duplicate active operation IDs within a session, and checks client-observed server session IDs. Client user IDs distinguish session keys but are not authenticated identities. +Protobuf nesting follows the selected Prost features; dependency feature +unification can disable its recursion limit, and the frontend adds no independent +nesting guard. ## Arrow results A prepared query declares an Arrow schema before execution. The frontend converts supported types to the Spark schema representation and checks each batch against -that declaration. Equivalent Arrow representations and differences in batch -nullability or metadata are accepted. Declared field metadata and unsupported -types fail explicitly. +that declaration. Batches may use Spark-equivalent Arrow representations and +nullability different from the declaration. Within one result, emitted IPC +batches must share physical types, field names and nullability, so clients can +assemble them into one Arrow table. Metadata may differ between batches. +Declared field metadata and unsupported types fail explicitly. Engines may yield Arrow record batches or complete Arrow IPC streams containing one batch and its declared row count. The frontend encodes record batches and forwards IPC that fits the response limit; oversized batches are split by rows. +Encoding materializes string/binary view arrays, including nested children, to +`LargeUtf8`/`LargeBinary`. Small view-typed IPC remains unchanged, so mixing it with +oversized view-typed IPC can produce different physical schemas. The frontend +reports `INTERNAL` before sending a batch whose output schema differs from earlier +batches; it does not complete a result containing mixed schemas. Engines should +use consistent non-view IPC types when mixing batch sizes. Result envelopes, including schema and batch responses, are limited to 3 MiB. A single row or schema that cannot fit fails with `RESOURCE_EXHAUSTED`. Each batch carries a cumulative row offset, and empty results still include an empty batch. diff --git a/crates/spark-connect-server/src/engine/mod.rs b/crates/spark-connect-server/src/engine/mod.rs index c4a337da..26798364 100644 --- a/crates/spark-connect-server/src/engine/mod.rs +++ b/crates/spark-connect-server/src/engine/mod.rs @@ -88,9 +88,13 @@ pub enum PreparedOperation { pub trait PreparedQuery: Send { /// The declared Spark schema. Batches must have the same field names and /// Spark-visible types; equivalent Arrow representations are accepted. - /// Batch nullability and metadata may differ and remain in each IPC batch, - /// while the protocol schema header uses this declaration. The declared - /// fields must not contain metadata, which Spark schema conversion rejects. + /// Batch nullability and metadata may differ from this declaration, but + /// emitted IPC batches in one result must share physical types, field names + /// and nullability. Metadata may differ between batches. Encoding materializes + /// view arrays to large-offset types, including when splitting oversized IPC; + /// mixing those with unchanged view IPC fails explicitly before the changed + /// output schema is sent. Declared fields must not contain metadata, which + /// Spark schema conversion rejects. fn schema(&self) -> SchemaRef; /// Dropping this future or its returned stream must cancel backend work; diff --git a/crates/spark-connect-server/src/lib.rs b/crates/spark-connect-server/src/lib.rs index 92f19de6..026e4f6f 100644 --- a/crates/spark-connect-server/src/lib.rs +++ b/crates/spark-connect-server/src/lib.rs @@ -19,6 +19,8 @@ pub mod engine; mod schema; mod service; +/// Generated server bindings for custom embedding. This experimental API follows +/// the RPC table in `build.rs` and may change with the protocol. // tonic's generated async-trait methods duplicate Future's must_use annotation. #[allow(clippy::double_must_use)] pub mod grpc { diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index b4f951da..87460eed 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -20,6 +20,7 @@ use std::panic::AssertUnwindSafe; use std::pin::Pin; use std::sync::{Arc, Mutex}; +use arrow::datatypes::Schema; use arrow::ipc::{reader::StreamReader, writer::StreamWriter}; use arrow::record_batch::RecordBatch; use futures::{FutureExt, Stream, StreamExt}; @@ -36,7 +37,7 @@ use crate::schema::{self, result_schema}; type Output = Pin> + Send>>; type SessionKey = (String, String); -const MAX_ARROW_IPC_BYTES: usize = crate::schema::MAX_RESULT_BYTES; +const MAX_RESULT_BYTES: usize = crate::schema::MAX_RESULT_BYTES; const MAX_RELEASED_SESSIONS: usize = 4096; const RELEASE_WAIT: std::time::Duration = std::time::Duration::from_secs(2); @@ -268,7 +269,7 @@ fn response( fn checked_response( message: proto::ExecutePlanResponse, ) -> Result { - if message.encoded_len() > MAX_ARROW_IPC_BYTES { + if message.encoded_len() > MAX_RESULT_BYTES { return Err(Status::resource_exhausted( "ExecutePlan response exceeds the 3 MiB result limit", )); @@ -372,6 +373,45 @@ fn encode_batch(batch: &RecordBatch) -> Result, Status> { Ok(bytes) } +// PySpark assembles one table from all IPC batches. Its physical types and +// nullability must agree across batches, while field/schema metadata may differ. +fn check_output_schema(data: &[u8], expected: &mut Option) -> Result<(), Status> { + use arrow::datatypes::{DataType, Field, FieldRef}; + + fn field(input: &FieldRef) -> FieldRef { + Arc::new(Field::new( + input.name(), + data_type(input.data_type()), + input.is_nullable(), + )) + } + + fn data_type(input: &DataType) -> DataType { + match input { + DataType::List(child) => DataType::List(field(child)), + DataType::LargeList(child) => DataType::LargeList(field(child)), + DataType::Struct(fields) => DataType::Struct(fields.iter().map(field).collect()), + DataType::Map(entries, sorted) => DataType::Map(field(entries), *sorted), + other => other.clone(), + } + } + + let schema = StreamReader::try_new(data, None) + .map_err(|error| Status::internal(format!("invalid output IPC: {error}")))? + .schema(); + let actual = Schema::new(schema.fields().iter().map(field).collect::>()); + match expected { + Some(expected) if *expected != actual => Err(Status::internal( + "Arrow output schema changed within one result", + )), + Some(_) => Ok(()), + None => { + *expected = Some(actual); + Ok(()) + } + } +} + fn estimated_rows_per_batch(batch: &RecordBatch) -> usize { let bytes = batch.get_array_memory_size(); // Start below the exact IPC limit to allow schema/offset overhead. This is @@ -382,7 +422,7 @@ fn estimated_rows_per_batch(batch: &RecordBatch) -> usize { } fn rows_for_bytes(rows: usize, bytes: usize) -> usize { - (rows as u128 * (MAX_ARROW_IPC_BYTES / 2) as u128 / bytes.max(1) as u128) + (rows as u128 * (MAX_RESULT_BYTES / 2) as u128 / bytes.max(1) as u128) .clamp(1, rows.max(1) as u128) as usize } @@ -406,6 +446,7 @@ async fn send_batch( operation: &str, batch: ResultBatch, offset: &mut i64, + output_schema: &mut Option, guard: &Arc, ) -> Result { let (batch, rows_per_batch) = match batch { @@ -431,7 +472,8 @@ async fn send_batch( batch.clone(), )), ); - if result.encoded_len() <= MAX_ARROW_IPC_BYTES { + if result.encoded_len() <= MAX_RESULT_BYTES { + check_output_schema(&batch.data, output_schema)?; *offset = offset .checked_add(row_count) .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; @@ -467,7 +509,17 @@ async fn send_batch( .map_err(|_| Status::internal("Arrow IPC decoding task failed"))?? } }; - send_arrow_batch(sender, id, operation, batch, rows_per_batch, offset, guard).await + send_arrow_batch( + sender, + id, + operation, + batch, + rows_per_batch, + offset, + output_schema, + guard, + ) + .await } async fn send_arrow_batch( @@ -477,6 +529,7 @@ async fn send_arrow_batch( batch: RecordBatch, rows_per_batch: usize, offset: &mut i64, + output_schema: &mut Option, guard: &Arc, ) -> Result { let mut pending = vec![(batch, rows_per_batch)]; @@ -503,21 +556,22 @@ async fn send_arrow_batch( .await .map_err(|_| Status::internal("Arrow result encoding task failed"))??; let batch = &encoded.batch; + let payload = proto::execute_plan_response::ArrowBatch { + row_count: batch.num_rows() as i64, + data: data.into(), + start_offset: Some(*offset), + ..Default::default() + }; let result = response( id, &guard.session, operation, Some(proto::execute_plan_response::ResponseType::ArrowBatch( - proto::execute_plan_response::ArrowBatch { - row_count: batch.num_rows() as i64, - data: data.into(), - start_offset: Some(*offset), - ..Default::default() - }, + payload.clone(), )), ); let result_bytes = result.encoded_len(); - if result_bytes > MAX_ARROW_IPC_BYTES { + if result_bytes > MAX_RESULT_BYTES { if batch.num_rows() <= 1 { return Err(Status::resource_exhausted( "one Arrow row/schema exceeds the 3 MiB result limit", @@ -525,13 +579,14 @@ async fn send_arrow_batch( } // Recheck every slice including the envelope: skew and schema // overhead can exceed the initial memory estimate. - let rows_per_batch = ((batch.num_rows() as u128 * MAX_ARROW_IPC_BYTES as u128 * 9 + let rows_per_batch = ((batch.num_rows() as u128 * MAX_RESULT_BYTES as u128 * 9 / (result_bytes as u128 * 10)) as usize) .clamp(1, batch.num_rows() - 1); drop(result); pending.push((batch.clone(), rows_per_batch)); continue; } + check_output_schema(&payload.data, output_schema)?; *offset = offset .checked_add(batch.num_rows() as i64) .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; @@ -613,6 +668,7 @@ impl SparkConnectService for ConnectService { } let mut batches = query.execute().await?; let mut offset = 0; + let mut output_schema = None; let mut sent = false; while let Some(batch) = batches.next().await { let batch = batch?; @@ -642,6 +698,7 @@ impl SparkConnectService for ConnectService { &operation, batch, &mut offset, + &mut output_schema, &guard, ) .await? @@ -661,6 +718,7 @@ impl SparkConnectService for ConnectService { &operation, batch.into(), &mut offset, + &mut output_schema, &guard, ) .await? @@ -935,13 +993,13 @@ mod tests { let schema = proto::DataType { kind: Some(proto::data_type::Kind::Struct(proto::data_type::Struct { fields: vec![proto::data_type::StructField { - name: "x".repeat(MAX_ARROW_IPC_BYTES - 64), + name: "x".repeat(MAX_RESULT_BYTES - 64), ..Default::default() }], ..Default::default() })), }; - assert!(schema.encoded_len() < MAX_ARROW_IPC_BYTES); + assert!(schema.encoded_len() < MAX_RESULT_BYTES); let mut header = response( &Uuid::new_v4().to_string(), &session, @@ -971,7 +1029,7 @@ mod tests { .unwrap(); let rows = estimated_rows_per_batch(&batch); assert!(rows > 0 && rows < batch.num_rows()); - assert!(rows * std::mem::size_of::() <= MAX_ARROW_IPC_BYTES / 2); + assert!(rows * std::mem::size_of::() <= MAX_RESULT_BYTES / 2); assert_eq!( estimated_rows_per_batch(&RecordBatch::new_empty(batch.schema())), 1 @@ -1004,6 +1062,7 @@ mod tests { let (guard, _) = session.operation("ipc").unwrap(); let (sender, mut receiver) = mpsc::channel(1); let mut offset = 7; + let mut output_schema = None; assert!(send_batch( &sender, "session", @@ -1013,6 +1072,7 @@ mod tests { row_count: 10 }, &mut offset, + &mut output_schema, &guard, ) .await @@ -1050,12 +1110,13 @@ mod tests { ) .unwrap(); let data = encode_batch(&batch).unwrap(); - assert!(data.len() > MAX_ARROW_IPC_BYTES); - let expected_batches = data.len().div_ceil(MAX_ARROW_IPC_BYTES / 2); + assert!(data.len() > MAX_RESULT_BYTES); + let expected_batches = data.len().div_ceil(MAX_RESULT_BYTES / 2); let session = Arc::new(Session::new(Arc::new(RegistrySession))); let (guard, _) = session.operation("ipc").unwrap(); let (sender, mut receiver) = mpsc::channel(64); let mut offset = 0; + let mut output_schema = None; assert!(send_batch( &sender, "session", @@ -1065,6 +1126,7 @@ mod tests { row_count: 50_000 }, &mut offset, + &mut output_schema, &guard, ) .await @@ -1072,7 +1134,7 @@ mod tests { let (mut rows, mut batches) = (0, 0); while let Ok(result) = receiver.try_recv() { let result = result.unwrap(); - assert!(result.encoded_len() <= MAX_ARROW_IPC_BYTES); + assert!(result.encoded_len() <= MAX_RESULT_BYTES); let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = result.response_type else { @@ -1109,6 +1171,7 @@ mod tests { row_count, }, &mut offset, + &mut output_schema, &guard, ) .await diff --git a/crates/spark-connect-server/tests/engine_injection.rs b/crates/spark-connect-server/tests/engine_injection.rs index a3a81bd3..739a8124 100644 --- a/crates/spark-connect-server/tests/engine_injection.rs +++ b/crates/spark-connect-server/tests/engine_injection.rs @@ -17,6 +17,9 @@ //! Real gRPC tests of the injected backend boundary. The fake is independent of Spark execution. +#[path = "support/server.rs"] +mod server; + use std::collections::HashMap; use std::io::Cursor; use std::pin::Pin; @@ -34,14 +37,12 @@ use futures::{Stream, StreamExt}; use prost::Message; use proto::execute_plan_response::ResponseType; use proto::spark_connect_service_client::SparkConnectServiceClient; +use server::Server; use spark_connect_proto as proto; use spark_connect_server::engine::{ BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, ResultBatch, SessionContext, }; -use tokio::sync::oneshot; -use tokio::task::JoinHandle; -use tonic::transport::Channel; use tonic::{Code, Status}; use uuid::Uuid; @@ -509,56 +510,8 @@ impl Drop for SpyStream { } } -struct Server { - address: std::net::SocketAddr, - client: SparkConnectServiceClient, - shutdown: Option>, - task: Option>, -} - -impl Server { - async fn start(spy: Arc) -> Self { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let (shutdown, stopped) = oneshot::channel(); - let task = tokio::spawn(async move { - spark_connect_server::serve_with_engine(listener, Arc::new(FakeEngine(spy)), async { - let _ = stopped.await; - }) - .await - .unwrap(); - }); - let client = SparkConnectServiceClient::connect(format!("http://{address}")) - .await - .unwrap(); - Self { - address, - client, - shutdown: Some(shutdown), - task: Some(task), - } - } - - async fn stop(mut self) { - if let Some(shutdown) = self.shutdown.take() { - let _ = shutdown.send(()); - } - tokio::time::timeout(Duration::from_secs(5), self.task.take().unwrap()) - .await - .unwrap() - .unwrap(); - } -} - -impl Drop for Server { - fn drop(&mut self) { - if let Some(shutdown) = self.shutdown.take() { - let _ = shutdown.send(()); - } - if let Some(task) = self.task.take() { - task.abort(); - } - } +async fn start_server(spy: Arc) -> Server { + Server::start_with_engine(Arc::new(FakeEngine(spy))).await } fn user(id: &str) -> Option { @@ -727,7 +680,7 @@ async fn start_live( #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn injected_backend_owns_query_analysis_config_and_catalog_state() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; let session = Uuid::new_v4().to_string(); let mut first = server.client.clone(); let mut second = server.client.clone(); @@ -904,7 +857,7 @@ async fn injected_backend_owns_query_analysis_config_and_catalog_state() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn invalid_requests_do_not_open_backends_and_factory_failures_can_retry() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; let session = Uuid::new_v4().to_string(); assert_eq!( server @@ -966,7 +919,7 @@ async fn invalid_requests_do_not_open_backends_and_factory_failures_can_retry() #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn native_streams_drop_on_release_disconnect_session_release_and_shutdown() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; let session = Uuid::new_v4().to_string(); let live = request(&session, LIVE); let operation = live.operation_id.clone().unwrap(); @@ -1060,7 +1013,7 @@ async fn native_streams_drop_on_release_disconnect_session_release_and_shutdown( #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn shutdown_releases_backend_session_while_a_result_remains_unread() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; let (mut client, connection) = h2::client::Builder::new() .initial_window_size(1024) .initial_connection_window_size(1024) @@ -1119,7 +1072,7 @@ async fn shutdown_releases_backend_session_while_a_result_remains_unread() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn operation_reservation_precedes_prepare_and_cleans_up_after_cancellation_and_errors() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; let session = Uuid::new_v4().to_string(); let blocked = request(&session, BLOCKED_PREPARE); let operation = blocked.operation_id.clone().unwrap(); @@ -1195,7 +1148,7 @@ async fn operation_reservation_precedes_prepare_and_cleans_up_after_cancellation #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn release_and_shutdown_cancel_backend_metadata_futures_before_dropping_sessions() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; let session = Uuid::new_v4().to_string(); let mut analysis_client = server.client.clone(); let analysis_session = session.clone(); @@ -1264,7 +1217,7 @@ async fn release_and_shutdown_cancel_backend_metadata_futures_before_dropping_se #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn slow_backend_drop_does_not_hold_the_global_session_registry_lock() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; // Channel clones share an HTTP/2 connection. Establish a new TCP connection // before blocking Drop so the probe isolates the global registry lock. let mut independent = SparkConnectServiceClient::connect(format!("http://{}", server.address)) @@ -1321,7 +1274,7 @@ async fn slow_backend_drop_does_not_hold_the_global_session_registry_lock() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn frontend_rejects_oversized_analysis_and_batch_type_mismatches() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; let session = Uuid::new_v4().to_string(); let status = server .client @@ -1369,7 +1322,7 @@ async fn frontend_rejects_oversized_analysis_and_batch_type_mismatches() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn frontend_rejects_mismatched_and_invalid_ipc_before_forwarding() { - let mut server = Server::start(Arc::new(Spy::default())).await; + let mut server = start_server(Arc::new(Spy::default())).await; let session = Uuid::new_v4().to_string(); for (selector, message) in [ ( @@ -1399,7 +1352,7 @@ async fn frontend_rejects_mismatched_and_invalid_ipc_before_forwarding() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn panicking_execution_reports_internal_after_header_or_partial_result() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; let session = Uuid::new_v4().to_string(); for selector in [PANICKING_EXECUTION, PANICKING_STREAM, PANICKING_STREAM_DROP] { let mut stream = server @@ -1435,7 +1388,7 @@ async fn panicking_execution_reports_internal_after_header_or_partial_result() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn oversized_command_payload_fails_before_sending_any_success_message() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy).await; + let mut server = start_server(spy).await; let session = Uuid::new_v4().to_string(); let command = command_request( &session, @@ -1471,7 +1424,7 @@ async fn release_all_timeout_keeps_cancellation_and_operation_ownership_until_dr let gate = Arc::new(DropGate::default()); *spy.stream_drop_gate.lock().unwrap() = Some(gate.clone()); let unblock = UnblockDrop(gate.clone()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; let session = Uuid::new_v4().to_string(); let original = request(&session, LIVE); let operation = original.operation_id.clone().unwrap(); diff --git a/crates/spark-connect-server/tests/protocol.rs b/crates/spark-connect-server/tests/protocol.rs index f863426c..1377bcc2 100644 --- a/crates/spark-connect-server/tests/protocol.rs +++ b/crates/spark-connect-server/tests/protocol.rs @@ -17,6 +17,10 @@ //! Verify observable Spark Connect wire behavior independently of DataFrame conveniences. +#[path = "support/server.rs"] +mod server; +mod support; + use std::collections::HashSet; use std::io::Cursor; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -27,59 +31,16 @@ use arrow::array::Int64Array; use arrow::ipc::reader::StreamReader; use prost::Message; use proto::execute_plan_response::ResponseType; -use proto::spark_connect_service_client::SparkConnectServiceClient; use spark_connect_proto as proto; use spark_connect_server::engine::{Engine, EngineSession, SessionContext}; -mod support; +use server::Server; use support::{FixtureEngine, LIVE_FIXTURE}; -use tokio::sync::oneshot; -use tokio::task::JoinHandle; -use tonic::transport::Channel; use tonic::Code; use uuid::Uuid; -struct Server { - client: SparkConnectServiceClient, - shutdown: Option>, - task: Option>, -} - -impl Server { - async fn start() -> Self { - Self::start_with_engine(Arc::new(FixtureEngine::default())).await - } - - async fn start_with_engine(engine: Arc) -> Self { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let (shutdown, stopped) = oneshot::channel(); - let task = tokio::spawn(async move { - spark_connect_server::serve_with_engine(listener, engine, async { - let _ = stopped.await; - }) - .await - .unwrap(); - }); - let client = SparkConnectServiceClient::connect(format!("http://{address}")) - .await - .unwrap(); - Self { - client, - shutdown: Some(shutdown), - task: Some(task), - } - } - - async fn stop(mut self) { - if let Some(shutdown) = self.shutdown.take() { - let _ = shutdown.send(()); - } - tokio::time::timeout(Duration::from_secs(5), self.task.take().unwrap()) - .await - .unwrap() - .unwrap(); - } +async fn start_server() -> Server { + Server::start_with_engine(Arc::new(FixtureEngine::default())).await } #[derive(Default)] @@ -97,17 +58,6 @@ impl Engine for CountingEngine { } } -impl Drop for Server { - fn drop(&mut self) { - if let Some(shutdown) = self.shutdown.take() { - let _ = shutdown.send(()); - } - if let Some(task) = self.task.take() { - task.abort(); - } - } -} - fn range(session: &str, end: i64) -> proto::ExecutePlanRequest { proto::ExecutePlanRequest { session_id: session.into(), @@ -176,7 +126,7 @@ async fn assert_cancelled(stream: &mut tonic::Streaming strings + DataType::Utf8 => strings .as_any() .downcast_ref::() .unwrap() .value(index), - arrow::datatypes::DataType::LargeUtf8 => strings + DataType::LargeUtf8 => strings .as_any() .downcast_ref::() .unwrap() .value(index), - arrow::datatypes::DataType::Utf8View => strings + DataType::Utf8View => strings .as_any() .downcast_ref::() .unwrap() @@ -467,7 +417,7 @@ async fn arrow_payloads_are_byte_limited_and_oversized_rows_fail() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn release_session_cancels_live_stream_and_live_operation_ids_are_unique() { - let mut server = Server::start().await; + let mut server = start_server().await; let session = Uuid::new_v4().to_string(); let request = range(&session, LIVE_FIXTURE); let mut stream = server @@ -646,7 +596,7 @@ async fn invalid_requests_do_not_allocate_sessions_and_unknown_release_is_reject #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn disconnect_removes_operation_and_shutdown_cancels_live_streams() { - let mut server = Server::start().await; + let mut server = start_server().await; let session = Uuid::new_v4().to_string(); let request = range(&session, LIVE_FIXTURE); let operation = request.operation_id.clone().unwrap(); @@ -741,7 +691,7 @@ async fn backend_status_codes_survive_prepare_and_stream_failures() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn deeply_nested_requests_follow_the_configured_protobuf_decoder_limit() { - let mut server = Server::start().await; + let mut server = start_server().await; let session = Uuid::new_v4().to_string(); let mut request = range(&session, 1); let Some(proto::plan::OpType::Root(mut relation)) = request.plan.take().unwrap().op_type else { @@ -790,8 +740,6 @@ async fn deeply_nested_requests_follow_the_configured_protobuf_decoder_limit() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn view_backing_buffers_are_compacted_before_splitting_including_struct_children() { - use std::sync::Arc; - use arrow::array::{BinaryArray, BinaryViewArray, StringArray, StringViewArray, StructArray}; use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatch; diff --git a/crates/spark-connect-server/tests/schema_consistency.rs b/crates/spark-connect-server/tests/schema_consistency.rs new file mode 100644 index 00000000..efb9e8df --- /dev/null +++ b/crates/spark-connect-server/tests/schema_consistency.rs @@ -0,0 +1,492 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Result schemas remain consumable as one Arrow table across output batches. + +#[path = "support/server.rs"] +mod server; + +use std::collections::HashMap; +use std::io::Cursor; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::{ + Array, ArrayRef, BinaryViewArray, LargeBinaryArray, LargeListArray, LargeStringArray, + ListArray, MapArray, StringArray, StringViewArray, StructArray, +}; +use arrow::buffer::OffsetBuffer; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::ipc::{reader::StreamReader, writer::StreamWriter}; +use arrow::record_batch::RecordBatch; +use prost::Message; +use spark_connect_proto as proto; +use spark_connect_server::engine::{ + BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, + ResultBatch, SessionContext, +}; +use tonic::{Code, Status}; +use uuid::Uuid; + +use server::Server; + +const RESULT_LIMIT: usize = 3 * 1024 * 1024; +const LARGE_ROWS: usize = 4096; + +#[derive(Clone)] +enum Batch { + Arrow(RecordBatch), + Ipc(Vec, i64), +} + +#[derive(Clone)] +struct BatchEngine { + schema: SchemaRef, + batches: Vec, +} + +impl Engine for BatchEngine { + fn create_session(&self, _: SessionContext) -> Result, Status> { + Ok(Arc::new(self.clone())) + } +} + +#[tonic::async_trait] +impl EngineSession for BatchEngine { + async fn prepare(&self, _: proto::plan::OpType) -> Result { + Ok(PreparedOperation::Query(Box::new(self.clone()))) + } + + async fn analyze( + &self, + _: proto::analyze_plan_request::Analyze, + ) -> Result { + Err(Status::unimplemented("schema fixture has no analysis")) + } + + async fn config( + &self, + _: proto::config_request::operation::OpType, + ) -> Result { + Err(Status::unimplemented("schema fixture has no configuration")) + } +} + +#[tonic::async_trait] +impl PreparedQuery for BatchEngine { + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + async fn execute(self: Box) -> Result { + Ok(Box::pin(futures::stream::iter( + self.batches.into_iter().map(|batch| { + Ok(match batch { + Batch::Arrow(batch) => ResultBatch::Arrow(batch), + Batch::Ipc(data, row_count) => ResultBatch::Ipc { data, row_count }, + }) + }), + ))) + } +} + +fn ipc(batch: &RecordBatch) -> Vec { + let mut data = Vec::new(); + let mut writer = StreamWriter::try_new(&mut data, batch.schema().as_ref()).unwrap(); + writer.write(batch).unwrap(); + writer.finish().unwrap(); + drop(writer); + data +} + +async fn execute(engine: BatchEngine) -> (Vec, Option) { + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let request = proto::ExecutePlanRequest { + session_id: Uuid::new_v4().to_string(), + operation_id: Some(Uuid::new_v4().to_string()), + plan: Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(proto::Relation { + rel_type: Some(proto::relation::RelType::Range(proto::Range { + end: 1, + step: 1, + ..Default::default() + })), + ..Default::default() + })), + }), + ..Default::default() + }; + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + let result = tokio::time::timeout(Duration::from_secs(30), async { + let mut responses = Vec::new(); + loop { + match stream.message().await { + Ok(Some(response)) => { + assert!(response.encoded_len() <= RESULT_LIMIT); + responses.push(response); + } + Ok(None) => break (responses, None), + Err(error) => break (responses, Some(error)), + } + } + }) + .await + .expect("schema fixture execution must terminate"); + server.stop().await; + result +} + +fn output_batches(responses: &[proto::ExecutePlanResponse]) -> Vec { + let mut offset = 0; + let mut batches = Vec::new(); + for response in responses { + if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + &response.response_type + { + assert_eq!(batch.start_offset, Some(offset)); + let mut reader = StreamReader::try_new(Cursor::new(&batch.data), None).unwrap(); + let decoded = reader.next().unwrap().unwrap(); + assert_eq!(decoded.num_rows() as i64, batch.row_count); + assert!(reader.next().is_none()); + offset += batch.row_count; + batches.push(decoded); + } + } + batches +} + +fn assert_schema_error(responses: &[proto::ExecutePlanResponse], status: Option) { + let status = status.expect("a changed output schema must fail"); + assert_eq!(status.code(), Code::Internal); + assert_eq!( + status.message(), + "Arrow output schema changed within one result" + ); + assert!(!completed(responses)); +} + +fn completed(responses: &[proto::ExecutePlanResponse]) -> bool { + responses.iter().any(|response| { + matches!( + response.response_type, + Some(proto::execute_plan_response::ResponseType::ResultComplete( + _ + )) + ) + }) +} + +fn strings(data_type: DataType, nullable: bool) -> RecordBatch { + let values: ArrayRef = match data_type { + DataType::Utf8 => Arc::new(StringArray::from(vec!["value"])), + DataType::LargeUtf8 => Arc::new(LargeStringArray::from(vec!["value"])), + _ => unreachable!(), + }; + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("value", data_type, nullable)])), + vec![values], + ) + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn changed_physical_types_and_nullability_fail_before_the_second_batch() { + let first = strings(DataType::Utf8, false); + for second in [ + strings(DataType::LargeUtf8, false), + strings(DataType::Utf8, true), + ] { + let (responses, status) = execute(BatchEngine { + schema: first.schema(), + batches: vec![Batch::Arrow(first.clone()), Batch::Arrow(second)], + }) + .await; + assert_schema_error(&responses, status); + let batches = output_batches(&responses); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].schema(), first.schema()); + } +} + +fn metadata_batch(note: &str) -> RecordBatch { + let metadata = HashMap::from([("producer.note".to_string(), note.to_string())]); + let item = + Arc::new(Field::new("item", DataType::LargeUtf8, true).with_metadata(metadata.clone())); + let values = ListArray::new( + item.clone(), + OffsetBuffer::new(vec![0, 1].into()), + Arc::new(LargeStringArray::from(vec!["value"])), + None, + ); + RecordBatch::try_new( + Arc::new(Schema::new_with_metadata( + vec![Field::new("value", DataType::List(item), true).with_metadata(metadata.clone())], + metadata, + )), + vec![Arc::new(values)], + ) + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn metadata_can_change_and_batch_schema_can_differ_from_the_declaration() { + let declared = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::List(Arc::new(Field::new("item", DataType::Utf8, false))), + false, + )])); + let (responses, status) = execute(BatchEngine { + schema: declared, + batches: vec![ + Batch::Arrow(metadata_batch("first")), + Batch::Arrow(metadata_batch("second")), + ], + }) + .await; + assert!(status.is_none(), "{status:?}"); + assert!(completed(&responses)); + let batches = output_batches(&responses); + assert_eq!(batches.len(), 2); + for (batch, note) in batches.iter().zip(["first", "second"]) { + let schema = batch.schema(); + assert_eq!(schema.metadata()["producer.note"], note); + assert_eq!(schema.field(0).metadata()["producer.note"], note); + let DataType::List(item) = schema.field(0).data_type() else { + panic!("expected list output"); + }; + assert_eq!(item.metadata()["producer.note"], note); + assert_eq!(item.data_type(), &DataType::LargeUtf8); + assert!(schema.field(0).is_nullable()); + assert!(item.is_nullable()); + } +} + +fn text(row: usize) -> String { + format!("{row:04}{}", "x".repeat(1020)) +} + +fn views(rows: usize) -> RecordBatch { + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Utf8View, + false, + )])), + vec![Arc::new(StringViewArray::from_iter_values( + (0..rows).map(text), + ))], + ) + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_ipc_is_unchanged_when_small_and_materialized_when_oversized() { + for rows in [2, LARGE_ROWS] { + let batch = views(rows); + let data = ipc(&batch); + assert_eq!(data.len() > RESULT_LIMIT, rows == LARGE_ROWS); + let (responses, status) = execute(BatchEngine { + schema: batch.schema(), + batches: vec![Batch::Ipc(data.clone(), rows as i64)], + }) + .await; + assert!(status.is_none(), "{status:?}"); + assert!(completed(&responses)); + let batches = output_batches(&responses); + if rows == 2 { + let payload = responses + .iter() + .find_map(|response| match &response.response_type { + Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) => { + Some(&batch.data) + } + _ => None, + }) + .unwrap(); + assert_eq!(payload.as_ref(), data.as_slice()); + assert_eq!( + batches[0].schema().field(0).data_type(), + &DataType::Utf8View + ); + } else { + assert!(batches.len() > 1); + let mut offset = 0; + for batch in batches { + assert_eq!(batch.schema().field(0).data_type(), &DataType::LargeUtf8); + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for row in 0..batch.num_rows() { + assert_eq!(values.value(row), text(offset + row)); + } + offset += batch.num_rows(); + } + assert_eq!(offset, rows); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn mixing_small_and_oversized_view_ipc_rejects_the_changed_output_schema() { + let small = views(2); + let large = views(LARGE_ROWS); + let small = Batch::Ipc(ipc(&small), 2); + let large = Batch::Ipc(ipc(&large), LARGE_ROWS as i64); + for large_first in [false, true] { + let (responses, status) = execute(BatchEngine { + schema: views(0).schema(), + batches: if large_first { + vec![large.clone(), small.clone()] + } else { + vec![small.clone(), large.clone()] + }, + }) + .await; + assert_schema_error(&responses, status); + let batches = output_batches(&responses); + let expected_type = if large_first { + DataType::LargeUtf8 + } else { + DataType::Utf8View + }; + let expected_rows = if large_first { LARGE_ROWS } else { 2 }; + assert_eq!( + batches.iter().map(RecordBatch::num_rows).sum::(), + expected_rows + ); + for batch in batches { + assert_eq!(batch.schema().field(0).data_type(), &expected_type); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_children_in_lists_large_lists_and_maps_are_materialized_when_split() { + let values: ArrayRef = Arc::new(StringViewArray::from_iter_values((0..LARGE_ROWS).map(text))); + let offsets = OffsetBuffer::new((0..=LARGE_ROWS as i32).collect::>().into()); + let item = Arc::new(Field::new("item", DataType::Utf8View, false)); + let lists = ListArray::new(item.clone(), offsets.clone(), values.clone(), None); + let large_lists = LargeListArray::new( + item, + OffsetBuffer::new((0..=LARGE_ROWS as i64).collect::>().into()), + values.clone(), + None, + ); + let entries = StructArray::new( + vec![ + Field::new("key", DataType::Utf8View, false), + Field::new("value", DataType::BinaryView, false), + ] + .into(), + vec![ + values, + Arc::new(BinaryViewArray::from_iter_values( + (0..LARGE_ROWS).map(|row| text(row).into_bytes()), + )), + ], + None, + ); + let maps = MapArray::new( + Arc::new(Field::new("entries", entries.data_type().clone(), false)), + offsets, + entries, + None, + false, + ); + let columns: Vec = vec![Arc::new(lists), Arc::new(large_lists), Arc::new(maps)]; + let schema = Arc::new(Schema::new( + columns + .iter() + .enumerate() + .map(|(index, column)| { + Field::new(format!("value{index}"), column.data_type().clone(), false) + }) + .collect::>(), + )); + let batch = RecordBatch::try_new(schema.clone(), columns).unwrap(); + let (responses, status) = execute(BatchEngine { + schema, + batches: vec![Batch::Arrow(batch)], + }) + .await; + assert!(status.is_none(), "{status:?}"); + assert!(completed(&responses)); + let batches = output_batches(&responses); + assert!(batches.len() > 1); + let mut offset = 0; + for batch in batches { + let lists = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let large_lists = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let maps = batch.column(2).as_any().downcast_ref::().unwrap(); + for row in 0..batch.num_rows() { + let expected = text(offset + row); + let list = lists.value(row); + assert_eq!( + list.as_any() + .downcast_ref::() + .unwrap() + .value(0), + expected + ); + let large_list = large_lists.value(row); + assert_eq!( + large_list + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + expected + ); + let entries = maps.value(row); + assert_eq!( + entries + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + expected + ); + assert_eq!( + entries + .column(1) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + expected.as_bytes() + ); + } + offset += batch.num_rows(); + } + assert_eq!(offset, LARGE_ROWS); +} diff --git a/crates/spark-connect-server/tests/support/server.rs b/crates/spark-connect-server/tests/support/server.rs new file mode 100644 index 00000000..0ec88720 --- /dev/null +++ b/crates/spark-connect-server/tests/support/server.rs @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use spark_connect_proto::spark_connect_service_client::SparkConnectServiceClient; +use spark_connect_server::engine::Engine; +use tokio::sync::oneshot; +use tokio::task::JoinHandle; +use tonic::transport::Channel; + +pub struct Server { + // Some test binaries need a second connection to the same listener. + #[allow(dead_code)] + pub address: SocketAddr, + pub client: SparkConnectServiceClient, + pub shutdown: Option>, + task: Option>, +} + +impl Server { + pub async fn start_with_engine(engine: Arc) -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (shutdown, stopped) = oneshot::channel(); + let task = tokio::spawn(async move { + spark_connect_server::serve_with_engine(listener, engine, async { + let _ = stopped.await; + }) + .await + .unwrap(); + }); + let client = SparkConnectServiceClient::connect(format!("http://{address}")) + .await + .unwrap(); + Self { + address, + client, + shutdown: Some(shutdown), + task: Some(task), + } + } + + pub async fn stop(mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + tokio::time::timeout(Duration::from_secs(5), self.task.take().unwrap()) + .await + .unwrap() + .unwrap(); + } +} + +impl Drop for Server { + fn drop(&mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + if let Some(task) = self.task.take() { + task.abort(); + } + } +} diff --git a/dev/design/ci.md b/dev/design/ci.md index e2f68161..d30dcf3b 100644 --- a/dev/design/ci.md +++ b/dev/design/ci.md @@ -16,8 +16,8 @@ The CI consists of these main workflows: and macOS universal2 wheels using the release recipe 4. **Coverage** (`.github/workflows/coverage.yml`) - Rust and Python coverage, with regression floors -5. **Experimental server frontend** (`.github/workflows/server-frontend.yml`) - Test and lint - the optional Rust protocol frontend with stable Rust and MSRV 1.88 +5. **Experimental server frontend** (`.github/workflows/server-frontend.yml`) - Test the + optional Rust protocol frontend with stable Rust and MSRV 1.88 The other workflows are `lint.yml` (rustfmt, clippy, rustdoc, ruff), `generated-drift.yml`, `vendor-drift.yml`, `docs.yml`, `pandas-parity.yml`, `pipelines-parity.yml` and `release.yml`. @@ -75,8 +75,10 @@ This job should always pass; failures indicate incomplete implementation or test This focused workflow runs on frontend, protocol, workspace manifest, lockfile, Cargo configuration, toolchain and workflow changes. Each stable Rust and Rust -1.88.0 job has a 45-minute limit and runs `cargo test --locked -p spark-connect-server` -and strict Clippy for all frontend targets. Ubuntu 24.04 supplies a protobuf compiler with +1.88.0 job has a 45-minute limit and runs `cargo test --locked -p spark-connect-server`. +The existing workspace lint job covers the frontend with its pinned Rust toolchain; +test jobs do not promote warnings from a floating or older compiler to errors. +Ubuntu 24.04 supplies a protobuf compiler with proto3 optional support. Tests use injected engine fixtures and require no execution backend; they cover frontend protocol behavior rather than Spark execution compatibility. From 9441b6cd3a08d9f82bf258bf506587bcf9aedc59 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 15:14:03 -0800 Subject: [PATCH 04/17] Fix frontend slicing, identifiers and shutdown lifecycle --- Cargo.lock | 1 + Cargo.toml | 3 + crates/spark-connect-server/Cargo.toml | 9 +- crates/spark-connect-server/README.md | 19 +- crates/spark-connect-server/build.rs | 45 +- crates/spark-connect-server/src/engine/mod.rs | 14 +- crates/spark-connect-server/src/lib.rs | 44 +- crates/spark-connect-server/src/service.rs | 151 ++++++- .../tests/review_regressions.rs | 425 ++++++++++++++++++ .../tests/server_shutdown.rs | 192 ++++++++ 10 files changed, 865 insertions(+), 38 deletions(-) create mode 100644 crates/spark-connect-server/tests/review_regressions.rs create mode 100644 crates/spark-connect-server/tests/server_shutdown.rs diff --git a/Cargo.lock b/Cargo.lock index 61cee6fd..7e32c1e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4057,6 +4057,7 @@ dependencies = [ "futures", "h2", "prost", + "prost-types", "tokio", "tokio-stream", "tonic", diff --git a/Cargo.toml b/Cargo.toml index 473aebeb..9ba59607 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,11 +90,14 @@ should_implement_trait = "allow" tonic = { version = "0.14", default-features = false, features = ["channel", "codegen"] } tonic-prost = "0.14" tonic-prost-build = "0.14" +tonic-build = "0.14" +h2 = "0.4" prost = "0.14" prost-types = "0.14" # async runtime tokio = { version = "1", features = ["rt-multi-thread", "sync", "macros", "time"] } +tokio-stream = "0.1" futures = "0.3" # Arrow (result decoding). diff --git a/crates/spark-connect-server/Cargo.toml b/crates/spark-connect-server/Cargo.toml index a27d0413..a923d66e 100644 --- a/crates/spark-connect-server/Cargo.toml +++ b/crates/spark-connect-server/Cargo.toml @@ -33,13 +33,16 @@ arrow.workspace = true futures.workspace = true prost.workspace = true tokio = { workspace = true, features = ["net"] } -tokio-stream = { version = "0.1", features = ["net"] } +tokio-stream = { workspace = true, features = ["net"] } tonic = { workspace = true, features = ["server", "router"] } tonic-prost.workspace = true uuid.workspace = true [build-dependencies] -tonic-build = "0.14" +tonic-build.workspace = true +spark-connect-proto = { package = "apache-spark-connect-proto", path = "../spark-connect-proto" } +prost.workspace = true +prost-types.workspace = true [dev-dependencies] -h2 = "0.4" +h2.workspace = true diff --git a/crates/spark-connect-server/README.md b/crates/spark-connect-server/README.md index b00cdd0c..564b6531 100644 --- a/crates/spark-connect-server/README.md +++ b/crates/spark-connect-server/README.md @@ -40,8 +40,9 @@ Completed responses are not retained. Reattachment, artifact upload/status, `Interrupt`, enriched error details, session cloning and status queries return `UNIMPLEMENTED`. -The frontend validates UUID session and operation IDs, rejects duplicate active -operation IDs within a session, and checks client-observed server session IDs. +The frontend requires lowercase, hyphenated UUID session and operation IDs, +rejects duplicate active operation IDs within a session, and checks +client-observed server session IDs. Client user IDs distinguish session keys but are not authenticated identities. Protobuf nesting follows the selected Prost features; dependency feature unification can disable its recursion limit, and the frontend adds no independent @@ -74,7 +75,8 @@ carries a cumulative row offset, and empty results still include an empty batch. Engine methods and streams must cancel their owned work when dropped. The frontend cancels them when a client disconnects, an operation is released, a -session is released or server shutdown begins. Results pass through a bounded +session is released, server shutdown begins or the serving future is dropped. +Results pass through a bounded channel, so unread responses eventually stop polling the engine. Blocking Arrow encoding retains operation ownership until its worker finishes. @@ -84,6 +86,17 @@ for transport drain; returning from `serve_with_engine` does not prove that an engine's independently scheduled or blocking work has finished. The caller must honor the engine's cleanup contract before releasing its resources. +`ReleaseSession(allow_reconnect=true)` permits a new backend session with the +same protocol ID while the previous incarnation still drains. Engines must keep +the two incarnations' resources independent. Commands may commit effects before +frontend response validation fails, for example on an oversized payload; that +failure provides no rollback or safe-retry guarantee. + +With panic unwinding enabled, panics in `prepare`, `analyze`, `config` and query +execution are translated to `INTERNAL`. Synchronous session creation and query +schema access are outside that boundary and must not panic. Resource cleanup +must not panic either. + This is an experiment for trusted local clients and trusted engine outputs. Keep the supplied listener on loopback: this frontend supplies no authentication, authorization or TLS. Requests can be up to 64 MiB. Active sessions and operations diff --git a/crates/spark-connect-server/build.rs b/crates/spark-connect-server/build.rs index 3ffc7807..5041d662 100644 --- a/crates/spark-connect-server/build.rs +++ b/crates/spark-connect-server/build.rs @@ -15,13 +15,14 @@ // specific language governing permissions and limitations // under the License. +use prost::Message; use tonic_build::manual::{Builder, Method, Service}; fn main() { // The official protocol crate already generates messages and client bindings. // Generate only the standard service routes, using those exact message types. - // Keep this RPC list and its streaming signatures synchronized with - // spark-connect-proto/proto/spark/connect/base.proto on each protocol bump. + // Check this table against the generated protocol descriptor before emitting + // routes: streaming signatures are part of wire compatibility. let methods = [ ( "execute_plan", @@ -120,6 +121,46 @@ fn main() { false, ), ]; + let descriptors = + prost_types::FileDescriptorSet::decode(spark_connect_proto::FILE_DESCRIPTOR_SET) + .expect("valid Spark Connect descriptor set"); + let descriptor = descriptors + .file + .iter() + .filter(|file| file.package.as_deref() == Some("spark.connect")) + .flat_map(|file| &file.service) + .find(|service| service.name.as_deref() == Some("SparkConnectService")) + .expect("SparkConnectService descriptor"); + assert_eq!(methods.len(), descriptor.method.len(), "RPC table drift"); + let mut routes = std::collections::HashSet::new(); + for &(_, route, input, output, client_stream, server_stream) in &methods { + assert!(routes.insert(route), "duplicate RPC route: {route}"); + let method = descriptor + .method + .iter() + .find(|method| method.name.as_deref() == Some(route)) + .unwrap_or_else(|| panic!("missing protocol RPC: {route}")); + assert_eq!( + method.input_type.as_deref(), + Some(format!(".spark.connect.{input}").as_str()), + "{route} request type drift" + ); + assert_eq!( + method.output_type.as_deref(), + Some(format!(".spark.connect.{output}").as_str()), + "{route} response type drift" + ); + assert_eq!( + method.client_streaming.unwrap_or(false), + client_stream, + "{route} client streaming drift" + ); + assert_eq!( + method.server_streaming.unwrap_or(false), + server_stream, + "{route} server streaming drift" + ); + } let mut service = Service::builder() .name("SparkConnectService") .package("spark.connect"); diff --git a/crates/spark-connect-server/src/engine/mod.rs b/crates/spark-connect-server/src/engine/mod.rs index 26798364..d2f44b46 100644 --- a/crates/spark-connect-server/src/engine/mod.rs +++ b/crates/spark-connect-server/src/engine/mod.rs @@ -43,9 +43,12 @@ pub struct SessionContext { /// Factory for sessions bound to one native engine. pub trait Engine: Send + Sync { - /// Allocate local session state without blocking or performing I/O. + /// Allocate local session state without blocking, performing I/O or panicking. /// Defer asynchronous initialization to the session methods. The frontend - /// serializes creation to ensure one backend session per protocol session. + /// serializes creation to ensure one backend session per active incarnation. + /// A reconnecting session release permits a replacement with the same + /// `session_id` while references to the previous incarnation are still + /// draining. The engine must keep those incarnations' resources independent. fn create_session(&self, context: SessionContext) -> Result, Status>; } @@ -53,6 +56,7 @@ pub trait Engine: Send + Sync { /// /// Dropping a method future must cancel its work. Dropping the final session /// reference releases its resources after in-flight operations have finished. +/// Resource cleanup must not panic. // async_trait repeats Future's must_use annotation on generated trait methods. #[allow(clippy::double_must_use)] #[tonic::async_trait] @@ -78,6 +82,9 @@ pub trait EngineSession: Send + Sync { pub enum PreparedOperation { Query(Box), /// The frontend adds envelope IDs and the final ResultComplete message. + /// Eager command effects may already be committed when later frontend + /// response validation fails, for example when a payload exceeds the size + /// limit. Such a failure does not roll back effects or make retries safe. Command(Vec), } @@ -86,7 +93,8 @@ pub enum PreparedOperation { #[allow(clippy::double_must_use)] #[tonic::async_trait] pub trait PreparedQuery: Send { - /// The declared Spark schema. Batches must have the same field names and + /// The declared Spark schema. This synchronous accessor must not panic. + /// Batches must have the same field names and /// Spark-visible types; equivalent Arrow representations are accepted. /// Batch nullability and metadata may differ from this declaration, but /// emitted IPC batches in one result must share physical types, field names diff --git a/crates/spark-connect-server/src/lib.rs b/crates/spark-connect-server/src/lib.rs index 026e4f6f..dece96a2 100644 --- a/crates/spark-connect-server/src/lib.rs +++ b/crates/spark-connect-server/src/lib.rs @@ -30,29 +30,42 @@ pub mod grpc { )); } +struct CancelOnDrop(std::sync::Arc); + +impl Drop for CancelOnDrop { + fn drop(&mut self) { + self.0.cancel_all(); + } +} + +fn tcp_incoming(listener: tokio::net::TcpListener) -> tonic::transport::server::TcpIncoming { + // Builder socket options do not apply to serve_with_incoming_shutdown. + tonic::transport::server::TcpIncoming::from(listener).with_nodelay(Some(true)) +} + /// Serve with an injected execution engine and a fresh session registry. /// /// The frontend binds each protocol session to the engine session created for it. /// The caller owns the engine and signals shutdown before releasing its resources. +/// Dropping this future also cancels registered engine work, without waiting for +/// the bounded transport drain used when shutdown is signalled. pub async fn serve_with_engine( listener: tokio::net::TcpListener, engine: std::sync::Arc, shutdown: impl std::future::Future + Send + 'static, ) -> Result<(), tonic::transport::Error> { let service = std::sync::Arc::new(service::ConnectService::new(engine)); + let _cancel_on_drop = CancelOnDrop(service.clone()); let router = tonic::transport::Server::builder().add_service( grpc::spark_connect_service_server::SparkConnectServiceServer::from_arc(service.clone()) .max_decoding_message_size(64 * 1024 * 1024), ); let (draining, drain_started) = tokio::sync::oneshot::channel(); - let server = router.serve_with_incoming_shutdown( - tokio_stream::wrappers::TcpListenerStream::new(listener), - async move { - shutdown.await; - service.cancel_all(); - let _ = draining.send(()); - }, - ); + let server = router.serve_with_incoming_shutdown(tcp_incoming(listener), async move { + shutdown.await; + service.cancel_all(); + let _ = draining.send(()); + }); tokio::pin!(server); tokio::select! { result = &mut server => result, @@ -63,3 +76,18 @@ pub async fn serve_with_engine( } } } + +#[cfg(test)] +mod tests { + use futures::StreamExt; + + #[tokio::test] + async fn accepted_connections_disable_nagle() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let mut incoming = super::tcp_incoming(listener); + let _client = tokio::net::TcpStream::connect(address).await.unwrap(); + let accepted = incoming.next().await.unwrap().unwrap(); + assert!(accepted.nodelay().unwrap()); + } +} diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index 87460eed..968f3ffb 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -41,6 +41,25 @@ const MAX_RESULT_BYTES: usize = crate::schema::MAX_RESULT_BYTES; const MAX_RELEASED_SESSIONS: usize = 4096; const RELEASE_WAIT: std::time::Duration = std::time::Duration::from_secs(2); +fn require_canonical_uuid(value: &str, name: &str) -> Result<(), Status> { + if Uuid::parse_str(value).is_ok_and(|id| id.hyphenated().to_string() == value) { + Ok(()) + } else { + Err(Status::invalid_argument(format!( + "{name} must be a canonical lowercase hyphenated UUID" + ))) + } +} + +async fn catch_engine_panic( + action: impl std::future::Future> + Send, +) -> Result { + AssertUnwindSafe(action) + .catch_unwind() + .await + .unwrap_or_else(|_| Err(Status::internal("engine request panicked"))) +} + struct Session { id: String, backend: Arc, @@ -172,7 +191,7 @@ impl ConnectService { } fn key(id: &str, user: Option<&proto::UserContext>) -> Result { - Uuid::parse_str(id).map_err(|_| Status::invalid_argument("session_id must be a UUID"))?; + require_canonical_uuid(id, "session_id")?; Ok(( user.map(|u| u.user_id.clone()).unwrap_or_default(), id.into(), @@ -375,7 +394,7 @@ fn encode_batch(batch: &RecordBatch) -> Result, Status> { // PySpark assembles one table from all IPC batches. Its physical types and // nullability must agree across batches, while field/schema metadata may differ. -fn check_output_schema(data: &[u8], expected: &mut Option) -> Result<(), Status> { +fn check_output_schema(schema: &Schema, expected: &mut Option) -> Result<(), Status> { use arrow::datatypes::{DataType, Field, FieldRef}; fn field(input: &FieldRef) -> FieldRef { @@ -396,9 +415,6 @@ fn check_output_schema(data: &[u8], expected: &mut Option) -> Result<(), } } - let schema = StreamReader::try_new(data, None) - .map_err(|error| Status::internal(format!("invalid output IPC: {error}")))? - .schema(); let actual = Schema::new(schema.fields().iter().map(field).collect::>()); match expected { Some(expected) if *expected != actual => Err(Status::internal( @@ -413,17 +429,43 @@ fn check_output_schema(data: &[u8], expected: &mut Option) -> Result<(), } fn estimated_rows_per_batch(batch: &RecordBatch) -> usize { - let bytes = batch.get_array_memory_size(); + use arrow::datatypes::DataType; + + fn contains_view(data_type: &DataType) -> bool { + match data_type { + DataType::Utf8View | DataType::BinaryView => true, + DataType::List(field) | DataType::LargeList(field) | DataType::Map(field, _) => { + contains_view(field.data_type()) + } + DataType::Struct(fields) => fields.iter().any(|field| contains_view(field.data_type())), + _ => false, + } + } + + let bytes = batch.columns().iter().fold(0usize, |bytes, column| { + // ArrayData's slice estimate omits variadic view buffers. Keep the + // conservative allocation estimate for those until encoding measures + // their materialized payload; ordinary slices need only their own bytes. + let size = if contains_view(column.data_type()) { + column.get_array_memory_size() + } else { + column + .to_data() + .get_slice_memory_size() + .unwrap_or_else(|_| column.get_array_memory_size()) + }; + bytes.saturating_add(size) + }); // Start below the exact IPC limit to allow schema/offset overhead. This is // only an estimate: skewed rows and annotations are checked after encoding. - // Include retained backing allocations so even sliced/view inputs begin - // conservatively without materializing an entire oversized engine batch. - rows_for_bytes(batch.num_rows(), bytes) + // Nested children may still retain unsliced buffers, making the initial + // estimate conservative. Successful encodings can grow the next row budget. + rows_for_bytes(batch.num_rows(), bytes).min(batch.num_rows().max(1)) } fn rows_for_bytes(rows: usize, bytes: usize) -> usize { (rows as u128 * (MAX_RESULT_BYTES / 2) as u128 / bytes.max(1) as u128) - .clamp(1, rows.max(1) as u128) as usize + .clamp(1, usize::MAX as u128) as usize } // Fields drop in declaration order, including when encoding panics or a @@ -445,6 +487,7 @@ async fn send_batch( id: &str, operation: &str, batch: ResultBatch, + batch_schema: &Schema, offset: &mut i64, output_schema: &mut Option, guard: &Arc, @@ -473,7 +516,7 @@ async fn send_batch( )), ); if result.encoded_len() <= MAX_RESULT_BYTES { - check_output_schema(&batch.data, output_schema)?; + check_output_schema(batch_schema, output_schema)?; *offset = offset .checked_add(row_count) .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; @@ -586,7 +629,17 @@ async fn send_arrow_batch( pending.push((batch.clone(), rows_per_batch)); continue; } - check_output_schema(&payload.data, output_schema)?; + let emitted_schema = StreamReader::try_new(&payload.data[..], None) + .map_err(|error| Status::internal(format!("invalid output IPC: {error}")))? + .schema(); + check_output_schema(emitted_schema.as_ref(), output_schema)?; + if let Some((_, next_rows)) = pending.last_mut() { + // A retained backing allocation can dwarf this slice's IPC. Adapt + // from actual output, but limit growth so one tiny sample cannot + // request an arbitrarily large/skewed materialization next. + *next_rows = rows_for_bytes(batch.num_rows(), result_bytes) + .min(batch.num_rows().saturating_mul(4).max(1)); + } *offset = offset .checked_add(batch.num_rows() as i64) .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; @@ -610,8 +663,7 @@ impl SparkConnectService for ConnectService { let operation = req .operation_id .unwrap_or_else(|| Uuid::new_v4().to_string()); - Uuid::parse_str(&operation) - .map_err(|_| Status::invalid_argument("operation_id must be a UUID"))?; + require_canonical_uuid(&operation, "operation_id")?; let op = req .plan .and_then(|p| p.op_type) @@ -631,7 +683,7 @@ impl SparkConnectService for ConnectService { biased; _ = cancelled.changed() => return Err(Status::cancelled("session released")), _ = operation_cancelled.changed() => return Err(Status::cancelled("operation released")), - prepared = session.backend.prepare(op) => prepared?, + prepared = catch_engine_panic(async { session.backend.prepare(op).await }) => prepared?, }; let query = match prepared { PreparedOperation::Query(query) => query, @@ -697,6 +749,7 @@ impl SparkConnectService for ConnectService { &req.session_id, &operation, batch, + batch_schema.as_ref(), &mut offset, &mut output_schema, &guard, @@ -712,11 +765,13 @@ impl SparkConnectService for ConnectService { drop(batches); if !sent { let batch = RecordBatch::new_empty(arrow_schema); + let batch_schema = batch.schema(); if !send_batch( &sender, &req.session_id, &operation, batch.into(), + batch_schema.as_ref(), &mut offset, &mut output_schema, &guard, @@ -771,7 +826,11 @@ impl SparkConnectService for ConnectService { req.user_context.as_ref(), req.client_observed_server_side_session_id.as_deref(), )?; - let result = session.while_open(session.backend.analyze(input)).await?; + let result = session + .while_open(catch_engine_panic(async { + session.backend.analyze(input).await + })) + .await?; if let proto::analyze_plan_response::Result::Schema(result) = &result { if let Some(schema) = &result.schema { schema::validate_schema_size(schema)?; @@ -798,7 +857,11 @@ impl SparkConnectService for ConnectService { req.user_context.as_ref(), req.client_observed_server_side_session_id.as_deref(), )?; - let result = session.while_open(session.backend.config(input)).await?; + let result = session + .while_open(catch_engine_panic(async { + session.backend.config(input).await + })) + .await?; Ok(Response::new(proto::ConfigResponse { session_id: req.session_id, server_side_session_id: session.id.clone(), @@ -812,8 +875,7 @@ impl SparkConnectService for ConnectService { request: Request, ) -> Result, Status> { let req = request.into_inner(); - Uuid::parse_str(&req.operation_id) - .map_err(|_| Status::invalid_argument("operation_id must be a UUID"))?; + require_canonical_uuid(&req.operation_id, "operation_id")?; let session = self.find_session( &req.session_id, req.user_context.as_ref(), @@ -1036,6 +1098,54 @@ mod tests { ); } + #[tokio::test] + async fn successful_encoding_grows_a_conservative_row_budget() { + use arrow::array::Int64Array; + use arrow::datatypes::{DataType, Field, Schema}; + + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), + vec![Arc::new(Int64Array::from_iter_values(0..1000))], + ) + .unwrap(); + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("adaptive").unwrap(); + let (sender, mut receiver) = mpsc::channel(1024); + let mut offset = 0; + let mut output_schema = None; + assert!(send_arrow_batch( + &sender, + "session", + "adaptive", + batch, + 1, + &mut offset, + &mut output_schema, + &guard, + ) + .await + .unwrap()); + let (mut batches, mut rows) = (0, 0); + while let Ok(result) = receiver.try_recv() { + let result = result.unwrap(); + assert!(result.encoded_len() <= MAX_RESULT_BYTES); + let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + result.response_type + else { + panic!("expected Arrow response"); + }; + assert_eq!(batch.start_offset, Some(rows)); + batches += 1; + rows += batch.row_count; + } + assert_eq!(rows, 1000); + assert_eq!(offset, rows); + assert!( + batches <= 8, + "a conservative estimate must adapt, got {batches} batches" + ); + } + #[tokio::test] async fn existing_ipc_is_forwarded_unchanged() { use arrow::array::Int64Array; @@ -1071,6 +1181,7 @@ mod tests { data: data.clone(), row_count: 10 }, + batch.schema().as_ref(), &mut offset, &mut output_schema, &guard, @@ -1125,6 +1236,7 @@ mod tests { data: data.clone(), row_count: 50_000 }, + batch.schema().as_ref(), &mut offset, &mut output_schema, &guard, @@ -1170,6 +1282,7 @@ mod tests { data: data.clone(), row_count, }, + batch.schema().as_ref(), &mut offset, &mut output_schema, &guard, diff --git a/crates/spark-connect-server/tests/review_regressions.rs b/crates/spark-connect-server/tests/review_regressions.rs new file mode 100644 index 00000000..e821bd81 --- /dev/null +++ b/crates/spark-connect-server/tests/review_regressions.rs @@ -0,0 +1,425 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Regressions for sliced output, protocol IDs and backend panic boundaries. + +#[path = "support/server.rs"] +mod server; + +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::Int64Array; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::ipc::reader::StreamReader; +use arrow::record_batch::RecordBatch; +use prost::Message; +use server::Server; +use spark_connect_proto as proto; +use spark_connect_server::engine::{ + BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, + ResultBatch, SessionContext, +}; +use tonic::{Code, Status}; +use uuid::Uuid; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Call { + Prepare, + Analyze, + Config, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum PanicAt { + Construction, + Poll, +} + +struct State { + batches: Vec, + panic: Option<(Call, PanicAt)>, + panicked: AtomicBool, + sessions: AtomicUsize, + prepares: AtomicUsize, +} + +impl State { + fn new(batches: Vec, panic: Option<(Call, PanicAt)>) -> Arc { + Arc::new(Self { + batches, + panic, + panicked: AtomicBool::new(false), + sessions: AtomicUsize::new(0), + prepares: AtomicUsize::new(0), + }) + } + + fn panic_once(&self, call: Call) -> Option { + self.panic + .filter(|(target, _)| *target == call) + .filter(|_| !self.panicked.swap(true, Ordering::SeqCst)) + .map(|(_, at)| at) + } +} + +struct Fixture(Arc); + +impl Engine for Fixture { + fn create_session(&self, _: SessionContext) -> Result, Status> { + self.0.sessions.fetch_add(1, Ordering::SeqCst); + Ok(Arc::new(Self(self.0.clone()))) + } +} + +type EngineFuture<'a, T> = Pin> + Send + 'a>>; + +// Implement the expanded async-trait signatures directly so the fixture can +// panic both before returning a future and while that future is polled. +impl EngineSession for Fixture { + fn prepare<'life0, 'async_trait>( + &'life0 self, + _: proto::plan::OpType, + ) -> EngineFuture<'async_trait, PreparedOperation> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + self.0.prepares.fetch_add(1, Ordering::SeqCst); + let panic = self.0.panic_once(Call::Prepare); + assert!(panic != Some(PanicAt::Construction), "prepare construction"); + Box::pin(async move { + assert!(panic != Some(PanicAt::Poll), "prepare poll"); + Ok(PreparedOperation::Query(Box::new(Query( + self.0.batches.clone(), + )))) + }) + } + + fn analyze<'life0, 'async_trait>( + &'life0 self, + _: proto::analyze_plan_request::Analyze, + ) -> EngineFuture<'async_trait, proto::analyze_plan_response::Result> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + let panic = self.0.panic_once(Call::Analyze); + assert!(panic != Some(PanicAt::Construction), "analyze construction"); + Box::pin(async move { + assert!(panic != Some(PanicAt::Poll), "analyze poll"); + Ok(proto::analyze_plan_response::Result::SparkVersion( + proto::analyze_plan_response::SparkVersion { + version: "fixture".into(), + }, + )) + }) + } + + fn config<'life0, 'async_trait>( + &'life0 self, + _: proto::config_request::operation::OpType, + ) -> EngineFuture<'async_trait, ConfigResult> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + let panic = self.0.panic_once(Call::Config); + assert!(panic != Some(PanicAt::Construction), "config construction"); + Box::pin(async move { + assert!(panic != Some(PanicAt::Poll), "config poll"); + Ok(ConfigResult::default()) + }) + } +} + +struct Query(Vec); + +#[tonic::async_trait] +impl PreparedQuery for Query { + fn schema(&self) -> SchemaRef { + self.0[0].schema() + } + + async fn execute(self: Box) -> Result { + Ok(Box::pin(futures::stream::iter( + self.0 + .into_iter() + .map(|batch| Ok(ResultBatch::Arrow(batch))), + ))) + } +} + +fn integers(rows: usize) -> RecordBatch { + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), + vec![Arc::new(Int64Array::from_iter_values(0..rows as i64))], + ) + .unwrap() +} + +fn execute(session: &str, operation: &str) -> proto::ExecutePlanRequest { + proto::ExecutePlanRequest { + session_id: session.into(), + operation_id: Some(operation.into()), + plan: Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(proto::Relation::default())), + }), + ..Default::default() + } +} + +fn config(session: &str) -> proto::ConfigRequest { + proto::ConfigRequest { + session_id: session.into(), + operation: Some(proto::config_request::Operation { + op_type: Some(proto::config_request::operation::OpType::GetAll( + Default::default(), + )), + }), + ..Default::default() + } +} + +fn analyze(session: &str) -> proto::AnalyzePlanRequest { + proto::AnalyzePlanRequest { + session_id: session.into(), + analyze: Some(proto::analyze_plan_request::Analyze::SparkVersion( + Default::default(), + )), + ..Default::default() + } +} + +fn release(session: &str, operation: &str) -> proto::ReleaseExecuteRequest { + proto::ReleaseExecuteRequest { + session_id: session.into(), + operation_id: operation.into(), + release: Some(proto::release_execute_request::Release::ReleaseAll( + Default::default(), + )), + ..Default::default() + } +} + +async fn collect( + server: &mut Server, + session: &str, + operation: &str, +) -> Vec { + tokio::time::timeout(Duration::from_secs(15), async { + let mut stream = server + .client + .execute_plan(execute(session, operation)) + .await + .unwrap() + .into_inner(); + let mut output = Vec::new(); + while let Some(message) = stream.message().await.unwrap() { + output.push(message); + } + assert!(matches!( + output.last().unwrap().response_type.as_ref(), + Some(proto::execute_plan_response::ResponseType::ResultComplete( + _ + )) + )); + output + }) + .await + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn slices_of_a_large_allocation_remain_one_response_per_small_batch() { + let backing = integers(10_000_000); + let batches = (0..20) + .map(|index| backing.slice(index * 1000, 1000)) + .collect(); + let state = State::new(batches, None); + let mut server = Server::start_with_engine(Arc::new(Fixture(state))).await; + let output = collect( + &mut server, + &Uuid::new_v4().to_string(), + &Uuid::new_v4().to_string(), + ) + .await; + let (mut batches, mut rows) = (0, 0); + for message in output { + assert!(message.encoded_len() <= 3 * 1024 * 1024); + if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + message.response_type + { + batches += 1; + assert_eq!(batch.start_offset, Some(rows)); + assert_eq!(batch.row_count, 1000); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + let decoded = reader.next().unwrap().unwrap(); + let values = decoded + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(values.values().iter().copied().eq(rows..rows + 1000)); + assert!(reader.next().is_none()); + rows += batch.row_count; + } + } + assert_eq!(batches, 20); + assert_eq!(rows, 20_000); + server.stop().await; +} + +#[tokio::test] +async fn noncanonical_ids_are_rejected_without_creating_or_releasing_state() { + let state = State::new(vec![integers(1)], None); + let mut server = Server::start_with_engine(Arc::new(Fixture(state.clone()))).await; + let uuid = Uuid::parse_str("abcdef12-3456-4789-abcd-0123456789ab").unwrap(); + let session = uuid.to_string(); + let operation = Uuid::new_v4().to_string(); + let original = server + .client + .config(config(&session)) + .await + .unwrap() + .into_inner() + .server_side_session_id; + for alias in [ + uuid.simple().to_string(), + uuid.braced().to_string(), + uuid.urn().to_string(), + session.to_uppercase(), + ] { + assert_eq!( + server + .client + .config(config(&alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .analyze_plan(analyze(&alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .execute_plan(execute(&alias, &operation)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_execute(release(&alias, &operation)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_session(proto::ReleaseSessionRequest { + session_id: alias.clone(), + ..Default::default() + }) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .execute_plan(execute(&Uuid::new_v4().to_string(), &alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_execute(release(&session, &alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .config(config(&session)) + .await + .unwrap() + .into_inner() + .server_side_session_id, + original + ); + } + assert_eq!(state.sessions.load(Ordering::SeqCst), 1); + assert_eq!(state.prepares.load(Ordering::SeqCst), 0); + collect(&mut server, &session, &operation).await; + server.stop().await; +} + +#[tokio::test] +async fn backend_request_panics_have_internal_status_and_leave_the_session_usable() { + for call in [Call::Prepare, Call::Analyze, Call::Config] { + for at in [PanicAt::Construction, PanicAt::Poll] { + let state = State::new(vec![integers(1)], Some((call, at))); + let mut server = Server::start_with_engine(Arc::new(Fixture(state.clone()))).await; + let session = Uuid::new_v4().to_string(); + let operation = Uuid::new_v4().to_string(); + let status = match call { + Call::Prepare => server + .client + .execute_plan(execute(&session, &operation)) + .await + .unwrap_err(), + Call::Analyze => server + .client + .analyze_plan(analyze(&session)) + .await + .unwrap_err(), + Call::Config => server.client.config(config(&session)).await.unwrap_err(), + }; + assert_eq!(status.code(), Code::Internal); + assert_eq!(status.message(), "engine request panicked"); + server.client.config(config(&session)).await.unwrap(); + server.client.analyze_plan(analyze(&session)).await.unwrap(); + // Reuse the operation ID after a preparation panic: its reservation + // must have been released before the failed RPC returned. + collect(&mut server, &session, &operation).await; + assert_eq!(state.sessions.load(Ordering::SeqCst), 1); + server.stop().await; + } + } +} diff --git a/crates/spark-connect-server/tests/server_shutdown.rs b/crates/spark-connect-server/tests/server_shutdown.rs new file mode 100644 index 00000000..ac624c93 --- /dev/null +++ b/crates/spark-connect-server/tests/server_shutdown.rs @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Duration; + +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use futures::Stream; +use spark_connect_proto as proto; +use spark_connect_server::engine::{ + BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, + ResultBatch, SessionContext, +}; +use tonic::Status; + +#[derive(Default)] +struct Spy { + streams_started: AtomicUsize, + streams_dropped: AtomicUsize, + configs_started: AtomicUsize, + configs_dropped: AtomicUsize, + sessions_dropped: AtomicUsize, +} + +struct PendingEngine(Arc); + +impl Engine for PendingEngine { + fn create_session(&self, _context: SessionContext) -> Result, Status> { + Ok(Arc::new(PendingSession(self.0.clone()))) + } +} + +struct PendingSession(Arc); + +impl Drop for PendingSession { + fn drop(&mut self) { + self.0.sessions_dropped.fetch_add(1, Ordering::SeqCst); + } +} + +#[tonic::async_trait] +impl EngineSession for PendingSession { + async fn prepare(&self, _operation: proto::plan::OpType) -> Result { + Ok(PreparedOperation::Query(Box::new(PendingQuery( + self.0.clone(), + )))) + } + + async fn analyze( + &self, + _input: proto::analyze_plan_request::Analyze, + ) -> Result { + Err(Status::unimplemented("shutdown fixture does not analyze")) + } + + async fn config( + &self, + _input: proto::config_request::operation::OpType, + ) -> Result { + let _call = PendingConfig(self.0.clone()); + self.0.configs_started.fetch_add(1, Ordering::SeqCst); + std::future::pending().await + } +} + +struct PendingConfig(Arc); + +impl Drop for PendingConfig { + fn drop(&mut self) { + self.0.configs_dropped.fetch_add(1, Ordering::SeqCst); + } +} + +struct PendingQuery(Arc); + +#[tonic::async_trait] +impl PreparedQuery for PendingQuery { + fn schema(&self) -> SchemaRef { + Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])) + } + + async fn execute(self: Box) -> Result { + self.0.streams_started.fetch_add(1, Ordering::SeqCst); + Ok(Box::pin(PendingResults(self.0.clone()))) + } +} + +struct PendingResults(Arc); + +impl Stream for PendingResults { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Pending + } +} + +impl Drop for PendingResults { + fn drop(&mut self) { + self.0.streams_dropped.fetch_add(1, Ordering::SeqCst); + } +} + +async fn wait_for_count(counter: &AtomicUsize, expected: usize) { + tokio::time::timeout(Duration::from_secs(5), async { + while counter.load(Ordering::SeqCst) != expected { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("backend work must finish without disconnecting its client"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn aborting_serve_cancels_backend_work_before_clients_disconnect() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let spy = Arc::new(Spy::default()); + let server = tokio::spawn(spark_connect_server::serve_with_engine( + listener, + Arc::new(PendingEngine(spy.clone())), + std::future::pending(), + )); + let mut client = proto::spark_connect_service_client::SparkConnectServiceClient::connect( + format!("http://{address}"), + ) + .await + .unwrap(); + let session = uuid::Uuid::new_v4().to_string(); + let mut result = client + .execute_plan(proto::ExecutePlanRequest { + session_id: session.clone(), + operation_id: Some(uuid::Uuid::new_v4().to_string()), + plan: Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(proto::Relation::default())), + }), + ..Default::default() + }) + .await + .unwrap() + .into_inner(); + assert!(result.message().await.unwrap().unwrap().schema.is_some()); + wait_for_count(&spy.streams_started, 1).await; + + let mut config_client = client.clone(); + let config = tokio::spawn(async move { + config_client + .config(proto::ConfigRequest { + session_id: session, + operation: Some(proto::config_request::Operation { + op_type: Some(proto::config_request::operation::OpType::GetAll( + Default::default(), + )), + }), + ..Default::default() + }) + .await + }); + wait_for_count(&spy.configs_started, 1).await; + server.abort(); + assert!(server.await.unwrap_err().is_cancelled()); + + // Retain both the streamed response and its client connection: cancellation + // must come from dropping serve_with_engine, not from a peer disconnect. + wait_for_count(&spy.streams_dropped, 1).await; + wait_for_count(&spy.configs_dropped, 1).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert!(tokio::time::timeout(Duration::from_secs(5), config) + .await + .unwrap() + .unwrap() + .is_err()); + drop(result); + drop(client); +} From 4cadf0705f7ca3d042c8d5bb2a389866b63438ca Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 15:40:04 -0800 Subject: [PATCH 05/17] Size nested slices before Arrow encoding --- crates/spark-connect-core/Cargo.toml | 2 +- crates/spark-connect-server/Cargo.toml | 2 +- crates/spark-connect-server/src/engine/mod.rs | 3 + crates/spark-connect-server/src/service.rs | 264 ++++++++++++++++-- .../tests/review_regressions.rs | 189 ++++++++++++- 5 files changed, 415 insertions(+), 45 deletions(-) diff --git a/crates/spark-connect-core/Cargo.toml b/crates/spark-connect-core/Cargo.toml index a34fb898..9d4e6c9d 100644 --- a/crates/spark-connect-core/Cargo.toml +++ b/crates/spark-connect-core/Cargo.toml @@ -32,7 +32,7 @@ prost-types.workspace = true crc32fast = "1" sha2 = "0.10" # Transport errors that tonic keeps as a status's source, for `RetryPolicy::can_retry`. -h2 = "0.4" +h2.workspace = true hyper = "1" # The Basic credential and the percent-decoded userinfo of a proxy URI, already built for tonic. base64 = "0.22" diff --git a/crates/spark-connect-server/Cargo.toml b/crates/spark-connect-server/Cargo.toml index a923d66e..5fe56bcc 100644 --- a/crates/spark-connect-server/Cargo.toml +++ b/crates/spark-connect-server/Cargo.toml @@ -33,7 +33,7 @@ arrow.workspace = true futures.workspace = true prost.workspace = true tokio = { workspace = true, features = ["net"] } -tokio-stream = { workspace = true, features = ["net"] } +tokio-stream.workspace = true tonic = { workspace = true, features = ["server", "router"] } tonic-prost.workspace = true uuid.workspace = true diff --git a/crates/spark-connect-server/src/engine/mod.rs b/crates/spark-connect-server/src/engine/mod.rs index d2f44b46..64f6a8a1 100644 --- a/crates/spark-connect-server/src/engine/mod.rs +++ b/crates/spark-connect-server/src/engine/mod.rs @@ -57,6 +57,9 @@ pub trait Engine: Send + Sync { /// Dropping a method future must cancel its work. Dropping the final session /// reference releases its resources after in-flight operations have finished. /// Resource cleanup must not panic. +/// With unwinding enabled, panics in these methods become `INTERNAL` errors; +/// the frontend keeps the session available for later requests. Implementations +/// must leave their state usable when unwinding. // async_trait repeats Future's must_use annotation on generated trait methods. #[allow(clippy::double_must_use)] #[tonic::async_trait] diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index 968f3ffb..e57bbcc8 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -70,6 +70,8 @@ struct Session { struct Operation { cancelled: watch::Sender, done: watch::Sender, + #[cfg(test)] + encoding_attempts: std::sync::atomic::AtomicUsize, } impl Session { @@ -107,6 +109,8 @@ impl Session { let running = Arc::new(Operation { cancelled, done: watch::channel(false).0, + #[cfg(test)] + encoding_attempts: std::sync::atomic::AtomicUsize::new(0), }); let mut operations = self .operations @@ -428,38 +432,70 @@ fn check_output_schema(schema: &Schema, expected: &mut Option) -> Result } } -fn estimated_rows_per_batch(batch: &RecordBatch) -> usize { +fn estimated_array_bytes(array: &dyn arrow::array::Array) -> usize { + use arrow::array::{Array, AsArray, OffsetSizeTrait}; use arrow::datatypes::DataType; - fn contains_view(data_type: &DataType) -> bool { - match data_type { - DataType::Utf8View | DataType::BinaryView => true, - DataType::List(field) | DataType::LargeList(field) | DataType::Map(field, _) => { - contains_view(field.data_type()) - } - DataType::Struct(fields) => fields.iter().any(|field| contains_view(field.data_type())), - _ => false, - } + fn nested_bytes(offsets: &[O], values: &dyn Array) -> usize { + let start = offsets[0].as_usize(); + let end = offsets[offsets.len() - 1].as_usize(); + // List and map slices retain their full child arrays. Like the IPC + // writer, inspect only the child range referenced by these offsets. + offsets + .len() + .saturating_mul(std::mem::size_of::()) + .saturating_add(estimated_array_bytes( + values.slice(start, end - start).as_ref(), + )) } - let bytes = batch.columns().iter().fold(0usize, |bytes, column| { - // ArrayData's slice estimate omits variadic view buffers. Keep the - // conservative allocation estimate for those until encoding measures - // their materialized payload; ordinary slices need only their own bytes. - let size = if contains_view(column.data_type()) { - column.get_array_memory_size() - } else { - column + fn view_bytes(len: usize, lengths: impl Iterator) -> usize { + // Encoding materializes views with 64-bit offsets. Count repeated + // references separately, including null-slot lengths: Arrow's cast + // reserves their bytes even though it emits no value for a null slot. + lengths.fold(len.saturating_add(1).saturating_mul(8), |bytes, len| { + bytes.saturating_add(len as usize) + }) + } + + let bytes = match array.data_type() { + DataType::List(_) => { + let list = array.as_list::(); + nested_bytes(list.value_offsets(), list.values().as_ref()) + } + DataType::LargeList(_) => { + let list = array.as_list::(); + nested_bytes(list.value_offsets(), list.values().as_ref()) + } + DataType::Map(_, _) => { + let map = array.as_map(); + nested_bytes(map.value_offsets(), map.entries()) + } + DataType::Struct(_) => array + .as_struct() + .columns() + .iter() + .fold(0usize, |bytes, child| { + bytes.saturating_add(estimated_array_bytes(child.as_ref())) + }), + DataType::Utf8View => view_bytes(array.len(), array.as_string_view().lengths()), + DataType::BinaryView => view_bytes(array.len(), array.as_binary_view().lengths()), + _ => { + return array .to_data() .get_slice_memory_size() - .unwrap_or_else(|_| column.get_array_memory_size()) - }; - bytes.saturating_add(size) + .unwrap_or_else(|_| array.get_array_memory_size()); + } + }; + bytes.saturating_add(array.nulls().map_or(0, |_| array.len().div_ceil(8))) +} + +fn estimated_rows_per_batch(batch: &RecordBatch) -> usize { + let bytes = batch.columns().iter().fold(0usize, |bytes, column| { + bytes.saturating_add(estimated_array_bytes(column.as_ref())) }); // Start below the exact IPC limit to allow schema/offset overhead. This is // only an estimate: skewed rows and annotations are checked after encoding. - // Nested children may still retain unsliced buffers, making the initial - // estimate conservative. Successful encodings can grow the next row budget. rows_for_bytes(batch.num_rows(), bytes).min(batch.num_rows().max(1)) } @@ -477,6 +513,11 @@ struct EncodingBatch { impl EncodingBatch { fn encode(self) -> Result<(Self, Vec), Status> { + #[cfg(test)] + self._guard + .running + .encoding_attempts + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); let data = encode_batch(&self.batch)?; Ok((self, data)) } @@ -577,9 +618,29 @@ async fn send_arrow_batch( ) -> Result { let mut pending = vec![(batch, rows_per_batch)]; while let Some((batch, rows_per_batch)) = pending.pop() { + let mut rows_per_batch = rows_per_batch.max(1).min(batch.num_rows().max(1)); + while rows_per_batch > 1 { + let prefix; + let candidate = if rows_per_batch >= batch.num_rows() { + &batch + } else { + prefix = batch.slice(0, rows_per_batch); + &prefix + }; + let estimated = estimated_rows_per_batch(candidate); + if estimated >= candidate.num_rows() { + break; + } + // Growth from a light prefix can reach a dense tail. Inspect its + // logical buffers before encoding rather than repeatedly copying + // an already-known oversized payload. Exact IPC/envelope checks + // below still handle schema overhead and an unfittable single row. + rows_per_batch = estimated; + } // Keep the remainder as one slice instead of allocating a pending entry // per row in a large engine batch. - let batch = if batch.num_rows() > rows_per_batch { + let has_remainder = batch.num_rows() > rows_per_batch; + let batch = if has_remainder { pending.push(( batch.slice(rows_per_batch, batch.num_rows() - rows_per_batch), rows_per_batch, @@ -633,12 +694,14 @@ async fn send_arrow_batch( .map_err(|error| Status::internal(format!("invalid output IPC: {error}")))? .schema(); check_output_schema(emitted_schema.as_ref(), output_schema)?; - if let Some((_, next_rows)) = pending.last_mut() { - // A retained backing allocation can dwarf this slice's IPC. Adapt - // from actual output, but limit growth so one tiny sample cannot - // request an arbitrarily large/skewed materialization next. - *next_rows = rows_for_bytes(batch.num_rows(), result_bytes) - .min(batch.num_rows().saturating_mul(4).max(1)); + if has_remainder { + if let Some((_, next_rows)) = pending.last_mut() { + // Adapt this piece's own remainder, without changing an outer + // sibling when the piece finishes. The row-growth limit is a + // sampling heuristic, not a bound on bytes for skewed rows. + *next_rows = rows_for_bytes(batch.num_rows(), result_bytes) + .min(batch.num_rows().saturating_mul(4).max(1)); + } } *offset = offset .checked_add(batch.num_rows() as i64) @@ -1098,6 +1161,31 @@ mod tests { ); } + #[test] + fn view_estimates_count_repeated_and_null_slot_lengths() { + use arrow::array::{BinaryViewArray, StringViewArray}; + use arrow::buffer::NullBuffer; + + let source = StringViewArray::from(vec!["abcdefghijklmno"]); + let (views, buffers, _) = source.into_parts(); + let repeated = vec![views[0]; 4].into(); + let nulls = Some(NullBuffer::from(vec![true, false, true, false])); + let strings = StringViewArray::new(repeated, buffers, nulls); + // Four logical values reserve four payloads during materialization, + // even though they share one buffer and two slots are null. IPC also + // needs five 64-bit offsets and a validity byte. + let expected = 4 * 15 + 5 * 8 + 1; + assert_eq!(estimated_array_bytes(&strings), expected); + assert_eq!(strings.total_bytes_len(), 2 * 15); + let (views, buffers, nulls) = strings.into_parts(); + let binary = BinaryViewArray::new(views, buffers, nulls); + assert_eq!(estimated_array_bytes(&binary), expected); + assert_eq!( + estimated_array_bytes(&binary.slice(1, 2)), + 2 * 15 + 3 * 8 + 1 + ); + } + #[tokio::test] async fn successful_encoding_grows_a_conservative_row_budget() { use arrow::array::Int64Array; @@ -1146,6 +1234,122 @@ mod tests { ); } + #[tokio::test] + async fn skewed_output_checks_prefixes_and_recovers_light_suffixes() { + use arrow::array::{Int64Array, LargeStringArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use std::sync::atomic::Ordering; + + for shape in ["large_tail", "large_head", "dense_middle"] { + let (prefix_rows, suffix_rows) = match shape { + "large_tail" => (5000, 0), + "large_head" => (0, 5000), + _ => (100_000, 100_000), + }; + let mut sizes = vec![0usize; prefix_rows]; + sizes.extend(std::iter::repeat_n(1024 * 1024, 40)); + sizes.extend(std::iter::repeat_n(0, suffix_rows)); + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::LargeUtf8, false), + ])), + vec![ + Arc::new(Int64Array::from_iter_values(0..sizes.len() as i64)), + Arc::new(LargeStringArray::from_iter_values( + sizes.iter().map(|size| "z".repeat(*size)), + )), + ], + ) + .unwrap(); + let budget = estimated_rows_per_batch(&batch); + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("skewed").unwrap(); + // Even an accidentally permanent one-row budget must not block + // this regression before its response-count assertion can fail. + let (sender, mut receiver) = mpsc::channel(sizes.len()); + let mut offset = 0; + let mut output_schema = None; + assert!(tokio::time::timeout( + std::time::Duration::from_secs(30), + send_arrow_batch( + &sender, + "session", + "skewed", + batch, + budget, + &mut offset, + &mut output_schema, + &guard, + ) + ) + .await + .unwrap() + .unwrap()); + let (mut responses, mut empty_responses, mut rows) = (0usize, 0usize, 0usize); + while let Ok(result) = receiver.try_recv() { + let result = result.unwrap(); + assert!(result.encoded_len() <= MAX_RESULT_BYTES); + let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + result.response_type + else { + panic!("expected Arrow response"); + }; + assert_eq!(batch.start_offset, Some(rows as i64)); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + let decoded = reader.next().unwrap().unwrap(); + assert_eq!(decoded.num_rows() as i64, batch.row_count); + assert!(reader.next().is_none()); + let ids = decoded + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let strings = decoded + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let mut empty = true; + for index in 0..decoded.num_rows() { + assert_eq!(ids.value(index), rows as i64); + let value = strings.value(index); + assert_eq!(value.len(), sizes[rows]); + assert!(value.bytes().all(|byte| byte == b'z')); + empty &= value.is_empty(); + rows += 1; + } + responses += 1; + empty_responses += usize::from(empty); + } + assert_eq!(rows, sizes.len()); + assert_eq!(offset, rows as i64); + let attempts = guard.running.encoding_attempts.load(Ordering::SeqCst); + // Logical prefix checks must avoid repeatedly materializing the + // same unconsumed dense region after successes on empty prefixes. + assert!(attempts <= responses + 4, + "{shape}: {attempts} encodes for {responses} responses repeatedly encoded an oversized region"); + if shape == "dense_middle" { + // A sticky reduced row budget also fragments a light suffix + // inside a failed piece. Let it recover once the dense rows + // are consumed, as well as across outer sibling slices. + assert!( + responses < 80, + "dense-middle output fragmented into {responses} responses" + ); + assert!( + empty_responses <= 35, + "dense-middle output emitted {empty_responses} all-empty responses" + ); + } else if shape == "large_head" { + assert!( + responses < 100, + "large-head output fragmented into {responses} responses" + ); + } + } + } + #[tokio::test] async fn existing_ipc_is_forwarded_unchanged() { use arrow::array::Int64Array; diff --git a/crates/spark-connect-server/tests/review_regressions.rs b/crates/spark-connect-server/tests/review_regressions.rs index e821bd81..992fc8a6 100644 --- a/crates/spark-connect-server/tests/review_regressions.rs +++ b/crates/spark-connect-server/tests/review_regressions.rs @@ -26,7 +26,11 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; -use arrow::array::Int64Array; +use arrow::array::{ + Array, ArrayRef, BinaryViewArray, Int64Array, LargeBinaryArray, LargeListArray, + LargeStringArray, ListArray, MapArray, StringViewArray, StructArray, +}; +use arrow::buffer::OffsetBuffer; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::ipc::reader::StreamReader; use arrow::record_batch::RecordBatch; @@ -247,9 +251,7 @@ async fn collect( .unwrap() } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn slices_of_a_large_allocation_remain_one_response_per_small_batch() { - let backing = integers(10_000_000); +async fn collect_small_slices(backing: RecordBatch) -> Vec { let batches = (0..20) .map(|index| backing.slice(index * 1000, 1000)) .collect(); @@ -261,30 +263,191 @@ async fn slices_of_a_large_allocation_remain_one_response_per_small_batch() { &Uuid::new_v4().to_string(), ) .await; - let (mut batches, mut rows) = (0, 0); + let (mut batches, mut rows) = (Vec::new(), 0); for message in output { assert!(message.encoded_len() <= 3 * 1024 * 1024); if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = message.response_type { - batches += 1; assert_eq!(batch.start_offset, Some(rows)); assert_eq!(batch.row_count, 1000); let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); let decoded = reader.next().unwrap().unwrap(); - let values = decoded - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert!(values.values().iter().copied().eq(rows..rows + 1000)); + assert_eq!(decoded.num_rows() as i64, batch.row_count); + batches.push(decoded); assert!(reader.next().is_none()); rows += batch.row_count; } } - assert_eq!(batches, 20); + assert_eq!(batches.len(), 20); assert_eq!(rows, 20_000); server.stop().await; + batches +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn slices_of_a_large_allocation_remain_one_response_per_small_batch() { + for (index, batch) in collect_small_slices(integers(10_000_000)) + .await + .iter() + .enumerate() + { + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let start = index as i64 * 1000; + assert!(values.values().iter().copied().eq(start..start + 1000)); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn nested_slices_only_budget_the_referenced_child_range() { + // The parent needs only 20,000 rows to retain the full 80 MB child. Exercise + // each offset width and maps sequentially, sharing the same allocation. + let child: ArrayRef = Arc::new(Int64Array::from_iter_values(0..10_000_000)); + for kind in 0..3 { + let item = Arc::new(Field::new("item", DataType::Int64, false)); + let column: ArrayRef = match kind { + 0 => Arc::new(ListArray::new( + item, + OffsetBuffer::new((0..=20_000i32).collect::>().into()), + child.clone(), + None, + )), + 1 => Arc::new(LargeListArray::new( + item, + OffsetBuffer::new((0..=20_000i64).collect::>().into()), + child.clone(), + None, + )), + _ => { + let fields = vec![ + Field::new("key", DataType::Int64, false), + Field::new("value", DataType::Int64, false), + ]; + let entries = StructArray::new( + fields.clone().into(), + vec![child.clone(), child.clone()], + None, + ); + Arc::new(MapArray::new( + Arc::new(Field::new( + "entries", + DataType::Struct(fields.into()), + false, + )), + OffsetBuffer::new((0..=20_000i32).collect::>().into()), + entries, + None, + false, + )) + } + }; + let schema = Arc::new(Schema::new(vec![Field::new( + "nested", + column.data_type().clone(), + false, + )])); + let backing = RecordBatch::try_new(schema, vec![column]).unwrap(); + let batches = collect_small_slices(backing).await; + let mut row = 0i64; + for batch in batches { + for index in 0..batch.num_rows() { + let value = match kind { + 0 => batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(index), + 1 => batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(index), + _ => { + let entry = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(index); + assert_eq!(entry.len(), 1); + let key = entry + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(key.value(0), row); + entry.column(1).clone() + } + }; + let value = value.as_any().downcast_ref::().unwrap(); + assert_eq!(value.len(), 1); + assert_eq!(value.value(0), row); + row += 1; + } + } + assert_eq!(row, 20_000); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_slices_only_budget_the_current_values() { + fn text(row: usize) -> String { + format!("{row:010}{}", "y".repeat(30)) + } + + let values = StringViewArray::from_iter_values((0..1_000_000).map(text)); + let (views, buffers, nulls) = values.into_parts(); + for binary in [false, true] { + let column: ArrayRef = if binary { + Arc::new(BinaryViewArray::new( + views.clone(), + buffers.clone(), + nulls.clone(), + )) + } else { + Arc::new(StringViewArray::new( + views.clone(), + buffers.clone(), + nulls.clone(), + )) + }; + let schema = Arc::new(Schema::new(vec![Field::new( + "view", + column.data_type().clone(), + false, + )])); + let backing = RecordBatch::try_new(schema, vec![column]).unwrap(); + let batches = collect_small_slices(backing).await; + let mut row = 0; + for batch in batches { + for index in 0..batch.num_rows() { + let expected = text(row); + if binary { + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(values.value(index), expected.as_bytes()); + } else { + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(values.value(index), expected); + } + row += 1; + } + } + assert_eq!(row, 20_000); + } } #[tokio::test] From 8c4bea4ca8d194d95708411019e3ab871b727824 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 16:42:30 -0800 Subject: [PATCH 06/17] Account for fixed IPC overhead and reject row count overflow --- crates/spark-connect-server/src/service.rs | 339 +++++++++++++++++- .../tests/review_regressions.rs | 64 ++++ 2 files changed, 390 insertions(+), 13 deletions(-) diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index e57bbcc8..beb0cf81 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -491,17 +491,30 @@ fn estimated_array_bytes(array: &dyn arrow::array::Array) -> usize { } fn estimated_rows_per_batch(batch: &RecordBatch) -> usize { + estimated_rows_for_payload(batch, MAX_RESULT_BYTES / 2) +} + +fn estimated_rows_for_payload(batch: &RecordBatch, payload_bytes: usize) -> usize { let bytes = batch.columns().iter().fold(0usize, |bytes, column| { bytes.saturating_add(estimated_array_bytes(column.as_ref())) }); // Start below the exact IPC limit to allow schema/offset overhead. This is // only an estimate: skewed rows and annotations are checked after encoding. - rows_for_bytes(batch.num_rows(), bytes).min(batch.num_rows().max(1)) + rows_for_bytes(batch.num_rows(), bytes, payload_bytes).min(batch.num_rows().max(1)) } -fn rows_for_bytes(rows: usize, bytes: usize) -> usize { - (rows as u128 * (MAX_RESULT_BYTES / 2) as u128 / bytes.max(1) as u128) - .clamp(1, usize::MAX as u128) as usize +fn rows_for_bytes(rows: usize, bytes: usize, payload_bytes: usize) -> usize { + (rows as u128 * payload_bytes as u128 / bytes.max(1) as u128).clamp(1, usize::MAX as u128) + as usize +} + +fn checked_arrow_row_count(rows: usize, offset: i64) -> Result { + let rows = i64::try_from(rows) + .map_err(|_| Status::resource_exhausted("result batch row count exceeds i64"))?; + offset + .checked_add(rows) + .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; + Ok(rows) } // Fields drop in declaration order, including when encoding panics or a @@ -535,6 +548,7 @@ async fn send_batch( ) -> Result { let (batch, rows_per_batch) = match batch { ResultBatch::Arrow(batch) => { + checked_arrow_row_count(batch.num_rows(), *offset)?; let rows = estimated_rows_per_batch(&batch); (batch, rows) } @@ -542,6 +556,9 @@ async fn send_batch( if row_count < 0 { return Err(Status::internal("negative engine IPC row count")); } + let next_offset = offset + .checked_add(row_count) + .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; let batch = proto::execute_plan_response::ArrowBatch { row_count, data: data.into(), @@ -558,9 +575,7 @@ async fn send_batch( ); if result.encoded_len() <= MAX_RESULT_BYTES { check_output_schema(batch_schema, output_schema)?; - *offset = offset - .checked_add(row_count) - .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; + *offset = next_offset; return Ok(sender.send(Ok(result)).await.is_ok()); } drop(result); @@ -586,7 +601,7 @@ async fn send_batch( } // Decoded columns can share the full IPC allocation. Use its // serialized size instead of counting that allocation per column. - let rows = rows_for_bytes(batch.num_rows(), ipc_bytes); + let rows = rows_for_bytes(batch.num_rows(), ipc_bytes, MAX_RESULT_BYTES / 2); Ok((batch, rows)) }) .await @@ -616,6 +631,35 @@ async fn send_arrow_batch( output_schema: &mut Option, guard: &Arc, ) -> Result { + // Validate the complete engine batch before slicing or IPC encoding. Even + // buffer-free arrays can represent more rows than the signed protocol. + checked_arrow_row_count(batch.num_rows(), *offset)?; + // Schema metadata and wide schemas repeat in every IPC response. Measure + // their fixed cost once, using fresh empty arrays so nested view buffers + // retained by a zero-row slice cannot be materialized here. + let empty = EncodingBatch { + batch: RecordBatch::new_empty(batch.schema()), + _guard: guard.clone(), + }; + let (empty, data) = tokio::task::spawn_blocking(move || empty.encode()) + .await + .map_err(|_| Status::internal("Arrow result encoding task failed"))??; + let fixed_bytes = response( + id, + &guard.session, + operation, + Some(proto::execute_plan_response::ResponseType::ArrowBatch( + proto::execute_plan_response::ArrowBatch { + row_count: 0, + data: data.into(), + start_offset: Some(*offset), + ..Default::default() + }, + )), + ) + .encoded_len(); + drop(empty); + let payload_bytes = (MAX_RESULT_BYTES.saturating_sub(fixed_bytes) / 2).max(1); let mut pending = vec![(batch, rows_per_batch)]; while let Some((batch, rows_per_batch)) = pending.pop() { let mut rows_per_batch = rows_per_batch.max(1).min(batch.num_rows().max(1)); @@ -627,7 +671,7 @@ async fn send_arrow_batch( prefix = batch.slice(0, rows_per_batch); &prefix }; - let estimated = estimated_rows_per_batch(candidate); + let estimated = estimated_rows_for_payload(candidate, payload_bytes); if estimated >= candidate.num_rows() { break; } @@ -649,6 +693,7 @@ async fn send_arrow_batch( } else { batch }; + let row_count = checked_arrow_row_count(batch.num_rows(), *offset)?; // Arrow materialization/IPC encoding is CPU work. Retain operation // ownership if cancellation drops the JoinHandle, so ReleaseAll cannot // claim drain while a result batch is still being encoded. @@ -661,7 +706,7 @@ async fn send_arrow_batch( .map_err(|_| Status::internal("Arrow result encoding task failed"))??; let batch = &encoded.batch; let payload = proto::execute_plan_response::ArrowBatch { - row_count: batch.num_rows() as i64, + row_count, data: data.into(), start_offset: Some(*offset), ..Default::default() @@ -699,12 +744,16 @@ async fn send_arrow_batch( // Adapt this piece's own remainder, without changing an outer // sibling when the piece finishes. The row-growth limit is a // sampling heuristic, not a bound on bytes for skewed rows. - *next_rows = rows_for_bytes(batch.num_rows(), result_bytes) - .min(batch.num_rows().saturating_mul(4).max(1)); + *next_rows = rows_for_bytes( + batch.num_rows(), + result_bytes.saturating_sub(fixed_bytes), + payload_bytes, + ) + .min(batch.num_rows().saturating_mul(4).max(1)); } } *offset = offset - .checked_add(batch.num_rows() as i64) + .checked_add(row_count) .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; if sender.send(Ok(result)).await.is_err() { return Ok(false); @@ -1186,6 +1235,163 @@ mod tests { ); } + #[cfg(target_pointer_width = "64")] + #[tokio::test] + async fn unrepresentable_arrow_row_counts_fail_before_encoding() { + use arrow::array::NullArray; + use arrow::datatypes::{DataType, Field, Schema}; + use arrow::record_batch::RecordBatchOptions; + use std::sync::atomic::Ordering; + + let rows = i64::MAX as usize + 1; + let empty = RecordBatch::try_new_with_options( + Arc::new(Schema::empty()), + vec![], + &RecordBatchOptions::new().with_row_count(Some(rows)), + ) + .unwrap(); + let nulls = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("nulls", DataType::Null, true)])), + vec![Arc::new(NullArray::new(rows))], + ) + .unwrap(); + for batch in [empty, nulls] { + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("row_count").unwrap(); + let (sender, mut receiver) = mpsc::channel(2); + let mut offset = 7; + let mut output_schema = None; + for direct in [false, true] { + let error = if direct { + send_arrow_batch( + &sender, + "session", + "row_count", + batch.clone(), + rows, + &mut offset, + &mut output_schema, + &guard, + ) + .await + .unwrap_err() + } else { + send_batch( + &sender, + "session", + "row_count", + batch.clone().into(), + batch.schema().as_ref(), + &mut offset, + &mut output_schema, + &guard, + ) + .await + .unwrap_err() + }; + assert_eq!(error.code(), tonic::Code::ResourceExhausted); + assert_eq!(error.message(), "result batch row count exceeds i64"); + assert_eq!(offset, 7); + assert!(output_schema.is_none()); + assert_eq!(guard.running.encoding_attempts.load(Ordering::SeqCst), 0); + assert!(receiver.try_recv().is_err()); + } + } + } + + #[tokio::test] + async fn overflowing_engine_batches_do_not_partially_emit() { + use arrow::array::LargeStringArray; + use arrow::datatypes::{DataType, Field, Schema}; + use std::sync::atomic::Ordering; + + let schema = Arc::new(Schema::new(vec![Field::new( + "text", + DataType::LargeUtf8, + false, + )])); + let small = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(LargeStringArray::from(vec![""]))], + ) + .unwrap(); + let large = "z".repeat(2 * 1024 * 1024); + let oversized = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(LargeStringArray::from(vec![large.as_str(); 2]))], + ) + .unwrap(); + for ipc in [false, true] { + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("row_offset").unwrap(); + let (sender, mut receiver) = mpsc::channel(4); + let mut offset = i64::MAX - 2; + let mut output_schema = None; + send_batch( + &sender, + "session", + "row_offset", + small.clone().into(), + schema.as_ref(), + &mut offset, + &mut output_schema, + &guard, + ) + .await + .unwrap(); + let before = guard.running.encoding_attempts.load(Ordering::SeqCst); + let batch = if ipc { + let data = encode_batch(&oversized).unwrap(); + assert!(data.len() > MAX_RESULT_BYTES); + ResultBatch::Ipc { data, row_count: 2 } + } else { + oversized.clone().into() + }; + let error = send_batch( + &sender, + "session", + "row_offset", + batch, + schema.as_ref(), + &mut offset, + &mut output_schema, + &guard, + ) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::ResourceExhausted); + assert_eq!(error.message(), "result row offset exceeds i64"); + assert_eq!(offset, i64::MAX - 1); + assert_eq!( + guard.running.encoding_attempts.load(Ordering::SeqCst), + before + ); + let previous = receiver.try_recv().unwrap().unwrap(); + let Some(proto::execute_plan_response::ResponseType::ArrowBatch(previous)) = + previous.response_type + else { + panic!("expected the preceding valid Arrow response"); + }; + assert_eq!(previous.start_offset, Some(i64::MAX - 2)); + assert_eq!(previous.row_count, 1); + assert!(receiver.try_recv().is_err()); + // A one-row batch still fits exactly at the signed boundary. + send_batch( + &sender, + "session", + "row_offset", + small.clone().into(), + schema.as_ref(), + &mut offset, + &mut output_schema, + &guard, + ) + .await + .unwrap(); + assert_eq!(offset, i64::MAX); + } + } + #[tokio::test] async fn successful_encoding_grows_a_conservative_row_budget() { use arrow::array::Int64Array; @@ -1234,6 +1440,108 @@ mod tests { ); } + #[tokio::test] + async fn fixed_schema_overhead_does_not_collapse_the_payload_budget() { + use arrow::array::{Int64Array, LargeStringArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use std::collections::HashMap; + use std::sync::atomic::Ordering; + + for (metadata_bytes, expected_rows, max_responses) in [ + ((1.49 * 1024.0 * 1024.0) as usize, 1500usize, 18usize), + ((2.6 * 1024.0 * 1024.0) as usize, 300usize, 14usize), + ] { + let schema = Arc::new(Schema::new_with_metadata( + vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::LargeUtf8, false), + ], + HashMap::from([("padding".into(), "x".repeat(metadata_bytes))]), + )); + let text = "z".repeat(8192); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int64Array::from_iter_values(0..expected_rows as i64)), + Arc::new(LargeStringArray::from_iter_values(std::iter::repeat_n( + text.as_str(), + expected_rows, + ))), + ], + ) + .unwrap(); + let budget = estimated_rows_per_batch(&batch); + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("fixed_overhead").unwrap(); + // Bound retained metadata even if the old one-row decay returns; + // the timeout then fails without buffering gigabytes of replies. + let (sender, mut receiver) = mpsc::channel(max_responses + 1); + let mut offset = 0; + let mut output_schema = None; + assert!(tokio::time::timeout( + std::time::Duration::from_secs(15), + send_arrow_batch( + &sender, + "session", + "fixed_overhead", + batch, + budget, + &mut offset, + &mut output_schema, + &guard + ) + ) + .await + .expect("fixed overhead exhausted the bounded response queue") + .unwrap()); + let (mut responses, mut rows) = (0usize, 0usize); + while let Ok(result) = receiver.try_recv() { + let result = result.unwrap(); + assert!(result.encoded_len() <= MAX_RESULT_BYTES); + let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + result.response_type + else { + panic!("expected Arrow response"); + }; + assert_eq!(batch.start_offset, Some(rows as i64)); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + assert_eq!(reader.schema().metadata()["padding"].len(), metadata_bytes); + let decoded = reader.next().unwrap().unwrap(); + assert_eq!(decoded.num_rows() as i64, batch.row_count); + assert!(reader.next().is_none()); + let ids = decoded + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let strings = decoded + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + for index in 0..decoded.num_rows() { + assert_eq!(ids.value(index), rows as i64); + assert_eq!(strings.value(index), text); + rows += 1; + } + responses += 1; + } + assert_eq!(rows, expected_rows); + assert_eq!(offset, expected_rows as i64); + assert!( + responses <= max_responses, + "{metadata_bytes} fixed bytes caused {responses} responses" + ); + // One encode measures the fresh empty schema. Only a small + // constant of exact-envelope retries is acceptable after that. + let attempts = guard.running.encoding_attempts.load(Ordering::SeqCst); + assert!( + attempts <= responses + 2, + "{attempts} encodes for {responses} responses" + ); + } + } + #[tokio::test] async fn skewed_output_checks_prefixes_and_recovers_light_suffixes() { use arrow::array::{Int64Array, LargeStringArray}; @@ -1346,6 +1654,11 @@ mod tests { responses < 100, "large-head output fragmented into {responses} responses" ); + } else { + assert!( + responses < 80, + "large-tail output fragmented into {responses} responses" + ); } } } diff --git a/crates/spark-connect-server/tests/review_regressions.rs b/crates/spark-connect-server/tests/review_regressions.rs index 992fc8a6..43f46c99 100644 --- a/crates/spark-connect-server/tests/review_regressions.rs +++ b/crates/spark-connect-server/tests/review_regressions.rs @@ -450,6 +450,70 @@ async fn view_slices_only_budget_the_current_values() { } } +#[cfg(target_pointer_width = "64")] +#[tokio::test] +async fn unrepresentable_engine_batches_preserve_preceding_results_and_fail_cleanly() { + use arrow::array::NullArray; + use arrow::record_batch::RecordBatchOptions; + + let rows = i64::MAX as usize + 1; + for null_column in [false, true] { + let oversized = if null_column { + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("nulls", DataType::Null, true)])), + vec![Arc::new(NullArray::new(rows))], + ) + .unwrap() + } else { + RecordBatch::try_new_with_options( + Arc::new(Schema::empty()), + vec![], + &RecordBatchOptions::new().with_row_count(Some(rows)), + ) + .unwrap() + }; + let state = State::new(vec![oversized.slice(0, 3), oversized], None); + let mut server = Server::start_with_engine(Arc::new(Fixture(state))).await; + let mut stream = server + .client + .execute_plan(execute( + &Uuid::new_v4().to_string(), + &Uuid::new_v4().to_string(), + )) + .await + .unwrap() + .into_inner(); + let (status, counts) = tokio::time::timeout(Duration::from_secs(15), async { + let mut counts = Vec::new(); + loop { + match stream.message().await { + Ok(Some(message)) => match message.response_type { + Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) => { + assert_eq!(batch.start_offset, Some(0)); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + assert_eq!(reader.next().unwrap().unwrap().num_rows(), 3); + assert!(reader.next().is_none()); + counts.push(batch.row_count); + } + Some(proto::execute_plan_response::ResponseType::ResultComplete(_)) => { + panic!("unrepresentable output must not complete successfully"); + } + _ => {} + }, + Err(status) => break (status, counts), + Ok(None) => panic!("expected row-count overflow status"), + } + } + }) + .await + .unwrap(); + assert_eq!(status.code(), Code::ResourceExhausted); + assert_eq!(status.message(), "result batch row count exceeds i64"); + assert_eq!(counts, vec![3]); + server.stop().await; + } +} + #[tokio::test] async fn noncanonical_ids_are_rejected_without_creating_or_releasing_state() { let state = State::new(vec![integers(1)], None); From dd6069bdbd36bd795cae68d2c0f6c3597c6b8afc Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 17:02:18 -0800 Subject: [PATCH 07/17] Preserve fitting Arrow budgets across padded buffers --- crates/spark-connect-server/src/service.rs | 120 +++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index beb0cf81..ad9e09c3 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -732,6 +732,13 @@ async fn send_arrow_batch( / (result_bytes as u128 * 10)) as usize) .clamp(1, batch.num_rows() - 1); drop(result); + if has_remainder { + if let Some((_, next_rows)) = pending.last_mut() { + // Teach this piece's own remainder about IPC padding or + // other fixed costs absent from the logical estimate. + *next_rows = (*next_rows).min(rows_per_batch); + } + } pending.push((batch.clone(), rows_per_batch)); continue; } @@ -749,6 +756,9 @@ async fn send_arrow_batch( result_bytes.saturating_sub(fixed_bytes), payload_bytes, ) + // A successful size is already known to fit. Fixed buffer + // padding must not make that row budget decay on each send. + .max(batch.num_rows()) .min(batch.num_rows().saturating_mul(4).max(1)); } } @@ -1542,6 +1552,116 @@ mod tests { } } + #[tokio::test] + async fn wide_buffer_padding_does_not_decay_or_repeat_failed_budgets() { + use arrow::array::{Array, ArrayRef, Int64Array, Int8Array}; + use arrow::datatypes::{DataType, Field, Schema}; + use std::sync::atomic::Ordering; + + // Int64 values cross a 64-byte IPC padding boundary at 8 -> 9 rows; + // Int8 values occupy the same padded buffer for up to 64 rows. Nullable + // columns add a separately padded validity buffer absent from an empty + // schema measurement. Share inputs to keep this regression inexpensive. + for (columns, expected_rows, narrow, nullable, max_responses) in [ + (17_293, 64usize, false, false, 18usize), + (17_293, 200usize, true, false, 10usize), + (12_000, 64usize, false, true, 18usize), + ] { + let values: ArrayRef = if narrow { + Arc::new(Int8Array::from_iter_values( + (0..expected_rows).map(|row| (row % 127) as i8), + )) + } else if nullable { + Arc::new(Int64Array::from_iter( + (0..expected_rows).map(|row| (row % 3 != 0).then_some(row as i64)), + )) + } else { + Arc::new(Int64Array::from_iter_values(0..expected_rows as i64)) + }; + let fields = (0..columns) + .map(|column| { + Field::new( + format!("c{column:05}"), + if narrow { + DataType::Int8 + } else { + DataType::Int64 + }, + nullable, + ) + }) + .collect::>(); + let batch = + RecordBatch::try_new(Arc::new(Schema::new(fields)), vec![values; columns]).unwrap(); + let budget = estimated_rows_per_batch(&batch); + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("wide_padding").unwrap(); + let (sender, mut receiver) = mpsc::channel(max_responses + 1); + let mut offset = 0; + let mut output_schema = None; + assert!(tokio::time::timeout( + std::time::Duration::from_secs(30), + send_arrow_batch( + &sender, + "session", + "wide_padding", + batch, + budget, + &mut offset, + &mut output_schema, + &guard + ) + ) + .await + .expect("wide padding exhausted the bounded response queue") + .unwrap()); + let (mut responses, mut rows) = (0usize, 0usize); + while let Ok(result) = receiver.try_recv() { + let result = result.unwrap(); + assert!(result.encoded_len() <= MAX_RESULT_BYTES); + let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + result.response_type + else { + panic!("expected Arrow response"); + }; + assert_eq!(batch.start_offset, Some(rows as i64)); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + let decoded = reader.next().unwrap().unwrap(); + assert_eq!(decoded.num_columns(), columns); + assert_eq!(decoded.num_rows() as i64, batch.row_count); + assert!(reader.next().is_none()); + for column in decoded.columns() { + if narrow { + let values = column.as_any().downcast_ref::().unwrap(); + for index in 0..decoded.num_rows() { + assert_eq!(values.value(index), ((rows + index) % 127) as i8); + } + } else { + let values = column.as_any().downcast_ref::().unwrap(); + for index in 0..decoded.num_rows() { + let row = rows + index; + assert_eq!(values.is_null(index), nullable && row % 3 == 0); + if !values.is_null(index) { + assert_eq!(values.value(index), row as i64); + } + } + } + } + rows += decoded.num_rows(); + responses += 1; + } + assert_eq!(rows, expected_rows); + assert_eq!(offset, expected_rows as i64); + assert!(responses <= max_responses, + "{columns} columns (narrow={narrow}, nullable={nullable}) caused {responses} responses"); + // One measurement plus a small number of exact retries, including + // a nullable slice that crosses the 8 -> 9 row padding boundary. + let attempts = guard.running.encoding_attempts.load(Ordering::SeqCst); + assert!(attempts <= responses + 3, + "{columns} columns (narrow={narrow}, nullable={nullable}): {attempts} encodes for {responses} responses"); + } + } + #[tokio::test] async fn skewed_output_checks_prefixes_and_recovers_light_suffixes() { use arrow::array::{Int64Array, LargeStringArray}; From 8ba53a2d08300cb22e0a6addde037c61057c6d06 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 17:06:30 -0800 Subject: [PATCH 08/17] Avoid redundant logical sizing before Arrow preflight --- crates/spark-connect-server/src/engine/mod.rs | 4 ++-- crates/spark-connect-server/src/service.rs | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/spark-connect-server/src/engine/mod.rs b/crates/spark-connect-server/src/engine/mod.rs index 64f6a8a1..e2285e67 100644 --- a/crates/spark-connect-server/src/engine/mod.rs +++ b/crates/spark-connect-server/src/engine/mod.rs @@ -97,8 +97,8 @@ pub enum PreparedOperation { #[tonic::async_trait] pub trait PreparedQuery: Send { /// The declared Spark schema. This synchronous accessor must not panic. - /// Batches must have the same field names and - /// Spark-visible types; equivalent Arrow representations are accepted. + /// Batches must have the same field names and Spark-visible types; + /// equivalent Arrow representations are accepted. /// Batch nullability and metadata may differ from this declaration, but /// emitted IPC batches in one result must share physical types, field names /// and nullability. Metadata may differ between batches. Encoding materializes diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index ad9e09c3..2e411b09 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -490,6 +490,7 @@ fn estimated_array_bytes(array: &dyn arrow::array::Array) -> usize { bytes.saturating_add(array.nulls().map_or(0, |_| array.len().div_ceil(8))) } +#[cfg(test)] fn estimated_rows_per_batch(batch: &RecordBatch) -> usize { estimated_rows_for_payload(batch, MAX_RESULT_BYTES / 2) } @@ -549,7 +550,7 @@ async fn send_batch( let (batch, rows_per_batch) = match batch { ResultBatch::Arrow(batch) => { checked_arrow_row_count(batch.num_rows(), *offset)?; - let rows = estimated_rows_per_batch(&batch); + let rows = batch.num_rows().max(1); (batch, rows) } ResultBatch::Ipc { data, row_count } => { From 4a1219951693ec407c0efa750a3d85bd66629939 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 17:17:39 -0800 Subject: [PATCH 09/17] Align Arrow encoding regressions with production sizing --- crates/spark-connect-server/src/service.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index 2e411b09..0efb4134 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -549,7 +549,6 @@ async fn send_batch( ) -> Result { let (batch, rows_per_batch) = match batch { ResultBatch::Arrow(batch) => { - checked_arrow_row_count(batch.num_rows(), *offset)?; let rows = batch.num_rows().max(1); (batch, rows) } @@ -600,8 +599,8 @@ async fn send_batch( "engine IPC batch count differs from its declaration", )); } - // Decoded columns can share the full IPC allocation. Use its - // serialized size instead of counting that allocation per column. + // Seed oversized IPC splitting from its serialized size; the + // payload-aware preflight refines this before encoding. let rows = rows_for_bytes(batch.num_rows(), ipc_bytes, MAX_RESULT_BYTES / 2); Ok((batch, rows)) }) @@ -1481,7 +1480,7 @@ mod tests { ], ) .unwrap(); - let budget = estimated_rows_per_batch(&batch); + let budget = batch.num_rows().max(1); let session = Arc::new(Session::new(Arc::new(RegistrySession))); let (guard, _) = session.operation("fixed_overhead").unwrap(); // Bound retained metadata even if the old one-row decay returns; @@ -1594,7 +1593,7 @@ mod tests { .collect::>(); let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), vec![values; columns]).unwrap(); - let budget = estimated_rows_per_batch(&batch); + let budget = batch.num_rows().max(1); let session = Arc::new(Session::new(Arc::new(RegistrySession))); let (guard, _) = session.operation("wide_padding").unwrap(); let (sender, mut receiver) = mpsc::channel(max_responses + 1); @@ -1691,7 +1690,7 @@ mod tests { ], ) .unwrap(); - let budget = estimated_rows_per_batch(&batch); + let budget = batch.num_rows().max(1); let session = Arc::new(Session::new(Arc::new(RegistrySession))); let (guard, _) = session.operation("skewed").unwrap(); // Even an accidentally permanent one-row budget must not block From f1898dcb418353b2c233c02d8acc19e3127b1339 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 20:25:58 -0800 Subject: [PATCH 10/17] Share recursive Arrow schema field transformations --- crates/spark-connect-server/src/schema.rs | 39 +++++++------ crates/spark-connect-server/src/service.rs | 66 ++++++---------------- 2 files changed, 40 insertions(+), 65 deletions(-) diff --git a/crates/spark-connect-server/src/schema.rs b/crates/spark-connect-server/src/schema.rs index a6a37a49..76ab1b15 100644 --- a/crates/spark-connect-server/src/schema.rs +++ b/crates/spark-connect-server/src/schema.rs @@ -19,7 +19,7 @@ use std::sync::Arc; -use arrow::datatypes::{DataType, Field, FieldRef, Schema, TimeUnit}; +use arrow::datatypes::{DataType, Field, FieldRef, Fields, Schema, TimeUnit}; use prost::Message; use spark_connect_proto as proto; use tonic::Status; @@ -41,24 +41,31 @@ pub(crate) fn validate_schema_size(schema: &proto::DataType) -> Result<(), Statu Ok(()) } -/// Compare Spark-visible fields and types, allowing equivalent Arrow physical -/// representations and differences in nullability or field/schema metadata. -pub(crate) fn comparable_schema(schema: &Schema) -> Result { - fn field(input: &FieldRef) -> FieldRef { - Arc::new(Field::new(input.name(), data_type(input.data_type()), true)) - } - - fn data_type(input: &DataType) -> DataType { - match input { - DataType::List(child) => DataType::List(field(child)), - DataType::LargeList(child) => DataType::LargeList(field(child)), - DataType::Struct(fields) => DataType::Struct(fields.iter().map(field).collect()), - DataType::Map(entries, sorted) => DataType::Map(field(entries), *sorted), +/// Transform fields after recursively transforming their supported child fields. +pub(crate) fn map_fields( + fields: &Fields, + transform: &impl Fn(&FieldRef, DataType) -> FieldRef, +) -> Fields { + fn field(input: &FieldRef, transform: &impl Fn(&FieldRef, DataType) -> FieldRef) -> FieldRef { + let data_type = match input.data_type() { + DataType::List(child) => DataType::List(field(child, transform)), + DataType::LargeList(child) => DataType::LargeList(field(child, transform)), + DataType::Struct(children) => DataType::Struct(map_fields(children, transform)), + DataType::Map(entries, sorted) => DataType::Map(field(entries, transform), *sorted), other => other.clone(), - } + }; + transform(input, data_type) } - let fields: Vec<_> = schema.fields().iter().map(field).collect(); + fields.iter().map(|input| field(input, transform)).collect() +} + +/// Compare Spark-visible fields and types, allowing equivalent Arrow physical +/// representations and differences in nullability or field/schema metadata. +pub(crate) fn comparable_schema(schema: &Schema) -> Result { + let fields = map_fields(schema.fields(), &|field, data_type| { + Arc::new(Field::new(field.name(), data_type, true)) + }); spark_schema(&Schema::new(fields)) } diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index 0efb4134..11baf810 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -312,42 +312,26 @@ fn complete(id: &str, session: &Session, operation: &str) -> proto::ExecutePlanR } fn encode_batch(batch: &RecordBatch) -> Result, Status> { - use arrow::datatypes::{DataType, FieldRef, Schema}; + use arrow::datatypes::DataType; use arrow::record_batch::RecordBatchOptions; - fn materialized_field(field: &FieldRef) -> FieldRef { - let data_type = materialized_type(field.data_type()); - if data_type == *field.data_type() { - field.clone() - } else { - Arc::new(field.as_ref().clone().with_data_type(data_type)) - } - } - - fn materialized_type(data_type: &DataType) -> DataType { - match data_type { - DataType::Utf8View => DataType::LargeUtf8, - DataType::BinaryView => DataType::LargeBinary, - DataType::List(field) => DataType::List(materialized_field(field)), - DataType::LargeList(field) => DataType::LargeList(materialized_field(field)), - DataType::Struct(fields) => { - DataType::Struct(fields.iter().map(materialized_field).collect()) - } - DataType::Map(field, sorted) => DataType::Map(materialized_field(field), *sorted), - other => other.clone(), - } - } - // IPC writes all backing buffers of view arrays even after a row slice. Materialize // their bytes, including nested children, so splitting measures only retained rows. // Use 64-bit offsets so materializing a batch cannot overflow 32-bit byte offsets // before the result-size check has a chance to split it. let source_schema = batch.schema(); - let fields: Vec<_> = source_schema - .fields() - .iter() - .map(materialized_field) - .collect(); + let fields = schema::map_fields(source_schema.fields(), &|field, data_type| { + let data_type = match data_type { + DataType::Utf8View => DataType::LargeUtf8, + DataType::BinaryView => DataType::LargeBinary, + other => other, + }; + if data_type == *field.data_type() { + field.clone() + } else { + Arc::new(field.as_ref().clone().with_data_type(data_type)) + } + }); let changed = fields .iter() .zip(source_schema.fields()) @@ -399,27 +383,11 @@ fn encode_batch(batch: &RecordBatch) -> Result, Status> { // PySpark assembles one table from all IPC batches. Its physical types and // nullability must agree across batches, while field/schema metadata may differ. fn check_output_schema(schema: &Schema, expected: &mut Option) -> Result<(), Status> { - use arrow::datatypes::{DataType, Field, FieldRef}; - - fn field(input: &FieldRef) -> FieldRef { - Arc::new(Field::new( - input.name(), - data_type(input.data_type()), - input.is_nullable(), - )) - } - - fn data_type(input: &DataType) -> DataType { - match input { - DataType::List(child) => DataType::List(field(child)), - DataType::LargeList(child) => DataType::LargeList(field(child)), - DataType::Struct(fields) => DataType::Struct(fields.iter().map(field).collect()), - DataType::Map(entries, sorted) => DataType::Map(field(entries), *sorted), - other => other.clone(), - } - } + use arrow::datatypes::Field; - let actual = Schema::new(schema.fields().iter().map(field).collect::>()); + let actual = Schema::new(schema::map_fields(schema.fields(), &|field, data_type| { + Arc::new(Field::new(field.name(), data_type, field.is_nullable())) + })); match expected { Some(expected) if *expected != actual => Err(Status::internal( "Arrow output schema changed within one result", From 94ab6108d3074bcdfae66359f56988d2ab43dd7a Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 20:25:59 -0800 Subject: [PATCH 11/17] Generate optional server bindings from the protocol crate --- .github/workflows/server-frontend.yml | 5 + Cargo.lock | 3 - Cargo.toml | 1 - crates/spark-connect-proto/Cargo.toml | 3 + crates/spark-connect-proto/build.rs | 4 +- crates/spark-connect-proto/src/lib.rs | 2 + crates/spark-connect-server/Cargo.toml | 9 +- crates/spark-connect-server/README.md | 3 +- crates/spark-connect-server/build.rs | 186 ------------------------- crates/spark-connect-server/src/lib.rs | 9 +- 10 files changed, 18 insertions(+), 207 deletions(-) delete mode 100644 crates/spark-connect-server/build.rs diff --git a/.github/workflows/server-frontend.yml b/.github/workflows/server-frontend.yml index e088800f..6b77267f 100644 --- a/.github/workflows/server-frontend.yml +++ b/.github/workflows/server-frontend.yml @@ -86,3 +86,8 @@ jobs: - name: Test frontend # Protocol and injected-engine tests use local fixtures without an execution backend. run: cargo test --locked -p spark-connect-server + + - name: Check protocol feature combinations + run: | + cargo check --locked -p apache-spark-connect-proto --no-default-features + cargo check --locked -p apache-spark-connect-proto --no-default-features --features server diff --git a/Cargo.lock b/Cargo.lock index 7e32c1e5..41a9b1b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4057,12 +4057,9 @@ dependencies = [ "futures", "h2", "prost", - "prost-types", "tokio", "tokio-stream", "tonic", - "tonic-build", - "tonic-prost", "uuid", ] diff --git a/Cargo.toml b/Cargo.toml index 9ba59607..189b8bab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,7 +90,6 @@ should_implement_trait = "allow" tonic = { version = "0.14", default-features = false, features = ["channel", "codegen"] } tonic-prost = "0.14" tonic-prost-build = "0.14" -tonic-build = "0.14" h2 = "0.4" prost = "0.14" prost-types = "0.14" diff --git a/crates/spark-connect-proto/Cargo.toml b/crates/spark-connect-proto/Cargo.toml index efc4c833..330fdfa0 100644 --- a/crates/spark-connect-proto/Cargo.toml +++ b/crates/spark-connect-proto/Cargo.toml @@ -15,6 +15,9 @@ workspace = true [lib] name = "spark_connect_proto" +[features] +server = [] + [dependencies] tonic.workspace = true tonic-prost.workspace = true diff --git a/crates/spark-connect-proto/build.rs b/crates/spark-connect-proto/build.rs index 048ae15f..dd85bafe 100644 --- a/crates/spark-connect-proto/build.rs +++ b/crates/spark-connect-proto/build.rs @@ -35,8 +35,10 @@ fn main() -> Result<(), Box> { let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").ok_or("OUT_DIR is not set")?); let full_set = out_dir.join("spark_connect_descriptor_set_full.bin"); tonic_prost_build::configure() - .build_server(false) + .build_server(cfg!(feature = "server")) .build_client(true) + // async_trait repeats Future's must_use on generated server methods. + .trait_attribute("SparkConnectService", "#[allow(clippy::double_must_use)]") .bytes(".") .file_descriptor_set_path(&full_set) .compile_protos(&protos, &[proto_root])?; diff --git a/crates/spark-connect-proto/src/lib.rs b/crates/spark-connect-proto/src/lib.rs index 35689ba4..58eb65a9 100644 --- a/crates/spark-connect-proto/src/lib.rs +++ b/crates/spark-connect-proto/src/lib.rs @@ -3,6 +3,8 @@ //! All messages live in the `spark.connect` protobuf package and are re-exported //! here at the crate root. The generated gRPC client is //! [`spark_connect_service_client::SparkConnectServiceClient`]. +//! Enable the optional `server` feature for `spark_connect_service_server` +//! bindings. Embeddings configure tonic's transport server features separately. //! //! Baseline protocol version: see `proto/PROTO_VERSION.txt` / `proto/SPARK_SHA.txt`. diff --git a/crates/spark-connect-server/Cargo.toml b/crates/spark-connect-server/Cargo.toml index 5fe56bcc..96adcd4a 100644 --- a/crates/spark-connect-server/Cargo.toml +++ b/crates/spark-connect-server/Cargo.toml @@ -28,21 +28,14 @@ description = "Experimental Spark Connect server frontend with an injectable exe workspace = true [dependencies] -spark-connect-proto = { package = "apache-spark-connect-proto", path = "../spark-connect-proto" } +spark-connect-proto = { package = "apache-spark-connect-proto", path = "../spark-connect-proto", features = ["server"] } arrow.workspace = true futures.workspace = true prost.workspace = true tokio = { workspace = true, features = ["net"] } tokio-stream.workspace = true tonic = { workspace = true, features = ["server", "router"] } -tonic-prost.workspace = true uuid.workspace = true -[build-dependencies] -tonic-build.workspace = true -spark-connect-proto = { package = "apache-spark-connect-proto", path = "../spark-connect-proto" } -prost.workspace = true -prost-types.workspace = true - [dev-dependencies] h2.workspace = true diff --git a/crates/spark-connect-server/README.md b/crates/spark-connect-server/README.md index 564b6531..620a7df6 100644 --- a/crates/spark-connect-server/README.md +++ b/crates/spark-connect-server/README.md @@ -20,7 +20,8 @@ # Experimental Spark Connect server frontend This unpublished library serves a subset of the Spark Connect protocol using the -workspace's `apache-spark-connect-proto` messages. The caller supplies an +workspace's `apache-spark-connect-proto` messages and its optional `server` +bindings. The caller supplies an `Engine` implementation and a bound Tokio TCP listener to `serve_with_engine`. No execution engine or standalone server executable is included. diff --git a/crates/spark-connect-server/build.rs b/crates/spark-connect-server/build.rs deleted file mode 100644 index 5041d662..00000000 --- a/crates/spark-connect-server/build.rs +++ /dev/null @@ -1,186 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use prost::Message; -use tonic_build::manual::{Builder, Method, Service}; - -fn main() { - // The official protocol crate already generates messages and client bindings. - // Generate only the standard service routes, using those exact message types. - // Check this table against the generated protocol descriptor before emitting - // routes: streaming signatures are part of wire compatibility. - let methods = [ - ( - "execute_plan", - "ExecutePlan", - "ExecutePlanRequest", - "ExecutePlanResponse", - false, - true, - ), - ( - "analyze_plan", - "AnalyzePlan", - "AnalyzePlanRequest", - "AnalyzePlanResponse", - false, - false, - ), - ( - "config", - "Config", - "ConfigRequest", - "ConfigResponse", - false, - false, - ), - ( - "add_artifacts", - "AddArtifacts", - "AddArtifactsRequest", - "AddArtifactsResponse", - true, - false, - ), - ( - "artifact_status", - "ArtifactStatus", - "ArtifactStatusesRequest", - "ArtifactStatusesResponse", - false, - false, - ), - ( - "interrupt", - "Interrupt", - "InterruptRequest", - "InterruptResponse", - false, - false, - ), - ( - "reattach_execute", - "ReattachExecute", - "ReattachExecuteRequest", - "ExecutePlanResponse", - false, - true, - ), - ( - "release_execute", - "ReleaseExecute", - "ReleaseExecuteRequest", - "ReleaseExecuteResponse", - false, - false, - ), - ( - "release_session", - "ReleaseSession", - "ReleaseSessionRequest", - "ReleaseSessionResponse", - false, - false, - ), - ( - "fetch_error_details", - "FetchErrorDetails", - "FetchErrorDetailsRequest", - "FetchErrorDetailsResponse", - false, - false, - ), - ( - "clone_session", - "CloneSession", - "CloneSessionRequest", - "CloneSessionResponse", - false, - false, - ), - ( - "get_status", - "GetStatus", - "GetStatusRequest", - "GetStatusResponse", - false, - false, - ), - ]; - let descriptors = - prost_types::FileDescriptorSet::decode(spark_connect_proto::FILE_DESCRIPTOR_SET) - .expect("valid Spark Connect descriptor set"); - let descriptor = descriptors - .file - .iter() - .filter(|file| file.package.as_deref() == Some("spark.connect")) - .flat_map(|file| &file.service) - .find(|service| service.name.as_deref() == Some("SparkConnectService")) - .expect("SparkConnectService descriptor"); - assert_eq!(methods.len(), descriptor.method.len(), "RPC table drift"); - let mut routes = std::collections::HashSet::new(); - for &(_, route, input, output, client_stream, server_stream) in &methods { - assert!(routes.insert(route), "duplicate RPC route: {route}"); - let method = descriptor - .method - .iter() - .find(|method| method.name.as_deref() == Some(route)) - .unwrap_or_else(|| panic!("missing protocol RPC: {route}")); - assert_eq!( - method.input_type.as_deref(), - Some(format!(".spark.connect.{input}").as_str()), - "{route} request type drift" - ); - assert_eq!( - method.output_type.as_deref(), - Some(format!(".spark.connect.{output}").as_str()), - "{route} response type drift" - ); - assert_eq!( - method.client_streaming.unwrap_or(false), - client_stream, - "{route} client streaming drift" - ); - assert_eq!( - method.server_streaming.unwrap_or(false), - server_stream, - "{route} server streaming drift" - ); - } - let mut service = Service::builder() - .name("SparkConnectService") - .package("spark.connect"); - for (name, route, input, output, client_stream, server_stream) in methods { - let mut method = Method::builder() - .name(name) - .route_name(route) - .input_type(format!("spark_connect_proto::{input}")) - .output_type(format!("spark_connect_proto::{output}")) - .codec_path("tonic_prost::ProstCodec"); - if client_stream { - method = method.client_streaming(); - } - if server_stream { - method = method.server_streaming(); - } - service = service.method(method.build()); - } - Builder::new() - .build_client(false) - .compile(&[service.build()]); - println!("cargo:rerun-if-changed=build.rs"); -} diff --git a/crates/spark-connect-server/src/lib.rs b/crates/spark-connect-server/src/lib.rs index dece96a2..64db976b 100644 --- a/crates/spark-connect-server/src/lib.rs +++ b/crates/spark-connect-server/src/lib.rs @@ -20,14 +20,9 @@ mod schema; mod service; /// Generated server bindings for custom embedding. This experimental API follows -/// the RPC table in `build.rs` and may change with the protocol. -// tonic's generated async-trait methods duplicate Future's must_use annotation. -#[allow(clippy::double_must_use)] +/// the protocol schema and may change with the protocol. pub mod grpc { - include!(concat!( - env!("OUT_DIR"), - "/spark.connect.SparkConnectService.rs" - )); + pub use spark_connect_proto::spark_connect_service_server; } struct CancelOnDrop(std::sync::Arc); From 2edcde3ff3973e3d443346fb419fa73092706d22 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 20:26:00 -0800 Subject: [PATCH 12/17] Consolidate frontend fixtures and overlapping lifecycle tests --- .../tests/engine_injection.rs | 246 +++++++----- crates/spark-connect-server/tests/protocol.rs | 375 +----------------- .../tests/schema_consistency.rs | 91 +---- .../spark-connect-server/tests/support/mod.rs | 175 +++----- .../tests/support/requests.rs | 111 ++++++ 5 files changed, 333 insertions(+), 665 deletions(-) create mode 100644 crates/spark-connect-server/tests/support/requests.rs diff --git a/crates/spark-connect-server/tests/engine_injection.rs b/crates/spark-connect-server/tests/engine_injection.rs index 739a8124..a5797ed9 100644 --- a/crates/spark-connect-server/tests/engine_injection.rs +++ b/crates/spark-connect-server/tests/engine_injection.rs @@ -17,6 +17,8 @@ //! Real gRPC tests of the injected backend boundary. The fake is independent of Spark execution. +#[path = "support/requests.rs"] +mod requests; #[path = "support/server.rs"] mod server; @@ -37,6 +39,9 @@ use futures::{Stream, StreamExt}; use prost::Message; use proto::execute_plan_response::ResponseType; use proto::spark_connect_service_client::SparkConnectServiceClient; +use requests::{ + assert_cancelled, config_request, relation, release, release_session, request, user, +}; use server::Server; use spark_connect_proto as proto; use spark_connect_server::engine::{ @@ -514,36 +519,6 @@ async fn start_server(spy: Arc) -> Server { Server::start_with_engine(Arc::new(FakeEngine(spy))).await } -fn user(id: &str) -> Option { - Some(proto::UserContext { - user_id: id.into(), - ..Default::default() - }) -} - -fn relation(end: i64) -> proto::Relation { - proto::Relation { - rel_type: Some(proto::relation::RelType::Range(proto::Range { - end, - step: 1, - ..Default::default() - })), - ..Default::default() - } -} - -fn request(session: &str, end: i64) -> proto::ExecutePlanRequest { - proto::ExecutePlanRequest { - session_id: session.into(), - user_context: user("alice"), - operation_id: Some(Uuid::new_v4().to_string()), - plan: Some(proto::Plan { - op_type: Some(proto::plan::OpType::Root(relation(end))), - }), - ..Default::default() - } -} - fn command_request( session: &str, command_type: proto::command::CommandType, @@ -557,20 +532,6 @@ fn command_request( request } -fn config_request( - session: &str, - input: proto::config_request::operation::OpType, -) -> proto::ConfigRequest { - proto::ConfigRequest { - session_id: session.into(), - user_context: user("alice"), - operation: Some(proto::config_request::Operation { - op_type: Some(input), - }), - ..Default::default() - } -} - fn get_config(session: &str) -> proto::ConfigRequest { config_request( session, @@ -580,26 +541,6 @@ fn get_config(session: &str) -> proto::ConfigRequest { ) } -fn release(session: &str, operation: &str) -> proto::ReleaseExecuteRequest { - proto::ReleaseExecuteRequest { - session_id: session.into(), - user_context: user("alice"), - operation_id: operation.into(), - release: Some(proto::release_execute_request::Release::ReleaseAll( - proto::release_execute_request::ReleaseAll {}, - )), - ..Default::default() - } -} - -fn release_session(session: &str) -> proto::ReleaseSessionRequest { - proto::ReleaseSessionRequest { - session_id: session.into(), - user_context: user("alice"), - ..Default::default() - } -} - async fn wait_for_count(counter: &AtomicUsize, expected: usize) { tokio::time::timeout(Duration::from_secs(5), async { while counter.load(Ordering::SeqCst) < expected { @@ -639,26 +580,6 @@ fn values(responses: &[proto::ExecutePlanResponse]) -> Vec { values } -async fn assert_cancelled(stream: &mut tonic::Streaming) { - tokio::time::timeout(Duration::from_secs(5), async { - loop { - match stream.message().await { - Err(status) => { - assert_eq!(status.code(), Code::Cancelled); - break; - } - Ok(Some(response)) => assert!(!matches!( - response.response_type, - Some(ResponseType::ResultComplete(_)) - )), - Ok(None) => panic!("cancelled fake execution must report cancellation"), - } - } - }) - .await - .expect("cancelled fake execution must terminate"); -} - async fn start_live( server: &mut Server, request: proto::ExecutePlanRequest, @@ -862,23 +783,89 @@ async fn invalid_requests_do_not_open_backends_and_factory_failures_can_retry() assert_eq!( server .client - .execute_plan(request("invalid", 99)) + .execute_plan(request("not-a-uuid", 0)) .await .unwrap_err() .code(), Code::InvalidArgument ); - let mut invalid = request(&session, 99); - invalid.operation_id = Some("invalid".into()); + let mut invalid_operation = request(&session, 0); + invalid_operation.operation_id = Some("not-a-uuid".into()); assert_eq!( server .client - .execute_plan(invalid) + .execute_plan(invalid_operation) .await .unwrap_err() .code(), Code::InvalidArgument ); + for plan in [None, Some(proto::Plan { op_type: None })] { + let mut request = request(&session, 0); + request.plan = plan; + assert_eq!( + server + .client + .execute_plan(request) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + } + assert_eq!( + server + .client + .analyze_plan(proto::AnalyzePlanRequest { + session_id: session.clone(), + ..Default::default() + }) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + for operation in [ + None, + Some(proto::config_request::Operation { op_type: None }), + ] { + assert_eq!( + server + .client + .config(proto::ConfigRequest { + session_id: session.clone(), + operation, + ..Default::default() + }) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + } + // A stale observed ID cannot match a session that does not exist. Repeated + // requests must not consume backend session capacity while being rejected. + for _ in 0..3 { + let mut stale = request(&session, 0); + stale.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); + assert_eq!( + server.client.execute_plan(stale).await.unwrap_err().code(), + Code::FailedPrecondition + ); + } + let mut stale = config_request( + &session, + proto::config_request::operation::OpType::GetOption(proto::config_request::GetOption { + keys: vec!["missing".into()], + }), + ); + stale.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); + assert_eq!( + server.client.config(stale).await.unwrap_err().code(), + Code::FailedPrecondition + ); + // Identifier and envelope validation must precede backend session allocation. + assert_eq!(spy.factory_calls.lock().unwrap().len(), 0); assert_eq!( server .client @@ -888,7 +875,35 @@ async fn invalid_requests_do_not_open_backends_and_factory_failures_can_retry() .code(), Code::NotFound ); - assert!(spy.factory_calls.lock().unwrap().is_empty()); + assert_eq!( + server + .client + .release_execute(release("not-a-uuid", &session)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_execute(release(&session, "not-a-uuid")) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + let unknown = Uuid::new_v4().to_string(); + assert_eq!( + server + .client + .release_execute(release(&unknown, &session)) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + assert_eq!(spy.factory_calls.lock().unwrap().len(), 0); spy.fail_next.store(true, Ordering::SeqCst); assert_eq!( server @@ -924,6 +939,15 @@ async fn native_streams_drop_on_release_disconnect_session_release_and_shutdown( let live = request(&session, LIVE); let operation = live.operation_id.clone().unwrap(); let mut stream = start_live(&mut server, live.clone()).await; + assert_eq!( + server + .client + .execute_plan(live.clone()) + .await + .unwrap_err() + .code(), + Code::AlreadyExists + ); let mut duplicate = command_request( &session, proto::command::CommandType::SqlCommand(proto::SqlCommand { @@ -943,19 +967,51 @@ async fn native_streams_drop_on_release_disconnect_session_release_and_shutdown( ); assert_eq!(spy.commands.load(Ordering::SeqCst), 0); assert_eq!(spy.prepares.load(Ordering::SeqCst), 1); - server - .client - .release_execute(release(&session, &operation)) - .await - .unwrap(); + let mut release_request = release(&session, &operation); + release_request.client_observed_server_side_session_id = Some( + server + .client + .config(get_config(&session)) + .await + .unwrap() + .into_inner() + .server_side_session_id, + ); + let released = tokio::time::timeout( + Duration::from_secs(3), + server.client.release_execute(release_request.clone()), + ) + .await + .expect("ReleaseAll must finish without consuming the result") + .unwrap() + .into_inner(); + assert_eq!(released.operation_id.as_deref(), Some(operation.as_str())); assert_eq!( spy.streams_dropped.load(Ordering::SeqCst), 1, "release must wait for backend stream teardown" ); + let absent = server + .client + .release_execute(release_request) + .await + .unwrap() + .into_inner(); + assert_eq!( + absent.operation_id, None, + "teardown must remove the operation" + ); assert_cancelled(&mut stream).await; - let abandoned = start_live(&mut server, live.clone()).await; + let mut abandoned = server + .client + .execute_plan(live.clone()) + .await + .unwrap() + .into_inner(); + assert!(abandoned.message().await.unwrap().unwrap().schema.is_some()); + // Keep the result batch unread while proving the backend stream exists. + wait_for_count(&spy.executions, 2).await; drop(abandoned); wait_for_count(&spy.streams_dropped, 2).await; let mut restarted = tokio::time::timeout(Duration::from_secs(5), async { diff --git a/crates/spark-connect-server/tests/protocol.rs b/crates/spark-connect-server/tests/protocol.rs index 1377bcc2..38bad66f 100644 --- a/crates/spark-connect-server/tests/protocol.rs +++ b/crates/spark-connect-server/tests/protocol.rs @@ -17,25 +17,25 @@ //! Verify observable Spark Connect wire behavior independently of DataFrame conveniences. +#[path = "support/requests.rs"] +mod requests; #[path = "support/server.rs"] mod server; mod support; use std::collections::HashSet; use std::io::Cursor; -use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -use std::time::Duration; use arrow::array::Int64Array; use arrow::ipc::reader::StreamReader; use prost::Message; use proto::execute_plan_response::ResponseType; use spark_connect_proto as proto; -use spark_connect_server::engine::{Engine, EngineSession, SessionContext}; +use requests::request as range; use server::Server; -use support::{FixtureEngine, LIVE_FIXTURE}; +use support::FixtureEngine; use tonic::Code; use uuid::Uuid; @@ -43,87 +43,6 @@ async fn start_server() -> Server { Server::start_with_engine(Arc::new(FixtureEngine::default())).await } -#[derive(Default)] -struct CountingEngine { - sessions_created: AtomicUsize, -} - -impl Engine for CountingEngine { - fn create_session( - &self, - context: SessionContext, - ) -> Result, tonic::Status> { - self.sessions_created.fetch_add(1, Ordering::SeqCst); - FixtureEngine::default().create_session(context) - } -} - -fn range(session: &str, end: i64) -> proto::ExecutePlanRequest { - proto::ExecutePlanRequest { - session_id: session.into(), - user_context: Some(proto::UserContext::default()), - operation_id: Some(Uuid::new_v4().to_string()), - plan: Some(proto::Plan { - op_type: Some(proto::plan::OpType::Root(proto::Relation { - rel_type: Some(proto::relation::RelType::Range(proto::Range { - start: Some(0), - end, - step: 1, - num_partitions: None, - })), - ..Default::default() - })), - }), - ..Default::default() - } -} - -fn release(session: &str, operation: &str) -> proto::ReleaseExecuteRequest { - proto::ReleaseExecuteRequest { - session_id: session.into(), - user_context: Some(proto::UserContext::default()), - operation_id: operation.into(), - release: Some(proto::release_execute_request::Release::ReleaseAll( - proto::release_execute_request::ReleaseAll {}, - )), - ..Default::default() - } -} - -fn config_request( - session: &str, - operation: proto::config_request::operation::OpType, -) -> proto::ConfigRequest { - proto::ConfigRequest { - session_id: session.into(), - user_context: Some(proto::UserContext::default()), - operation: Some(proto::config_request::Operation { - op_type: Some(operation), - }), - ..Default::default() - } -} - -async fn assert_cancelled(stream: &mut tonic::Streaming) { - tokio::time::timeout(Duration::from_secs(5), async { - loop { - match stream.message().await { - Ok(Some(response)) => assert!(!matches!( - response.response_type, - Some(ResponseType::ResultComplete(_)) - )), - Ok(None) => panic!("cancelled execution must report cancellation"), - Err(status) => { - assert_eq!(status.code(), Code::Cancelled); - break; - } - } - } - }) - .await - .expect("cancelled stream must terminate"); -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn arrow_stream_preserves_schema_ids_offsets_and_empty_results() { let mut server = start_server().await; @@ -223,7 +142,7 @@ async fn observed_session_mismatch_and_released_sessions_are_rejected() { assert_eq!(error.code(), Code::FailedPrecondition); let release = proto::ReleaseSessionRequest { session_id: session.clone(), - user_context: Some(proto::UserContext::default()), + user_context: requests::user("alice"), ..Default::default() }; let released = server @@ -251,64 +170,6 @@ async fn observed_session_mismatch_and_released_sessions_are_rejected() { server.stop().await; } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn release_all_cancels_a_pending_backend_stream_and_waits_for_teardown() { - let engine = Arc::new(FixtureEngine::default()); - let mut server = Server::start_with_engine(engine.clone()).await; - let session = Uuid::new_v4().to_string(); - let request = range(&session, LIVE_FIXTURE); - let operation = request.operation_id.clone().unwrap(); - let mut stream = server - .client - .execute_plan(request) - .await - .unwrap() - .into_inner(); - let header = stream.message().await.unwrap().unwrap(); - assert!(header.schema.is_some()); - let batch = stream.message().await.unwrap().unwrap(); - assert!(matches!( - batch.response_type, - Some(ResponseType::ArrowBatch(_)) - )); - let release = proto::ReleaseExecuteRequest { - session_id: session.clone(), - user_context: Some(proto::UserContext::default()), - operation_id: operation.clone(), - client_observed_server_side_session_id: Some(header.server_side_session_id), - release: Some(proto::release_execute_request::Release::ReleaseAll( - proto::release_execute_request::ReleaseAll {}, - )), - ..Default::default() - }; - let released = tokio::time::timeout( - Duration::from_secs(3), - server.client.release_execute(release.clone()), - ) - .await - .expect("ReleaseAll must finish without consuming the result") - .unwrap() - .into_inner(); - assert_eq!(released.operation_id.as_deref(), Some(operation.as_str())); - assert_eq!( - engine.streams_dropped.load(Ordering::SeqCst), - 1, - "ReleaseAll must wait until the backend stream is dropped" - ); - let absent = server - .client - .release_execute(release) - .await - .unwrap() - .into_inner(); - assert_eq!( - absent.operation_id, None, - "completed teardown must remove the operation" - ); - assert_cancelled(&mut stream).await; - server.stop().await; -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn arrow_payloads_are_byte_limited_and_oversized_rows_fail() { use arrow::array::{LargeStringArray, StringArray, StringViewArray}; @@ -415,232 +276,6 @@ async fn arrow_payloads_are_byte_limited_and_oversized_rows_fail() { server.stop().await; } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn release_session_cancels_live_stream_and_live_operation_ids_are_unique() { - let mut server = start_server().await; - let session = Uuid::new_v4().to_string(); - let request = range(&session, LIVE_FIXTURE); - let mut stream = server - .client - .execute_plan(request.clone()) - .await - .unwrap() - .into_inner(); - assert!(stream.message().await.unwrap().unwrap().schema.is_some()); - let duplicate = server.client.execute_plan(request).await.unwrap_err(); - assert_eq!(duplicate.code(), Code::AlreadyExists); - server - .client - .release_session(proto::ReleaseSessionRequest { - session_id: session.clone(), - user_context: Some(proto::UserContext::default()), - ..Default::default() - }) - .await - .unwrap(); - assert_cancelled(&mut stream).await; - assert_eq!( - server - .client - .execute_plan(range(&session, 0)) - .await - .unwrap_err() - .code(), - Code::FailedPrecondition - ); - server.stop().await; -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn invalid_requests_do_not_allocate_sessions_and_unknown_release_is_rejected() { - let engine = Arc::new(CountingEngine::default()); - let mut server = Server::start_with_engine(engine.clone()).await; - let valid = Uuid::new_v4().to_string(); - assert_eq!( - server - .client - .execute_plan(range("not-a-uuid", 0)) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - let mut invalid_operation = range(&valid, 0); - invalid_operation.operation_id = Some("not-a-uuid".into()); - assert_eq!( - server - .client - .execute_plan(invalid_operation) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - for plan in [None, Some(proto::Plan { op_type: None })] { - let mut request = range(&valid, 0); - request.plan = plan; - assert_eq!( - server - .client - .execute_plan(request) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - } - assert_eq!( - server - .client - .analyze_plan(proto::AnalyzePlanRequest { - session_id: valid.clone(), - ..Default::default() - }) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - for operation in [ - None, - Some(proto::config_request::Operation { op_type: None }), - ] { - assert_eq!( - server - .client - .config(proto::ConfigRequest { - session_id: valid.clone(), - operation, - ..Default::default() - }) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - } - // A stale observed ID cannot match a session that does not exist. Repeated - // requests must not consume backend session capacity while being rejected. - for _ in 0..3 { - let mut stale = range(&valid, 0); - stale.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); - assert_eq!( - server.client.execute_plan(stale).await.unwrap_err().code(), - Code::FailedPrecondition - ); - } - let mut stale = config_request( - &valid, - proto::config_request::operation::OpType::GetOption(proto::config_request::GetOption { - keys: vec!["missing".into()], - }), - ); - stale.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); - assert_eq!( - server.client.config(stale).await.unwrap_err().code(), - Code::FailedPrecondition - ); - // Identifier and envelope validation must precede backend session allocation. - assert_eq!(engine.sessions_created.load(Ordering::SeqCst), 0); - assert_eq!( - server - .client - .release_execute(release(&valid, &Uuid::new_v4().to_string())) - .await - .unwrap_err() - .code(), - Code::NotFound - ); - assert_eq!( - server - .client - .release_execute(release("not-a-uuid", &valid)) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - assert_eq!( - server - .client - .release_execute(release(&valid, "not-a-uuid")) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - let unknown = Uuid::new_v4().to_string(); - assert_eq!( - server - .client - .release_execute(release(&unknown, &valid)) - .await - .unwrap_err() - .code(), - Code::NotFound - ); - assert_eq!(engine.sessions_created.load(Ordering::SeqCst), 0); - server - .client - .config(config_request( - &valid, - proto::config_request::operation::OpType::GetOption(proto::config_request::GetOption { - keys: vec!["spark.sql.session.timeZone".into()], - }), - )) - .await - .unwrap(); - assert_eq!(engine.sessions_created.load(Ordering::SeqCst), 1); - server.stop().await; -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn disconnect_removes_operation_and_shutdown_cancels_live_streams() { - let mut server = start_server().await; - let session = Uuid::new_v4().to_string(); - let request = range(&session, LIVE_FIXTURE); - let operation = request.operation_id.clone().unwrap(); - let mut abandoned = server - .client - .execute_plan(request.clone()) - .await - .unwrap() - .into_inner(); - assert!(abandoned.message().await.unwrap().unwrap().schema.is_some()); - drop(abandoned); - let mut resumed = tokio::time::timeout(Duration::from_secs(5), async { - loop { - match server.client.execute_plan(request.clone()).await { - Ok(response) => break response.into_inner(), - Err(status) if status.code() == Code::AlreadyExists => { - tokio::time::sleep(Duration::from_millis(20)).await; - } - Err(status) => panic!("unexpected retry failure: {status}"), - } - } - }) - .await - .expect("disconnect must remove the operation registry entry"); - assert!(resumed.message().await.unwrap().unwrap().schema.is_some()); - server - .client - .release_execute(release(&session, &operation)) - .await - .unwrap(); - assert_cancelled(&mut resumed).await; - - let mut live = server - .client - .execute_plan(range(&session, LIVE_FIXTURE)) - .await - .unwrap() - .into_inner(); - assert!(live.message().await.unwrap().unwrap().schema.is_some()); - server.shutdown.take().unwrap().send(()).unwrap(); - assert_cancelled(&mut live).await; - server.stop().await; -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn backend_status_codes_survive_prepare_and_stream_failures() { let engine = FixtureEngine::default() diff --git a/crates/spark-connect-server/tests/schema_consistency.rs b/crates/spark-connect-server/tests/schema_consistency.rs index efb9e8df..24f9bde5 100644 --- a/crates/spark-connect-server/tests/schema_consistency.rs +++ b/crates/spark-connect-server/tests/schema_consistency.rs @@ -17,8 +17,11 @@ //! Result schemas remain consumable as one Arrow table across output batches. +#[path = "support/requests.rs"] +mod requests; #[path = "support/server.rs"] mod server; +mod support; use std::collections::HashMap; use std::io::Cursor; @@ -35,75 +38,20 @@ use arrow::ipc::{reader::StreamReader, writer::StreamWriter}; use arrow::record_batch::RecordBatch; use prost::Message; use spark_connect_proto as proto; -use spark_connect_server::engine::{ - BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, - ResultBatch, SessionContext, -}; use tonic::{Code, Status}; use uuid::Uuid; use server::Server; +use support::{Batch, FixtureEngine}; const RESULT_LIMIT: usize = 3 * 1024 * 1024; const LARGE_ROWS: usize = 4096; -#[derive(Clone)] -enum Batch { - Arrow(RecordBatch), - Ipc(Vec, i64), -} - -#[derive(Clone)] -struct BatchEngine { +struct BatchCase { schema: SchemaRef, batches: Vec, } -impl Engine for BatchEngine { - fn create_session(&self, _: SessionContext) -> Result, Status> { - Ok(Arc::new(self.clone())) - } -} - -#[tonic::async_trait] -impl EngineSession for BatchEngine { - async fn prepare(&self, _: proto::plan::OpType) -> Result { - Ok(PreparedOperation::Query(Box::new(self.clone()))) - } - - async fn analyze( - &self, - _: proto::analyze_plan_request::Analyze, - ) -> Result { - Err(Status::unimplemented("schema fixture has no analysis")) - } - - async fn config( - &self, - _: proto::config_request::operation::OpType, - ) -> Result { - Err(Status::unimplemented("schema fixture has no configuration")) - } -} - -#[tonic::async_trait] -impl PreparedQuery for BatchEngine { - fn schema(&self) -> SchemaRef { - self.schema.clone() - } - - async fn execute(self: Box) -> Result { - Ok(Box::pin(futures::stream::iter( - self.batches.into_iter().map(|batch| { - Ok(match batch { - Batch::Arrow(batch) => ResultBatch::Arrow(batch), - Batch::Ipc(data, row_count) => ResultBatch::Ipc { data, row_count }, - }) - }), - ))) - } -} - fn ipc(batch: &RecordBatch) -> Vec { let mut data = Vec::new(); let mut writer = StreamWriter::try_new(&mut data, batch.schema().as_ref()).unwrap(); @@ -113,23 +61,10 @@ fn ipc(batch: &RecordBatch) -> Vec { data } -async fn execute(engine: BatchEngine) -> (Vec, Option) { +async fn execute(engine: BatchCase) -> (Vec, Option) { + let engine = FixtureEngine::default().with_results(1, engine.schema, engine.batches); let mut server = Server::start_with_engine(Arc::new(engine)).await; - let request = proto::ExecutePlanRequest { - session_id: Uuid::new_v4().to_string(), - operation_id: Some(Uuid::new_v4().to_string()), - plan: Some(proto::Plan { - op_type: Some(proto::plan::OpType::Root(proto::Relation { - rel_type: Some(proto::relation::RelType::Range(proto::Range { - end: 1, - step: 1, - ..Default::default() - })), - ..Default::default() - })), - }), - ..Default::default() - }; + let request = requests::request(&Uuid::new_v4().to_string(), 1); let mut stream = server .client .execute_plan(request) @@ -215,7 +150,7 @@ async fn changed_physical_types_and_nullability_fail_before_the_second_batch() { strings(DataType::LargeUtf8, false), strings(DataType::Utf8, true), ] { - let (responses, status) = execute(BatchEngine { + let (responses, status) = execute(BatchCase { schema: first.schema(), batches: vec![Batch::Arrow(first.clone()), Batch::Arrow(second)], }) @@ -254,7 +189,7 @@ async fn metadata_can_change_and_batch_schema_can_differ_from_the_declaration() DataType::List(Arc::new(Field::new("item", DataType::Utf8, false))), false, )])); - let (responses, status) = execute(BatchEngine { + let (responses, status) = execute(BatchCase { schema: declared, batches: vec![ Batch::Arrow(metadata_batch("first")), @@ -304,7 +239,7 @@ async fn view_ipc_is_unchanged_when_small_and_materialized_when_oversized() { let batch = views(rows); let data = ipc(&batch); assert_eq!(data.len() > RESULT_LIMIT, rows == LARGE_ROWS); - let (responses, status) = execute(BatchEngine { + let (responses, status) = execute(BatchCase { schema: batch.schema(), batches: vec![Batch::Ipc(data.clone(), rows as i64)], }) @@ -354,7 +289,7 @@ async fn mixing_small_and_oversized_view_ipc_rejects_the_changed_output_schema() let small = Batch::Ipc(ipc(&small), 2); let large = Batch::Ipc(ipc(&large), LARGE_ROWS as i64); for large_first in [false, true] { - let (responses, status) = execute(BatchEngine { + let (responses, status) = execute(BatchCase { schema: views(0).schema(), batches: if large_first { vec![large.clone(), small.clone()] @@ -425,7 +360,7 @@ async fn view_children_in_lists_large_lists_and_maps_are_materialized_when_split .collect::>(), )); let batch = RecordBatch::try_new(schema.clone(), columns).unwrap(); - let (responses, status) = execute(BatchEngine { + let (responses, status) = execute(BatchCase { schema, batches: vec![Batch::Arrow(batch)], }) diff --git a/crates/spark-connect-server/tests/support/mod.rs b/crates/spark-connect-server/tests/support/mod.rs index 0df3a439..6723e0aa 100644 --- a/crates/spark-connect-server/tests/support/mod.rs +++ b/crates/spark-connect-server/tests/support/mod.rs @@ -15,22 +15,20 @@ // specific language governing permissions and limitations // under the License. -//! Explicit backend outputs for frontend protocol tests. +//! Explicit backend outputs for frontend protocol and schema tests. //! //! Range.end is only a fixture lookup key. This backend neither interprets SQL -//! nor implements Spark operators; each case supplies Arrow batches, a pending -//! stream, or a chosen failure. These tests do not verify query execution semantics. +//! nor implements Spark operators; each case supplies batches or a chosen error. +//! Test binaries use different subsets of the fixture constructors. + +#![allow(dead_code)] use std::collections::HashMap; -use std::pin::Pin; -use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -use std::task::{Context, Poll}; use arrow::array::Int64Array; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; -use futures::{Stream, StreamExt}; use spark_connect_proto as proto; use spark_connect_server::engine::{ BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, @@ -38,28 +36,25 @@ use spark_connect_server::engine::{ }; use tonic::{Code, Status}; -pub const LIVE_FIXTURE: i64 = 1_000_000_000_000; - #[derive(Clone)] -enum Ending { - Complete, - Pending, - Error(Code), +pub enum Batch { + Arrow(RecordBatch), + Ipc(Vec, i64), } #[derive(Clone)] enum Output { Query { schema: SchemaRef, - batches: Vec, - ending: Ending, + batches: Vec, + error: Option, }, Rejected(Code), } +#[derive(Clone)] pub struct FixtureEngine { outputs: HashMap, - pub streams_dropped: Arc, } impl Default for FixtureEngine { @@ -72,59 +67,35 @@ impl Default for FixtureEngine { ) .unwrap() }; - let mut engine = Self { + Self { outputs: HashMap::new(), - streams_dropped: Arc::new(AtomicUsize::new(0)), - }; - engine.outputs.insert( - 0, - Output::Query { - schema: schema.clone(), - batches: Vec::new(), - ending: Ending::Complete, - }, - ); - engine.outputs.insert( - 1, - Output::Query { - schema: schema.clone(), - batches: vec![batch(0, 1)], - ending: Ending::Complete, - }, - ); - engine.outputs.insert( + } + .with_batches(0, schema.clone(), Vec::new()) + .with_batches(1, schema.clone(), vec![batch(0, 1)]) + .with_batches( 25_000, - Output::Query { - schema: schema.clone(), - batches: vec![batch(0, 8192), batch(8192, 16_384), batch(16_384, 25_000)], - ending: Ending::Complete, - }, - ); - engine.outputs.insert( - LIVE_FIXTURE, - Output::Query { - schema: schema.clone(), - batches: vec![batch(0, 8192)], - ending: Ending::Pending, - }, - ); - engine + schema.clone(), + vec![batch(0, 8192), batch(8192, 16_384), batch(16_384, 25_000)], + ) } } impl FixtureEngine { - pub fn with_batches( - mut self, - selector: i64, - schema: SchemaRef, - batches: Vec, - ) -> Self { + pub fn with_batches(self, selector: i64, schema: SchemaRef, batches: Vec) -> Self { + self.with_results( + selector, + schema, + batches.into_iter().map(Batch::Arrow).collect(), + ) + } + + pub fn with_results(mut self, selector: i64, schema: SchemaRef, batches: Vec) -> Self { self.outputs.insert( selector, Output::Query { schema, batches, - ending: Ending::Complete, + error: None, }, ); self @@ -141,7 +112,7 @@ impl FixtureEngine { Output::Query { schema: Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), batches: Vec::new(), - ending: Ending::Error(code), + error: Some(code), }, ); self @@ -149,21 +120,13 @@ impl FixtureEngine { } impl Engine for FixtureEngine { - fn create_session(&self, _context: SessionContext) -> Result, Status> { - Ok(Arc::new(FixtureSession { - outputs: self.outputs.clone(), - streams_dropped: self.streams_dropped.clone(), - })) + fn create_session(&self, _: SessionContext) -> Result, Status> { + Ok(Arc::new(self.clone())) } } -struct FixtureSession { - outputs: HashMap, - streams_dropped: Arc, -} - #[tonic::async_trait] -impl EngineSession for FixtureSession { +impl EngineSession for FixtureEngine { async fn prepare(&self, operation: proto::plan::OpType) -> Result { let proto::plan::OpType::Root(relation) = operation else { return Err(Status::unimplemented("fixture accepts root selectors only")); @@ -175,12 +138,11 @@ impl EngineSession for FixtureSession { Some(Output::Query { schema, batches, - ending, + error, }) => Ok(PreparedOperation::Query(Box::new(FixtureQuery { schema: schema.clone(), batches: batches.clone(), - ending: ending.clone(), - streams_dropped: self.streams_dropped.clone(), + error: *error, }))), Some(Output::Rejected(code)) => Err(Status::new(*code, "prepared fixture error")), None => Err(Status::unimplemented("unknown fixture selector")), @@ -189,7 +151,7 @@ impl EngineSession for FixtureSession { async fn analyze( &self, - _input: proto::analyze_plan_request::Analyze, + _: proto::analyze_plan_request::Analyze, ) -> Result { Err(Status::unimplemented( "fixture does not analyze query semantics", @@ -198,29 +160,18 @@ impl EngineSession for FixtureSession { async fn config( &self, - input: proto::config_request::operation::OpType, + _: proto::config_request::operation::OpType, ) -> Result { - let proto::config_request::operation::OpType::GetOption(input) = input else { - return Err(Status::unimplemented( - "fixture supports optional config lookup only", - )); - }; - Ok(ConfigResult { - pairs: input - .keys - .into_iter() - .map(|key| proto::KeyValue { key, value: None }) - .collect(), - warnings: Vec::new(), - }) + Err(Status::unimplemented( + "fixture does not configure query semantics", + )) } } struct FixtureQuery { schema: SchemaRef, - batches: Vec, - ending: Ending, - streams_dropped: Arc, + batches: Vec, + error: Option, } #[tonic::async_trait] @@ -230,36 +181,16 @@ impl PreparedQuery for FixtureQuery { } async fn execute(self: Box) -> Result { - let first = futures::stream::iter(self.batches.into_iter().map(|batch| Ok(batch.into()))); - let inner: BatchStream = match self.ending { - Ending::Complete => Box::pin(first), - Ending::Pending => Box::pin(first.chain(futures::stream::pending())), - Ending::Error(code) => Box::pin(first.chain(futures::stream::once(async move { - Err(Status::new(code, "streamed fixture error")) - }))), - }; - Ok(Box::pin(TrackedStream { - inner, - streams_dropped: self.streams_dropped, - })) - } -} - -struct TrackedStream { - inner: BatchStream, - streams_dropped: Arc, -} - -impl Stream for TrackedStream { - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { - self.inner.as_mut().poll_next(context) - } -} - -impl Drop for TrackedStream { - fn drop(&mut self) { - self.streams_dropped.fetch_add(1, Ordering::SeqCst); + let batches = self.batches.into_iter().map(|batch| { + Ok(match batch { + Batch::Arrow(batch) => ResultBatch::Arrow(batch), + Batch::Ipc(data, row_count) => ResultBatch::Ipc { data, row_count }, + }) + }); + let failure = self + .error + .into_iter() + .map(|code| Err(Status::new(code, "streamed fixture error"))); + Ok(Box::pin(futures::stream::iter(batches.chain(failure)))) } } diff --git a/crates/spark-connect-server/tests/support/requests.rs b/crates/spark-connect-server/tests/support/requests.rs new file mode 100644 index 00000000..6b054034 --- /dev/null +++ b/crates/spark-connect-server/tests/support/requests.rs @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared request envelopes and terminal-stream assertions. Individual test +//! binaries use different subsets of these helpers. + +#![allow(dead_code)] + +use std::time::Duration; + +use spark_connect_proto as proto; +use tonic::Code; +use uuid::Uuid; + +pub fn user(id: &str) -> Option { + Some(proto::UserContext { + user_id: id.into(), + ..Default::default() + }) +} + +pub fn relation(end: i64) -> proto::Relation { + proto::Relation { + rel_type: Some(proto::relation::RelType::Range(proto::Range { + end, + step: 1, + ..Default::default() + })), + ..Default::default() + } +} + +pub fn request(session: &str, end: i64) -> proto::ExecutePlanRequest { + proto::ExecutePlanRequest { + session_id: session.into(), + user_context: user("alice"), + operation_id: Some(Uuid::new_v4().to_string()), + plan: Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(relation(end))), + }), + ..Default::default() + } +} + +pub fn config_request( + session: &str, + input: proto::config_request::operation::OpType, +) -> proto::ConfigRequest { + proto::ConfigRequest { + session_id: session.into(), + user_context: user("alice"), + operation: Some(proto::config_request::Operation { + op_type: Some(input), + }), + ..Default::default() + } +} + +pub fn release(session: &str, operation: &str) -> proto::ReleaseExecuteRequest { + proto::ReleaseExecuteRequest { + session_id: session.into(), + user_context: user("alice"), + operation_id: operation.into(), + release: Some(proto::release_execute_request::Release::ReleaseAll( + proto::release_execute_request::ReleaseAll {}, + )), + ..Default::default() + } +} + +pub fn release_session(session: &str) -> proto::ReleaseSessionRequest { + proto::ReleaseSessionRequest { + session_id: session.into(), + user_context: user("alice"), + ..Default::default() + } +} + +pub async fn assert_cancelled(stream: &mut tonic::Streaming) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + match stream.message().await { + Err(status) => { + assert_eq!(status.code(), Code::Cancelled); + break; + } + Ok(Some(response)) => assert!(!matches!( + response.response_type, + Some(ResponseType::ResultComplete(_)) + )), + Ok(None) => panic!("cancelled execution must report cancellation"), + } + } + }) + .await + .expect("cancelled execution must terminate"); +} From 609f3c43e2e7f67d72809b23f557369eb27c90f6 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 20:28:39 -0800 Subject: [PATCH 13/17] Import shared cancellation response variant --- crates/spark-connect-server/tests/support/requests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/spark-connect-server/tests/support/requests.rs b/crates/spark-connect-server/tests/support/requests.rs index 6b054034..185fce4a 100644 --- a/crates/spark-connect-server/tests/support/requests.rs +++ b/crates/spark-connect-server/tests/support/requests.rs @@ -22,6 +22,7 @@ use std::time::Duration; +use proto::execute_plan_response::ResponseType; use spark_connect_proto as proto; use tonic::Code; use uuid::Uuid; From 28b44e51633821011598c12a8c6933ad813559c1 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 20:44:20 -0800 Subject: [PATCH 14/17] Preserve Arrow IPC with negotiated byte chunking --- crates/spark-connect-server/README.md | 42 +- crates/spark-connect-server/src/engine/mod.rs | 25 +- crates/spark-connect-server/src/lib.rs | 34 +- crates/spark-connect-server/src/service.rs | 1053 ++--------------- .../tests/engine_injection.rs | 581 ++++++--- crates/spark-connect-server/tests/protocol.rs | 723 ++++++++--- .../tests/review_regressions.rs | 652 ---------- .../tests/schema_consistency.rs | 125 +- .../tests/support/arrow.rs | 112 ++ .../tests/support/server.rs | 9 +- 10 files changed, 1383 insertions(+), 1973 deletions(-) delete mode 100644 crates/spark-connect-server/tests/review_regressions.rs create mode 100644 crates/spark-connect-server/tests/support/arrow.rs diff --git a/crates/spark-connect-server/README.md b/crates/spark-connect-server/README.md index 620a7df6..3b4ac6db 100644 --- a/crates/spark-connect-server/README.md +++ b/crates/spark-connect-server/README.md @@ -23,6 +23,8 @@ This unpublished library serves a subset of the Spark Connect protocol using the workspace's `apache-spark-connect-proto` messages and its optional `server` bindings. The caller supplies an `Engine` implementation and a bound Tokio TCP listener to `serve_with_engine`. +Use `serve_with_engine_and_options` with `ServerOptions` to configure Arrow +chunking. No execution engine or standalone server executable is included. Rust owns protocol sessions, operation IDs, response framing and result-stream @@ -60,16 +62,31 @@ assemble them into one Arrow table. Metadata may differ between batches. Declared field metadata and unsupported types fail explicitly. Engines may yield Arrow record batches or complete Arrow IPC streams containing -one batch and its declared row count. The frontend encodes record batches and -forwards IPC that fits the response limit; oversized batches are split by rows. -Encoding materializes string/binary view arrays, including nested children, to -`LargeUtf8`/`LargeBinary`. Small view-typed IPC remains unchanged, so mixing it with -oversized view-typed IPC can produce different physical schemas. The frontend -reports `INTERNAL` before sending a batch whose output schema differs from earlier -batches; it does not complete a result containing mixed schemas. Engines should -use consistent non-view IPC types when mixing batch sizes. -Result envelopes, including schema and batch responses, are limited to 3 MiB. -A single row or schema that cannot fit fails with `RESOURCE_EXHAUSTED`. Each batch +one batch and its actual nonnegative row count. The frontend encodes each record +batch once and forwards existing IPC bytes unchanged, including compressed IPC. +It reads the IPC schema for validation and trusts the producer's batch contents +and declared row count. Record-batch encoding materializes string/binary view +arrays, including nested children, to `LargeUtf8`/`LargeBinary`. Existing IPC +retains its physical types. The frontend reports `INTERNAL` before sending a +batch whose output schema differs from earlier batches; engines mixing record +batches with IPC must use consistent output representations. + +When the client sets `ResultChunkingOptions.allow_arrow_batch_chunking`, the +frontend divides serialized IPC bytes into chunks and sets `chunk_index` and +`num_chunks_in_batch`. Boundaries need not align with rows or Arrow buffers; the +client concatenates a batch's chunks before decoding or decompressing it. A +preferred chunk payload size between 1 KiB and the configured maximum is accepted; +otherwise the maximum is used. `ServerOptions.max_arrow_chunk_size` defaults to +120,795,955 bytes (90% of 128 MiB), matching the JVM server's default chunk payload +limit. It limits IPC payload bytes, excluding the protobuf envelope. Setting it +to zero disables chunking even when the client opts in. Clients must allow room +for the envelope in their gRPC receive limits. + +Without client opt-in, or when chunking is disabled, each IPC batch is sent whole. +Engines remain responsible for bounding batch production, including fitting the +client's gRPC receive limit when batches are sent whole. The frontend applies no +128 MiB batch-production cap. Schema and command response envelopes have a +separate 3 MiB limit and fail with `RESOURCE_EXHAUSTED` when oversized. Each batch carries a cumulative row offset, and empty results still include an empty batch. ## Lifetime and trust requirements @@ -79,7 +96,10 @@ frontend cancels them when a client disconnects, an operation is released, a session is released, server shutdown begins or the serving future is dropped. Results pass through a bounded channel, so unread responses eventually stop polling the engine. Blocking Arrow -encoding retains operation ownership until its worker finishes. +encoding retains operation ownership until its worker finishes. IPC chunks share +their batch's backing allocation, which remains live while any chunk references +it. Larger batches and chunk limits can therefore increase retained memory even +though the response channel is bounded. `ReleaseAll` waits up to two seconds for teardown and reports `DEADLINE_EXCEEDED` if it is still draining. Shutdown cancels sessions and waits up to two seconds diff --git a/crates/spark-connect-server/src/engine/mod.rs b/crates/spark-connect-server/src/engine/mod.rs index e2285e67..edfadfd4 100644 --- a/crates/spark-connect-server/src/engine/mod.rs +++ b/crates/spark-connect-server/src/engine/mod.rs @@ -101,11 +101,11 @@ pub trait PreparedQuery: Send { /// equivalent Arrow representations are accepted. /// Batch nullability and metadata may differ from this declaration, but /// emitted IPC batches in one result must share physical types, field names - /// and nullability. Metadata may differ between batches. Encoding materializes - /// view arrays to large-offset types, including when splitting oversized IPC; - /// mixing those with unchanged view IPC fails explicitly before the changed - /// output schema is sent. Declared fields must not contain metadata, which - /// Spark schema conversion rejects. + /// and nullability. Metadata may differ between batches. Record-batch encoding + /// materializes view arrays to large-offset types; existing IPC retains its + /// physical types. Mixing incompatible output representations fails before + /// the changed output schema is sent. Declared fields must not contain + /// metadata, which Spark schema conversion rejects. fn schema(&self) -> SchemaRef; /// Dropping this future or its returned stream must cancel backend work; @@ -114,14 +114,21 @@ pub trait PreparedQuery: Send { } /// Engine output, optionally already serialized for Spark Connect. +/// +/// Engines must bound batch production. Optional byte chunking bounds network +/// payloads, not the complete batch's allocation. When chunking is disabled or +/// the client does not opt in, each batch must fit the client's gRPC receive limit. pub enum ResultBatch { + /// Encoded once as a complete IPC stream before optional byte chunking. Arrow(RecordBatch), /// A complete IPC stream containing its schema, exactly one record batch, /// and EOS. The producer must supply that batch's actual nonnegative row count. - Ipc { - data: Vec, - row_count: i64, - }, + /// The frontend validates the schema and trusts the producer's batch contents + /// and declared row count. IPC bytes, including compression, are preserved. + /// Clients that opt in receive byte chunks for reassembly before decoding; + /// otherwise the whole stream is sent, so producers must respect the client's + /// gRPC receive limit when selecting batch sizes. + Ipc { data: Vec, row_count: i64 }, } impl From for ResultBatch { diff --git a/crates/spark-connect-server/src/lib.rs b/crates/spark-connect-server/src/lib.rs index 64db976b..f42f1895 100644 --- a/crates/spark-connect-server/src/lib.rs +++ b/crates/spark-connect-server/src/lib.rs @@ -25,6 +25,24 @@ pub mod grpc { pub use spark_connect_proto::spark_connect_service_server; } +/// Transport options for the experimental frontend. +#[derive(Clone, Debug)] +pub struct ServerOptions { + /// Maximum Arrow chunk payload in bytes, excluding the protobuf envelope. + /// Defaults to 90% of 128 MiB, as in Spark's JVM server. A client can request + /// a smaller size. Zero disables chunking, even when the client opts in. + /// Whole batches sent without chunking are not bounded by this option. + pub max_arrow_chunk_size: usize, +} + +impl Default for ServerOptions { + fn default() -> Self { + Self { + max_arrow_chunk_size: 128 * 1024 * 1024 * 9 / 10, + } + } +} + struct CancelOnDrop(std::sync::Arc); impl Drop for CancelOnDrop { @@ -49,7 +67,21 @@ pub async fn serve_with_engine( engine: std::sync::Arc, shutdown: impl std::future::Future + Send + 'static, ) -> Result<(), tonic::transport::Error> { - let service = std::sync::Arc::new(service::ConnectService::new(engine)); + serve_with_engine_and_options(listener, engine, ServerOptions::default(), shutdown).await +} + +/// Serve with an injected engine and explicit transport options. +/// +/// Session lifetime and shutdown follow [`serve_with_engine`]. Producers should +/// bound engine batches independently of this transport chunk size. Clients must +/// allow protobuf overhead in their gRPC receive limits. +pub async fn serve_with_engine_and_options( + listener: tokio::net::TcpListener, + engine: std::sync::Arc, + options: ServerOptions, + shutdown: impl std::future::Future + Send + 'static, +) -> Result<(), tonic::transport::Error> { + let service = std::sync::Arc::new(service::ConnectService::new(engine, options)); let _cancel_on_drop = CancelOnDrop(service.clone()); let router = tonic::transport::Server::builder().add_service( grpc::spark_connect_service_server::SparkConnectServiceServer::from_arc(service.clone()) diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index 11baf810..f2191251 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -70,8 +70,6 @@ struct Session { struct Operation { cancelled: watch::Sender, done: watch::Sender, - #[cfg(test)] - encoding_attempts: std::sync::atomic::AtomicUsize, } impl Session { @@ -109,8 +107,6 @@ impl Session { let running = Arc::new(Operation { cancelled, done: watch::channel(false).0, - #[cfg(test)] - encoding_attempts: std::sync::atomic::AtomicUsize::new(0), }); let mut operations = self .operations @@ -181,14 +177,16 @@ impl SessionRegistry { pub(crate) struct ConnectService { engine: Arc, + options: crate::ServerOptions, sessions: Mutex, closing: std::sync::atomic::AtomicBool, } impl ConnectService { - pub(crate) fn new(engine: Arc) -> Self { + pub(crate) fn new(engine: Arc, options: crate::ServerOptions) -> Self { Self { engine, + options, sessions: Mutex::new(SessionRegistry::default()), closing: std::sync::atomic::AtomicBool::new(false), } @@ -315,10 +313,8 @@ fn encode_batch(batch: &RecordBatch) -> Result, Status> { use arrow::datatypes::DataType; use arrow::record_batch::RecordBatchOptions; - // IPC writes all backing buffers of view arrays even after a row slice. Materialize - // their bytes, including nested children, so splitting measures only retained rows. - // Use 64-bit offsets so materializing a batch cannot overflow 32-bit byte offsets - // before the result-size check has a chance to split it. + // Materialize view arrays, including nested children, into the large-offset + // string/binary representation used for encoded RecordBatch results. let source_schema = batch.schema(); let fields = schema::map_fields(source_schema.fields(), &|field, data_type| { let data_type = match data_type { @@ -400,83 +396,6 @@ fn check_output_schema(schema: &Schema, expected: &mut Option) -> Result } } -fn estimated_array_bytes(array: &dyn arrow::array::Array) -> usize { - use arrow::array::{Array, AsArray, OffsetSizeTrait}; - use arrow::datatypes::DataType; - - fn nested_bytes(offsets: &[O], values: &dyn Array) -> usize { - let start = offsets[0].as_usize(); - let end = offsets[offsets.len() - 1].as_usize(); - // List and map slices retain their full child arrays. Like the IPC - // writer, inspect only the child range referenced by these offsets. - offsets - .len() - .saturating_mul(std::mem::size_of::()) - .saturating_add(estimated_array_bytes( - values.slice(start, end - start).as_ref(), - )) - } - - fn view_bytes(len: usize, lengths: impl Iterator) -> usize { - // Encoding materializes views with 64-bit offsets. Count repeated - // references separately, including null-slot lengths: Arrow's cast - // reserves their bytes even though it emits no value for a null slot. - lengths.fold(len.saturating_add(1).saturating_mul(8), |bytes, len| { - bytes.saturating_add(len as usize) - }) - } - - let bytes = match array.data_type() { - DataType::List(_) => { - let list = array.as_list::(); - nested_bytes(list.value_offsets(), list.values().as_ref()) - } - DataType::LargeList(_) => { - let list = array.as_list::(); - nested_bytes(list.value_offsets(), list.values().as_ref()) - } - DataType::Map(_, _) => { - let map = array.as_map(); - nested_bytes(map.value_offsets(), map.entries()) - } - DataType::Struct(_) => array - .as_struct() - .columns() - .iter() - .fold(0usize, |bytes, child| { - bytes.saturating_add(estimated_array_bytes(child.as_ref())) - }), - DataType::Utf8View => view_bytes(array.len(), array.as_string_view().lengths()), - DataType::BinaryView => view_bytes(array.len(), array.as_binary_view().lengths()), - _ => { - return array - .to_data() - .get_slice_memory_size() - .unwrap_or_else(|_| array.get_array_memory_size()); - } - }; - bytes.saturating_add(array.nulls().map_or(0, |_| array.len().div_ceil(8))) -} - -#[cfg(test)] -fn estimated_rows_per_batch(batch: &RecordBatch) -> usize { - estimated_rows_for_payload(batch, MAX_RESULT_BYTES / 2) -} - -fn estimated_rows_for_payload(batch: &RecordBatch, payload_bytes: usize) -> usize { - let bytes = batch.columns().iter().fold(0usize, |bytes, column| { - bytes.saturating_add(estimated_array_bytes(column.as_ref())) - }); - // Start below the exact IPC limit to allow schema/offset overhead. This is - // only an estimate: skewed rows and annotations are checked after encoding. - rows_for_bytes(batch.num_rows(), bytes, payload_bytes).min(batch.num_rows().max(1)) -} - -fn rows_for_bytes(rows: usize, bytes: usize, payload_bytes: usize) -> usize { - (rows as u128 * payload_bytes as u128 / bytes.max(1) as u128).clamp(1, usize::MAX as u128) - as usize -} - fn checked_arrow_row_count(rows: usize, offset: i64) -> Result { let rows = i64::try_from(rows) .map_err(|_| Status::resource_exhausted("result batch row count exceeds i64"))?; @@ -486,25 +405,46 @@ fn checked_arrow_row_count(rows: usize, offset: i64) -> Result { Ok(rows) } -// Fields drop in declaration order, including when encoding panics or a -// cancelled caller drops a completed JoinHandle without observing its output. +// Keep backend ownership until the batch is dropped, including on encoder panic. struct EncodingBatch { batch: RecordBatch, _guard: Arc, } impl EncodingBatch { - fn encode(self) -> Result<(Self, Vec), Status> { - #[cfg(test)] - self._guard - .running - .encoding_attempts - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - let data = encode_batch(&self.batch)?; - Ok((self, data)) + fn encode(self) -> Result, Status> { + encode_batch(&self.batch) } } +fn arrow_chunk_size( + options: &[proto::execute_plan_request::RequestOption], + maximum: usize, +) -> Option { + use proto::execute_plan_request::request_option::RequestOption; + + let mut options = options + .iter() + .filter_map(|option| match &option.request_option { + Some(RequestOption::ResultChunkingOptions(options)) => Some(options), + _ => None, + }); + if maximum == 0 + || !options + .clone() + .any(|options| options.allow_arrow_batch_chunking) + { + return None; + } + Some( + options + .find_map(|options| options.preferred_arrow_chunk_size) + .and_then(|bytes| usize::try_from(bytes).ok()) + .filter(|bytes| (1024..=maximum).contains(bytes)) + .unwrap_or(maximum), + ) +} + async fn send_batch( sender: &mpsc::Sender>, id: &str, @@ -514,172 +454,74 @@ async fn send_batch( offset: &mut i64, output_schema: &mut Option, guard: &Arc, + chunk_size: Option, ) -> Result { - let (batch, rows_per_batch) = match batch { - ResultBatch::Arrow(batch) => { - let rows = batch.num_rows().max(1); - (batch, rows) + // Reject unrepresentable counts/offsets before encoding or sending any chunk. + let row_count = match &batch { + ResultBatch::Arrow(batch) => checked_arrow_row_count(batch.num_rows(), *offset)?, + ResultBatch::Ipc { row_count, .. } if *row_count < 0 => { + return Err(Status::internal("negative engine IPC row count")); } - ResultBatch::Ipc { data, row_count } => { - if row_count < 0 { - return Err(Status::internal("negative engine IPC row count")); - } - let next_offset = offset - .checked_add(row_count) - .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; - let batch = proto::execute_plan_response::ArrowBatch { - row_count, - data: data.into(), - start_offset: Some(*offset), - ..Default::default() + ResultBatch::Ipc { row_count, .. } => *row_count, + }; + let next_offset = offset + .checked_add(row_count) + .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; + let data: prost::bytes::Bytes = match batch { + ResultBatch::Arrow(batch) => { + // A blocking encoder cannot be cancelled once started. Retain engine + // ownership until its batch is dropped, even if the caller goes away. + let encoding = EncodingBatch { + batch, + _guard: guard.clone(), }; - let result = response( - id, - &guard.session, - operation, - Some(proto::execute_plan_response::ResponseType::ArrowBatch( - batch.clone(), - )), - ); - if result.encoded_len() <= MAX_RESULT_BYTES { - check_output_schema(batch_schema, output_schema)?; - *offset = next_offset; - return Ok(sender.send(Ok(result)).await.is_ok()); - } - drop(result); - // Ordinary IPC is forwarded unchanged. Only oversized payloads need - // vectors so the existing row splitter can enforce the envelope limit. - let guard = guard.clone(); - tokio::task::spawn_blocking(move || { - let _guard = guard; - let ipc_bytes = batch.data.len(); - let mut reader = StreamReader::try_new(&batch.data[..], None) - .map_err(|e| Status::internal(format!("invalid engine IPC: {e}")))?; - let batch = reader - .next() - .transpose() - .map_err(|e| Status::internal(format!("invalid engine IPC: {e}")))? - .ok_or_else(|| Status::internal("engine IPC contains no record batch"))?; - if i64::try_from(batch.num_rows()).ok() != Some(row_count) - || reader.next().is_some() - { - return Err(Status::internal( - "engine IPC batch count differs from its declaration", - )); - } - // Seed oversized IPC splitting from its serialized size; the - // payload-aware preflight refines this before encoding. - let rows = rows_for_bytes(batch.num_rows(), ipc_bytes, MAX_RESULT_BYTES / 2); - Ok((batch, rows)) - }) - .await - .map_err(|_| Status::internal("Arrow IPC decoding task failed"))?? + let data = tokio::task::spawn_blocking(move || encoding.encode()) + .await + .map_err(|_| Status::internal("Arrow result encoding task failed"))??; + let schema = StreamReader::try_new(data.as_slice(), None) + .map_err(|e| Status::internal(format!("invalid output IPC: {e}")))? + .schema(); + check_output_schema(schema.as_ref(), output_schema)?; + data.into() + } + ResultBatch::Ipc { data, .. } => { + check_output_schema(batch_schema, output_schema)?; + data.into() } }; - send_arrow_batch( - sender, - id, - operation, - batch, - rows_per_batch, - offset, - output_schema, - guard, - ) - .await -} - -async fn send_arrow_batch( - sender: &mpsc::Sender>, - id: &str, - operation: &str, - batch: RecordBatch, - rows_per_batch: usize, - offset: &mut i64, - output_schema: &mut Option, - guard: &Arc, -) -> Result { - // Validate the complete engine batch before slicing or IPC encoding. Even - // buffer-free arrays can represent more rows than the signed protocol. - checked_arrow_row_count(batch.num_rows(), *offset)?; - // Schema metadata and wide schemas repeat in every IPC response. Measure - // their fixed cost once, using fresh empty arrays so nested view buffers - // retained by a zero-row slice cannot be materialized here. - let empty = EncodingBatch { - batch: RecordBatch::new_empty(batch.schema()), - _guard: guard.clone(), + let mut payload = proto::execute_plan_response::ArrowBatch { + row_count, + start_offset: Some(*offset), + ..Default::default() }; - let (empty, data) = tokio::task::spawn_blocking(move || empty.encode()) - .await - .map_err(|_| Status::internal("Arrow result encoding task failed"))??; - let fixed_bytes = response( - id, - &guard.session, - operation, - Some(proto::execute_plan_response::ResponseType::ArrowBatch( - proto::execute_plan_response::ArrowBatch { - row_count: 0, - data: data.into(), - start_offset: Some(*offset), - ..Default::default() - }, - )), - ) - .encoded_len(); - drop(empty); - let payload_bytes = (MAX_RESULT_BYTES.saturating_sub(fixed_bytes) / 2).max(1); - let mut pending = vec![(batch, rows_per_batch)]; - while let Some((batch, rows_per_batch)) = pending.pop() { - let mut rows_per_batch = rows_per_batch.max(1).min(batch.num_rows().max(1)); - while rows_per_batch > 1 { - let prefix; - let candidate = if rows_per_batch >= batch.num_rows() { - &batch - } else { - prefix = batch.slice(0, rows_per_batch); - &prefix - }; - let estimated = estimated_rows_for_payload(candidate, payload_bytes); - if estimated >= candidate.num_rows() { - break; - } - // Growth from a light prefix can reach a dense tail. Inspect its - // logical buffers before encoding rather than repeatedly copying - // an already-known oversized payload. Exact IPC/envelope checks - // below still handle schema overhead and an unfittable single row. - rows_per_batch = estimated; - } - // Keep the remainder as one slice instead of allocating a pending entry - // per row in a large engine batch. - let has_remainder = batch.num_rows() > rows_per_batch; - let batch = if has_remainder { - pending.push(( - batch.slice(rows_per_batch, batch.num_rows() - rows_per_batch), - rows_per_batch, - )); - batch.slice(0, rows_per_batch) - } else { - batch - }; - let row_count = checked_arrow_row_count(batch.num_rows(), *offset)?; - // Arrow materialization/IPC encoding is CPU work. Retain operation - // ownership if cancellation drops the JoinHandle, so ReleaseAll cannot - // claim drain while a result batch is still being encoded. - let encoding = EncodingBatch { - batch, - _guard: guard.clone(), - }; - let (encoded, data) = tokio::task::spawn_blocking(move || encoding.encode()) - .await - .map_err(|_| Status::internal("Arrow result encoding task failed"))??; - let batch = &encoded.batch; - let payload = proto::execute_plan_response::ArrowBatch { - row_count, - data: data.into(), - start_offset: Some(*offset), - ..Default::default() - }; - let result = response( + let Some(chunk_bytes) = chunk_size else { + // As on the JVM, clients without chunking opt-in receive the original + // whole IPC batch. Their gRPC receive limit must accommodate it. + payload.data = data; + let message = response( + id, + &guard.session, + operation, + Some(proto::execute_plan_response::ResponseType::ArrowBatch( + payload, + )), + ); + *offset = next_offset; + return Ok(sender.send(Ok(message)).await.is_ok()); + }; + let chunks = data.len().div_ceil(chunk_bytes).max(1); + payload.num_chunks_in_batch = Some( + i64::try_from(chunks) + .map_err(|_| Status::resource_exhausted("Arrow chunk count exceeds i64"))?, + ); + for index in 0..chunks { + let from = index * chunk_bytes; + let to = from.saturating_add(chunk_bytes).min(data.len()); + // Bytes slices share the original IPC allocation, including compressed + // buffers. Clients concatenate the chunks before decoding Arrow. + payload.data = data.slice(from..to); + payload.chunk_index = Some(index as i64); + let message = response( id, &guard.session, operation, @@ -687,56 +529,11 @@ async fn send_arrow_batch( payload.clone(), )), ); - let result_bytes = result.encoded_len(); - if result_bytes > MAX_RESULT_BYTES { - if batch.num_rows() <= 1 { - return Err(Status::resource_exhausted( - "one Arrow row/schema exceeds the 3 MiB result limit", - )); - } - // Recheck every slice including the envelope: skew and schema - // overhead can exceed the initial memory estimate. - let rows_per_batch = ((batch.num_rows() as u128 * MAX_RESULT_BYTES as u128 * 9 - / (result_bytes as u128 * 10)) as usize) - .clamp(1, batch.num_rows() - 1); - drop(result); - if has_remainder { - if let Some((_, next_rows)) = pending.last_mut() { - // Teach this piece's own remainder about IPC padding or - // other fixed costs absent from the logical estimate. - *next_rows = (*next_rows).min(rows_per_batch); - } - } - pending.push((batch.clone(), rows_per_batch)); - continue; - } - let emitted_schema = StreamReader::try_new(&payload.data[..], None) - .map_err(|error| Status::internal(format!("invalid output IPC: {error}")))? - .schema(); - check_output_schema(emitted_schema.as_ref(), output_schema)?; - if has_remainder { - if let Some((_, next_rows)) = pending.last_mut() { - // Adapt this piece's own remainder, without changing an outer - // sibling when the piece finishes. The row-growth limit is a - // sampling heuristic, not a bound on bytes for skewed rows. - *next_rows = rows_for_bytes( - batch.num_rows(), - result_bytes.saturating_sub(fixed_bytes), - payload_bytes, - ) - // A successful size is already known to fit. Fixed buffer - // padding must not make that row budget decay on each send. - .max(batch.num_rows()) - .min(batch.num_rows().saturating_mul(4).max(1)); - } - } - *offset = offset - .checked_add(row_count) - .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; - if sender.send(Ok(result)).await.is_err() { + if sender.send(Ok(message)).await.is_err() { return Ok(false); } } + *offset = next_offset; Ok(true) } @@ -750,6 +547,7 @@ impl SparkConnectService for ConnectService { request: Request, ) -> Result, Status> { let req = request.into_inner(); + let chunk_size = arrow_chunk_size(&req.request_options, self.options.max_arrow_chunk_size); let operation = req .operation_id .unwrap_or_else(|| Uuid::new_v4().to_string()); @@ -843,6 +641,7 @@ impl SparkConnectService for ConnectService { &mut offset, &mut output_schema, &guard, + chunk_size, ) .await? { @@ -865,6 +664,7 @@ impl SparkConnectService for ConnectService { &mut offset, &mut output_schema, &guard, + chunk_size, ) .await? { @@ -1165,61 +965,12 @@ mod tests { ); } - #[test] - fn initial_row_budget_splits_large_batches_before_encoding() { - use arrow::array::Int64Array; - use arrow::datatypes::{DataType, Field, Schema}; - - let batch = RecordBatch::try_new( - Arc::new(Schema::new(vec![Field::new( - "value", - DataType::Int64, - false, - )])), - vec![Arc::new(Int64Array::from_iter_values(0..1_000_000))], - ) - .unwrap(); - let rows = estimated_rows_per_batch(&batch); - assert!(rows > 0 && rows < batch.num_rows()); - assert!(rows * std::mem::size_of::() <= MAX_RESULT_BYTES / 2); - assert_eq!( - estimated_rows_per_batch(&RecordBatch::new_empty(batch.schema())), - 1 - ); - } - - #[test] - fn view_estimates_count_repeated_and_null_slot_lengths() { - use arrow::array::{BinaryViewArray, StringViewArray}; - use arrow::buffer::NullBuffer; - - let source = StringViewArray::from(vec!["abcdefghijklmno"]); - let (views, buffers, _) = source.into_parts(); - let repeated = vec![views[0]; 4].into(); - let nulls = Some(NullBuffer::from(vec![true, false, true, false])); - let strings = StringViewArray::new(repeated, buffers, nulls); - // Four logical values reserve four payloads during materialization, - // even though they share one buffer and two slots are null. IPC also - // needs five 64-bit offsets and a validity byte. - let expected = 4 * 15 + 5 * 8 + 1; - assert_eq!(estimated_array_bytes(&strings), expected); - assert_eq!(strings.total_bytes_len(), 2 * 15); - let (views, buffers, nulls) = strings.into_parts(); - let binary = BinaryViewArray::new(views, buffers, nulls); - assert_eq!(estimated_array_bytes(&binary), expected); - assert_eq!( - estimated_array_bytes(&binary.slice(1, 2)), - 2 * 15 + 3 * 8 + 1 - ); - } - #[cfg(target_pointer_width = "64")] #[tokio::test] async fn unrepresentable_arrow_row_counts_fail_before_encoding() { use arrow::array::NullArray; use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatchOptions; - use std::sync::atomic::Ordering; let rows = i64::MAX as usize + 1; let empty = RecordBatch::try_new_with_options( @@ -1239,41 +990,24 @@ mod tests { let (sender, mut receiver) = mpsc::channel(2); let mut offset = 7; let mut output_schema = None; - for direct in [false, true] { - let error = if direct { - send_arrow_batch( - &sender, - "session", - "row_count", - batch.clone(), - rows, - &mut offset, - &mut output_schema, - &guard, - ) - .await - .unwrap_err() - } else { - send_batch( - &sender, - "session", - "row_count", - batch.clone().into(), - batch.schema().as_ref(), - &mut offset, - &mut output_schema, - &guard, - ) - .await - .unwrap_err() - }; - assert_eq!(error.code(), tonic::Code::ResourceExhausted); - assert_eq!(error.message(), "result batch row count exceeds i64"); - assert_eq!(offset, 7); - assert!(output_schema.is_none()); - assert_eq!(guard.running.encoding_attempts.load(Ordering::SeqCst), 0); - assert!(receiver.try_recv().is_err()); - } + let error = send_batch( + &sender, + "session", + "row_count", + batch.clone().into(), + batch.schema().as_ref(), + &mut offset, + &mut output_schema, + &guard, + Some(MAX_RESULT_BYTES), + ) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::ResourceExhausted); + assert_eq!(error.message(), "result batch row count exceeds i64"); + assert_eq!(offset, 7); + assert!(output_schema.is_none()); + assert!(receiver.try_recv().is_err()); } } @@ -1281,7 +1015,6 @@ mod tests { async fn overflowing_engine_batches_do_not_partially_emit() { use arrow::array::LargeStringArray; use arrow::datatypes::{DataType, Field, Schema}; - use std::sync::atomic::Ordering; let schema = Arc::new(Schema::new(vec![Field::new( "text", @@ -1314,10 +1047,10 @@ mod tests { &mut offset, &mut output_schema, &guard, + Some(MAX_RESULT_BYTES), ) .await .unwrap(); - let before = guard.running.encoding_attempts.load(Ordering::SeqCst); let batch = if ipc { let data = encode_batch(&oversized).unwrap(); assert!(data.len() > MAX_RESULT_BYTES); @@ -1334,16 +1067,13 @@ mod tests { &mut offset, &mut output_schema, &guard, + Some(MAX_RESULT_BYTES), ) .await .unwrap_err(); assert_eq!(error.code(), tonic::Code::ResourceExhausted); assert_eq!(error.message(), "result row offset exceeds i64"); assert_eq!(offset, i64::MAX - 1); - assert_eq!( - guard.running.encoding_attempts.load(Ordering::SeqCst), - before - ); let previous = receiver.try_recv().unwrap().unwrap(); let Some(proto::execute_plan_response::ResponseType::ArrowBatch(previous)) = previous.response_type @@ -1363,6 +1093,7 @@ mod tests { &mut offset, &mut output_schema, &guard, + Some(MAX_RESULT_BYTES), ) .await .unwrap(); @@ -1370,536 +1101,6 @@ mod tests { } } - #[tokio::test] - async fn successful_encoding_grows_a_conservative_row_budget() { - use arrow::array::Int64Array; - use arrow::datatypes::{DataType, Field, Schema}; - - let batch = RecordBatch::try_new( - Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), - vec![Arc::new(Int64Array::from_iter_values(0..1000))], - ) - .unwrap(); - let session = Arc::new(Session::new(Arc::new(RegistrySession))); - let (guard, _) = session.operation("adaptive").unwrap(); - let (sender, mut receiver) = mpsc::channel(1024); - let mut offset = 0; - let mut output_schema = None; - assert!(send_arrow_batch( - &sender, - "session", - "adaptive", - batch, - 1, - &mut offset, - &mut output_schema, - &guard, - ) - .await - .unwrap()); - let (mut batches, mut rows) = (0, 0); - while let Ok(result) = receiver.try_recv() { - let result = result.unwrap(); - assert!(result.encoded_len() <= MAX_RESULT_BYTES); - let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = - result.response_type - else { - panic!("expected Arrow response"); - }; - assert_eq!(batch.start_offset, Some(rows)); - batches += 1; - rows += batch.row_count; - } - assert_eq!(rows, 1000); - assert_eq!(offset, rows); - assert!( - batches <= 8, - "a conservative estimate must adapt, got {batches} batches" - ); - } - - #[tokio::test] - async fn fixed_schema_overhead_does_not_collapse_the_payload_budget() { - use arrow::array::{Int64Array, LargeStringArray}; - use arrow::datatypes::{DataType, Field, Schema}; - use std::collections::HashMap; - use std::sync::atomic::Ordering; - - for (metadata_bytes, expected_rows, max_responses) in [ - ((1.49 * 1024.0 * 1024.0) as usize, 1500usize, 18usize), - ((2.6 * 1024.0 * 1024.0) as usize, 300usize, 14usize), - ] { - let schema = Arc::new(Schema::new_with_metadata( - vec![ - Field::new("id", DataType::Int64, false), - Field::new("text", DataType::LargeUtf8, false), - ], - HashMap::from([("padding".into(), "x".repeat(metadata_bytes))]), - )); - let text = "z".repeat(8192); - let batch = RecordBatch::try_new( - schema, - vec![ - Arc::new(Int64Array::from_iter_values(0..expected_rows as i64)), - Arc::new(LargeStringArray::from_iter_values(std::iter::repeat_n( - text.as_str(), - expected_rows, - ))), - ], - ) - .unwrap(); - let budget = batch.num_rows().max(1); - let session = Arc::new(Session::new(Arc::new(RegistrySession))); - let (guard, _) = session.operation("fixed_overhead").unwrap(); - // Bound retained metadata even if the old one-row decay returns; - // the timeout then fails without buffering gigabytes of replies. - let (sender, mut receiver) = mpsc::channel(max_responses + 1); - let mut offset = 0; - let mut output_schema = None; - assert!(tokio::time::timeout( - std::time::Duration::from_secs(15), - send_arrow_batch( - &sender, - "session", - "fixed_overhead", - batch, - budget, - &mut offset, - &mut output_schema, - &guard - ) - ) - .await - .expect("fixed overhead exhausted the bounded response queue") - .unwrap()); - let (mut responses, mut rows) = (0usize, 0usize); - while let Ok(result) = receiver.try_recv() { - let result = result.unwrap(); - assert!(result.encoded_len() <= MAX_RESULT_BYTES); - let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = - result.response_type - else { - panic!("expected Arrow response"); - }; - assert_eq!(batch.start_offset, Some(rows as i64)); - let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); - assert_eq!(reader.schema().metadata()["padding"].len(), metadata_bytes); - let decoded = reader.next().unwrap().unwrap(); - assert_eq!(decoded.num_rows() as i64, batch.row_count); - assert!(reader.next().is_none()); - let ids = decoded - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let strings = decoded - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - for index in 0..decoded.num_rows() { - assert_eq!(ids.value(index), rows as i64); - assert_eq!(strings.value(index), text); - rows += 1; - } - responses += 1; - } - assert_eq!(rows, expected_rows); - assert_eq!(offset, expected_rows as i64); - assert!( - responses <= max_responses, - "{metadata_bytes} fixed bytes caused {responses} responses" - ); - // One encode measures the fresh empty schema. Only a small - // constant of exact-envelope retries is acceptable after that. - let attempts = guard.running.encoding_attempts.load(Ordering::SeqCst); - assert!( - attempts <= responses + 2, - "{attempts} encodes for {responses} responses" - ); - } - } - - #[tokio::test] - async fn wide_buffer_padding_does_not_decay_or_repeat_failed_budgets() { - use arrow::array::{Array, ArrayRef, Int64Array, Int8Array}; - use arrow::datatypes::{DataType, Field, Schema}; - use std::sync::atomic::Ordering; - - // Int64 values cross a 64-byte IPC padding boundary at 8 -> 9 rows; - // Int8 values occupy the same padded buffer for up to 64 rows. Nullable - // columns add a separately padded validity buffer absent from an empty - // schema measurement. Share inputs to keep this regression inexpensive. - for (columns, expected_rows, narrow, nullable, max_responses) in [ - (17_293, 64usize, false, false, 18usize), - (17_293, 200usize, true, false, 10usize), - (12_000, 64usize, false, true, 18usize), - ] { - let values: ArrayRef = if narrow { - Arc::new(Int8Array::from_iter_values( - (0..expected_rows).map(|row| (row % 127) as i8), - )) - } else if nullable { - Arc::new(Int64Array::from_iter( - (0..expected_rows).map(|row| (row % 3 != 0).then_some(row as i64)), - )) - } else { - Arc::new(Int64Array::from_iter_values(0..expected_rows as i64)) - }; - let fields = (0..columns) - .map(|column| { - Field::new( - format!("c{column:05}"), - if narrow { - DataType::Int8 - } else { - DataType::Int64 - }, - nullable, - ) - }) - .collect::>(); - let batch = - RecordBatch::try_new(Arc::new(Schema::new(fields)), vec![values; columns]).unwrap(); - let budget = batch.num_rows().max(1); - let session = Arc::new(Session::new(Arc::new(RegistrySession))); - let (guard, _) = session.operation("wide_padding").unwrap(); - let (sender, mut receiver) = mpsc::channel(max_responses + 1); - let mut offset = 0; - let mut output_schema = None; - assert!(tokio::time::timeout( - std::time::Duration::from_secs(30), - send_arrow_batch( - &sender, - "session", - "wide_padding", - batch, - budget, - &mut offset, - &mut output_schema, - &guard - ) - ) - .await - .expect("wide padding exhausted the bounded response queue") - .unwrap()); - let (mut responses, mut rows) = (0usize, 0usize); - while let Ok(result) = receiver.try_recv() { - let result = result.unwrap(); - assert!(result.encoded_len() <= MAX_RESULT_BYTES); - let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = - result.response_type - else { - panic!("expected Arrow response"); - }; - assert_eq!(batch.start_offset, Some(rows as i64)); - let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); - let decoded = reader.next().unwrap().unwrap(); - assert_eq!(decoded.num_columns(), columns); - assert_eq!(decoded.num_rows() as i64, batch.row_count); - assert!(reader.next().is_none()); - for column in decoded.columns() { - if narrow { - let values = column.as_any().downcast_ref::().unwrap(); - for index in 0..decoded.num_rows() { - assert_eq!(values.value(index), ((rows + index) % 127) as i8); - } - } else { - let values = column.as_any().downcast_ref::().unwrap(); - for index in 0..decoded.num_rows() { - let row = rows + index; - assert_eq!(values.is_null(index), nullable && row % 3 == 0); - if !values.is_null(index) { - assert_eq!(values.value(index), row as i64); - } - } - } - } - rows += decoded.num_rows(); - responses += 1; - } - assert_eq!(rows, expected_rows); - assert_eq!(offset, expected_rows as i64); - assert!(responses <= max_responses, - "{columns} columns (narrow={narrow}, nullable={nullable}) caused {responses} responses"); - // One measurement plus a small number of exact retries, including - // a nullable slice that crosses the 8 -> 9 row padding boundary. - let attempts = guard.running.encoding_attempts.load(Ordering::SeqCst); - assert!(attempts <= responses + 3, - "{columns} columns (narrow={narrow}, nullable={nullable}): {attempts} encodes for {responses} responses"); - } - } - - #[tokio::test] - async fn skewed_output_checks_prefixes_and_recovers_light_suffixes() { - use arrow::array::{Int64Array, LargeStringArray}; - use arrow::datatypes::{DataType, Field, Schema}; - use std::sync::atomic::Ordering; - - for shape in ["large_tail", "large_head", "dense_middle"] { - let (prefix_rows, suffix_rows) = match shape { - "large_tail" => (5000, 0), - "large_head" => (0, 5000), - _ => (100_000, 100_000), - }; - let mut sizes = vec![0usize; prefix_rows]; - sizes.extend(std::iter::repeat_n(1024 * 1024, 40)); - sizes.extend(std::iter::repeat_n(0, suffix_rows)); - let batch = RecordBatch::try_new( - Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new("text", DataType::LargeUtf8, false), - ])), - vec![ - Arc::new(Int64Array::from_iter_values(0..sizes.len() as i64)), - Arc::new(LargeStringArray::from_iter_values( - sizes.iter().map(|size| "z".repeat(*size)), - )), - ], - ) - .unwrap(); - let budget = batch.num_rows().max(1); - let session = Arc::new(Session::new(Arc::new(RegistrySession))); - let (guard, _) = session.operation("skewed").unwrap(); - // Even an accidentally permanent one-row budget must not block - // this regression before its response-count assertion can fail. - let (sender, mut receiver) = mpsc::channel(sizes.len()); - let mut offset = 0; - let mut output_schema = None; - assert!(tokio::time::timeout( - std::time::Duration::from_secs(30), - send_arrow_batch( - &sender, - "session", - "skewed", - batch, - budget, - &mut offset, - &mut output_schema, - &guard, - ) - ) - .await - .unwrap() - .unwrap()); - let (mut responses, mut empty_responses, mut rows) = (0usize, 0usize, 0usize); - while let Ok(result) = receiver.try_recv() { - let result = result.unwrap(); - assert!(result.encoded_len() <= MAX_RESULT_BYTES); - let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = - result.response_type - else { - panic!("expected Arrow response"); - }; - assert_eq!(batch.start_offset, Some(rows as i64)); - let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); - let decoded = reader.next().unwrap().unwrap(); - assert_eq!(decoded.num_rows() as i64, batch.row_count); - assert!(reader.next().is_none()); - let ids = decoded - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let strings = decoded - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let mut empty = true; - for index in 0..decoded.num_rows() { - assert_eq!(ids.value(index), rows as i64); - let value = strings.value(index); - assert_eq!(value.len(), sizes[rows]); - assert!(value.bytes().all(|byte| byte == b'z')); - empty &= value.is_empty(); - rows += 1; - } - responses += 1; - empty_responses += usize::from(empty); - } - assert_eq!(rows, sizes.len()); - assert_eq!(offset, rows as i64); - let attempts = guard.running.encoding_attempts.load(Ordering::SeqCst); - // Logical prefix checks must avoid repeatedly materializing the - // same unconsumed dense region after successes on empty prefixes. - assert!(attempts <= responses + 4, - "{shape}: {attempts} encodes for {responses} responses repeatedly encoded an oversized region"); - if shape == "dense_middle" { - // A sticky reduced row budget also fragments a light suffix - // inside a failed piece. Let it recover once the dense rows - // are consumed, as well as across outer sibling slices. - assert!( - responses < 80, - "dense-middle output fragmented into {responses} responses" - ); - assert!( - empty_responses <= 35, - "dense-middle output emitted {empty_responses} all-empty responses" - ); - } else if shape == "large_head" { - assert!( - responses < 100, - "large-head output fragmented into {responses} responses" - ); - } else { - assert!( - responses < 80, - "large-tail output fragmented into {responses} responses" - ); - } - } - } - - #[tokio::test] - async fn existing_ipc_is_forwarded_unchanged() { - use arrow::array::Int64Array; - use arrow::datatypes::{DataType, Field, Schema}; - use arrow::ipc::writer::IpcWriteOptions; - - let batch = RecordBatch::try_new( - Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), - vec![Arc::new(Int64Array::from_iter_values(0..10))], - ) - .unwrap(); - // Legacy IPC framing differs from the frontend encoder's default, so - // byte equality also proves that the payload was not re-encoded. - let options = IpcWriteOptions::try_new(8, true, arrow::ipc::MetadataVersion::V4).unwrap(); - let mut data = Vec::new(); - let mut writer = - StreamWriter::try_new_with_options(&mut data, batch.schema().as_ref(), options) - .unwrap(); - writer.write(&batch).unwrap(); - writer.finish().unwrap(); - drop(writer); - assert_ne!(data, encode_batch(&batch).unwrap()); - let session = Arc::new(Session::new(Arc::new(RegistrySession))); - let (guard, _) = session.operation("ipc").unwrap(); - let (sender, mut receiver) = mpsc::channel(1); - let mut offset = 7; - let mut output_schema = None; - assert!(send_batch( - &sender, - "session", - "ipc", - ResultBatch::Ipc { - data: data.clone(), - row_count: 10 - }, - batch.schema().as_ref(), - &mut offset, - &mut output_schema, - &guard, - ) - .await - .unwrap()); - let result = receiver.recv().await.unwrap().unwrap(); - let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = - result.response_type - else { - panic!("expected Arrow response"); - }; - assert_eq!(&batch.data[..], data.as_slice()); - assert_eq!(batch.row_count, 10); - assert_eq!(batch.start_offset, Some(7)); - assert_eq!(offset, 17); - } - - #[tokio::test] - async fn oversized_ipc_is_split_by_rows_and_checks_its_declared_count() { - use arrow::array::Int64Array; - use arrow::datatypes::{DataType, Field, Schema}; - - let batch = RecordBatch::try_new( - Arc::new(Schema::new( - (0..10) - .map(|column| Field::new(format!("id{column}"), DataType::Int64, false)) - .collect::>(), - )), - (0..10) - .map(|column| { - Arc::new(Int64Array::from_iter_values( - (0..50_000).map(|row| row + column * 50_000), - )) as arrow::array::ArrayRef - }) - .collect(), - ) - .unwrap(); - let data = encode_batch(&batch).unwrap(); - assert!(data.len() > MAX_RESULT_BYTES); - let expected_batches = data.len().div_ceil(MAX_RESULT_BYTES / 2); - let session = Arc::new(Session::new(Arc::new(RegistrySession))); - let (guard, _) = session.operation("ipc").unwrap(); - let (sender, mut receiver) = mpsc::channel(64); - let mut offset = 0; - let mut output_schema = None; - assert!(send_batch( - &sender, - "session", - "ipc", - ResultBatch::Ipc { - data: data.clone(), - row_count: 50_000 - }, - batch.schema().as_ref(), - &mut offset, - &mut output_schema, - &guard, - ) - .await - .unwrap()); - let (mut rows, mut batches) = (0, 0); - while let Ok(result) = receiver.try_recv() { - let result = result.unwrap(); - assert!(result.encoded_len() <= MAX_RESULT_BYTES); - let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = - result.response_type - else { - panic!("expected Arrow response"); - }; - assert_eq!(batch.start_offset, Some(rows)); - let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); - let decoded = reader.next().unwrap().unwrap(); - assert_eq!(decoded.num_columns(), 10); - assert_eq!(decoded.num_rows() as i64, batch.row_count); - for (column, values) in decoded.columns().iter().enumerate() { - let ids = values.as_any().downcast_ref::().unwrap(); - let start = rows + column as i64 * 50_000; - assert!(ids - .values() - .iter() - .copied() - .eq(start..start + batch.row_count)); - } - rows += batch.row_count; - batches += 1; - assert!(reader.next().is_none()); - } - assert_eq!(rows, 50_000); - assert_eq!(offset, rows); - assert!((expected_batches..=expected_batches + 1).contains(&batches)); - for row_count in [-1, 49_999] { - let error = send_batch( - &sender, - "session", - "ipc", - ResultBatch::Ipc { - data: data.clone(), - row_count, - }, - batch.schema().as_ref(), - &mut offset, - &mut output_schema, - &guard, - ) - .await - .unwrap_err(); - assert_eq!(error.code(), tonic::Code::Internal); - assert_eq!(offset, rows); - } - assert!(receiver.try_recv().is_err()); - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cancelled_encoding_wait_retains_operation_until_blocking_task_finishes() { let session = Arc::new(Session::new(Arc::new(RegistrySession))); @@ -1944,7 +1145,8 @@ mod tests { #[tokio::test] async fn released_session_retention_is_bounded_without_evicting_active_sessions() { - let service = ConnectService::new(Arc::new(RegistryEngine)); + let service = + ConnectService::new(Arc::new(RegistryEngine), crate::ServerOptions::default()); let active_id = Uuid::new_v4().to_string(); let active = service.session(&active_id, None, None).unwrap(); @@ -2002,7 +1204,8 @@ mod tests { #[tokio::test] async fn reconnecting_release_of_live_session_allows_a_new_backend() { - let service = ConnectService::new(Arc::new(RegistryEngine)); + let service = + ConnectService::new(Arc::new(RegistryEngine), crate::ServerOptions::default()); let id = Uuid::new_v4().to_string(); let previous = service.session(&id, None, None).unwrap(); release(&service, &id, true).await; diff --git a/crates/spark-connect-server/tests/engine_injection.rs b/crates/spark-connect-server/tests/engine_injection.rs index a5797ed9..661d0450 100644 --- a/crates/spark-connect-server/tests/engine_injection.rs +++ b/crates/spark-connect-server/tests/engine_injection.rs @@ -17,12 +17,15 @@ //! Real gRPC tests of the injected backend boundary. The fake is independent of Spark execution. +#[path = "support/arrow.rs"] +mod arrow_support; #[path = "support/requests.rs"] mod requests; #[path = "support/server.rs"] mod server; use std::collections::HashMap; +use std::future::Future; use std::io::Cursor; use std::pin::Pin; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -35,6 +38,7 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::ipc::reader::StreamReader; use arrow::ipc::writer::StreamWriter; use arrow::record_batch::RecordBatch; +use arrow_support::chunking; use futures::{Stream, StreamExt}; use prost::Message; use proto::execute_plan_response::ResponseType; @@ -61,6 +65,7 @@ const PANICKING_STREAM: i64 = 7_000_000; const PANICKING_STREAM_DROP: i64 = 8_000_000; const MISMATCHED_IPC_BATCH: i64 = 9_000_000; const INVALID_IPC_BATCH: i64 = 10_000_000; +const CHUNKED_RESULT: i64 = 11_000_000; #[derive(Default)] struct Spy { @@ -80,8 +85,35 @@ struct Spy { metadata_drop_observations: Mutex>, slow_drop: Mutex)>>, stream_drop_gate: Mutex>>, + request_panic: Mutex>, } +#[derive(Clone, Copy, PartialEq, Eq)] +enum Call { + Prepare, + Analyze, + Config, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum PanicAt { + Construction, + Poll, +} + +impl Spy { + fn panic_once(&self, call: Call) -> Option { + let mut next = self.request_panic.lock().unwrap(); + if next.as_ref().is_some_and(|(target, _)| *target == call) { + next.take().map(|(_, at)| at) + } else { + None + } + } +} + +type EngineFuture<'a, T> = Pin> + Send + 'a>>; + #[derive(Default)] struct DropGate { released: Mutex, @@ -206,172 +238,208 @@ impl Drop for BlockedMetadata { } } -#[tonic::async_trait] +// Explicit async-trait signatures also exercise a panic before a future is returned. impl EngineSession for FakeSession { - async fn prepare(&self, op: proto::plan::OpType) -> Result { - self.spy.prepares.fetch_add(1, Ordering::SeqCst); - let (value, live, batch_mode) = match op { - proto::plan::OpType::Root(relation) => match relation.rel_type { - Some(proto::relation::RelType::Range(range)) => { - if range.end == BLOCKED_PREPARE { - let _dropped = BlockedPrepare(self.spy.clone()); - std::future::pending::<()>().await; - } - if range.end == -1 { - return Err(Status::invalid_argument("fake rejected the plan")); + fn prepare<'life0, 'async_trait>( + &'life0 self, + op: proto::plan::OpType, + ) -> EngineFuture<'async_trait, PreparedOperation> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + let panic = self.spy.panic_once(Call::Prepare); + assert!(panic != Some(PanicAt::Construction), "prepare construction"); + Box::pin(async move { + assert!(panic != Some(PanicAt::Poll), "prepare poll"); + self.spy.prepares.fetch_add(1, Ordering::SeqCst); + let (value, live, batch_mode) = match op { + proto::plan::OpType::Root(relation) => match relation.rel_type { + Some(proto::relation::RelType::Range(range)) => { + if range.end == BLOCKED_PREPARE { + let _dropped = BlockedPrepare(self.spy.clone()); + std::future::pending::<()>().await; + } + if range.end == -1 { + return Err(Status::invalid_argument("fake rejected the plan")); + } + // Deliberately unlike range semantics: this proves the injected + // backend, rather than the default planner, executed the request. + let batch_mode = match range.end { + MISMATCHED_BATCH => BatchMode::WrongType, + MISMATCHED_IPC_BATCH => BatchMode::WrongTypeIpc, + INVALID_IPC_BATCH => BatchMode::InvalidIpc, + ANNOTATED_BATCH => BatchMode::DifferentAnnotations, + FLOODED_RESULT => BatchMode::Flooded, + CHUNKED_RESULT => BatchMode::Chunked, + PANICKING_EXECUTION => BatchMode::PanicExecute, + PANICKING_STREAM => BatchMode::PanicStream, + PANICKING_STREAM_DROP => BatchMode::PanicDrop, + _ => BatchMode::Declared, + }; + (999, range.end == LIVE, batch_mode) } - // Deliberately unlike range semantics: this proves the injected - // backend, rather than the default planner, executed the request. - let batch_mode = match range.end { - MISMATCHED_BATCH => BatchMode::WrongType, - MISMATCHED_IPC_BATCH => BatchMode::WrongTypeIpc, - INVALID_IPC_BATCH => BatchMode::InvalidIpc, - ANNOTATED_BATCH => BatchMode::DifferentAnnotations, - FLOODED_RESULT => BatchMode::Flooded, - PANICKING_EXECUTION => BatchMode::PanicExecute, - PANICKING_STREAM => BatchMode::PanicStream, - PANICKING_STREAM_DROP => BatchMode::PanicDrop, - _ => BatchMode::Declared, - }; - (999, range.end == LIVE, batch_mode) - } - Some(proto::relation::RelType::Read(read)) => { - let Some(proto::read::ReadType::NamedTable(table)) = read.read_type else { - return Err(Status::unimplemented("fake read")); - }; - let value = self - .views - .lock() - .unwrap() - .get(&table.unparsed_identifier) - .copied() - .ok_or_else(|| Status::not_found("fake view not found"))?; - (value, false, BatchMode::Declared) - } - _ => return Err(Status::unimplemented("fake relation")), - }, - proto::plan::OpType::Command(command) => { - self.spy.commands.fetch_add(1, Ordering::SeqCst); - match command.command_type { - Some(proto::command::CommandType::CreateDataframeView(view)) => { - let Some(proto::relation::RelType::Range(input)) = - view.input.and_then(|input| input.rel_type) - else { - return Err(Status::invalid_argument("fake view requires range input")); + Some(proto::relation::RelType::Read(read)) => { + let Some(proto::read::ReadType::NamedTable(table)) = read.read_type else { + return Err(Status::unimplemented("fake read")); }; - let mut views = self.views.lock().unwrap(); - if !view.replace && views.contains_key(&view.name) { - return Err(Status::already_exists("fake view already exists")); - } - views.insert(view.name, input.end); - return Ok(PreparedOperation::Command(Vec::new())); + let value = self + .views + .lock() + .unwrap() + .get(&table.unparsed_identifier) + .copied() + .ok_or_else(|| Status::not_found("fake view not found"))?; + (value, false, BatchMode::Declared) } - Some(proto::command::CommandType::SqlCommand(command)) => { - return Ok(PreparedOperation::Command(vec![ - ResponseType::SqlCommandResult( - proto::execute_plan_response::SqlCommandResult { - relation: command.input, - }, - ), - ])); + _ => return Err(Status::unimplemented("fake relation")), + }, + proto::plan::OpType::Command(command) => { + self.spy.commands.fetch_add(1, Ordering::SeqCst); + match command.command_type { + Some(proto::command::CommandType::CreateDataframeView(view)) => { + let Some(proto::relation::RelType::Range(input)) = + view.input.and_then(|input| input.rel_type) + else { + return Err(Status::invalid_argument( + "fake view requires range input", + )); + }; + let mut views = self.views.lock().unwrap(); + if !view.replace && views.contains_key(&view.name) { + return Err(Status::already_exists("fake view already exists")); + } + views.insert(view.name, input.end); + return Ok(PreparedOperation::Command(Vec::new())); + } + Some(proto::command::CommandType::SqlCommand(command)) => { + return Ok(PreparedOperation::Command(vec![ + ResponseType::SqlCommandResult( + proto::execute_plan_response::SqlCommandResult { + relation: command.input, + }, + ), + ])); + } + _ => return Err(Status::unimplemented("fake command")), } - _ => return Err(Status::unimplemented("fake command")), } - } - _ => return Err(Status::unimplemented("fake operation")), - }; - Ok(PreparedOperation::Query(Box::new(FakeQuery { - spy: self.spy.clone(), - value, - live, - batch_mode, - }))) + _ => return Err(Status::unimplemented("fake operation")), + }; + Ok(PreparedOperation::Query(Box::new(FakeQuery { + spy: self.spy.clone(), + value, + live, + batch_mode, + }))) + }) } - async fn analyze( - &self, + fn analyze<'life0, 'async_trait>( + &'life0 self, input: proto::analyze_plan_request::Analyze, - ) -> Result { - self.spy.analyses.fetch_add(1, Ordering::SeqCst); - match input { - proto::analyze_plan_request::Analyze::Schema(_) => { - Ok(proto::analyze_plan_response::Result::Schema( - proto::analyze_plan_response::Schema { - schema: Some(proto::DataType { - kind: Some(proto::data_type::Kind::Struct(proto::data_type::Struct { - fields: vec![proto::data_type::StructField { - name: "x".repeat(3 * 1024 * 1024 + 1), - data_type: Some(proto::DataType { - kind: Some(proto::data_type::Kind::Long( - proto::data_type::Long::default(), - )), - }), - nullable: false, - metadata: None, - }], - ..Default::default() - })), - }), - }, - )) + ) -> EngineFuture<'async_trait, proto::analyze_plan_response::Result> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + let panic = self.spy.panic_once(Call::Analyze); + assert!(panic != Some(PanicAt::Construction), "analyze construction"); + Box::pin(async move { + assert!(panic != Some(PanicAt::Poll), "analyze poll"); + self.spy.analyses.fetch_add(1, Ordering::SeqCst); + match input { + proto::analyze_plan_request::Analyze::Schema(_) => { + Ok(proto::analyze_plan_response::Result::Schema( + proto::analyze_plan_response::Schema { + schema: Some(proto::DataType { + kind: Some(proto::data_type::Kind::Struct( + proto::data_type::Struct { + fields: vec![proto::data_type::StructField { + name: "x".repeat(3 * 1024 * 1024 + 1), + data_type: Some(proto::DataType { + kind: Some(proto::data_type::Kind::Long( + proto::data_type::Long::default(), + )), + }), + nullable: false, + metadata: None, + }], + ..Default::default() + }, + )), + }), + }, + )) + } + proto::analyze_plan_request::Analyze::IsStreaming(_) => { + let _dropped = BlockedMetadata { + spy: self.spy.clone(), + call: MetadataCall::Analysis, + }; + std::future::pending().await + } + proto::analyze_plan_request::Analyze::IsLocal(_) => { + Ok(proto::analyze_plan_response::Result::IsLocal( + proto::analyze_plan_response::IsLocal { is_local: true }, + )) + } + _ => Err(Status::unimplemented("fake analysis")), } - proto::analyze_plan_request::Analyze::IsStreaming(_) => { + }) + } + + fn config<'life0, 'async_trait>( + &'life0 self, + input: proto::config_request::operation::OpType, + ) -> EngineFuture<'async_trait, ConfigResult> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + let panic = self.spy.panic_once(Call::Config); + assert!(panic != Some(PanicAt::Construction), "config construction"); + Box::pin(async move { + assert!(panic != Some(PanicAt::Poll), "config poll"); + use proto::config_request::operation::OpType; + self.spy.configs.fetch_add(1, Ordering::SeqCst); + if matches!(&input, OpType::Get(get) if get.keys.iter().any(|key| key == "fake.block")) + { let _dropped = BlockedMetadata { spy: self.spy.clone(), - call: MetadataCall::Analysis, + call: MetadataCall::Config, }; - std::future::pending().await - } - proto::analyze_plan_request::Analyze::IsLocal(_) => { - Ok(proto::analyze_plan_response::Result::IsLocal( - proto::analyze_plan_response::IsLocal { is_local: true }, - )) + return std::future::pending().await; } - _ => Err(Status::unimplemented("fake analysis")), - } - } - - async fn config( - &self, - input: proto::config_request::operation::OpType, - ) -> Result { - use proto::config_request::operation::OpType; - self.spy.configs.fetch_add(1, Ordering::SeqCst); - if matches!(&input, OpType::Get(get) if get.keys.iter().any(|key| key == "fake.block")) { - let _dropped = BlockedMetadata { - spy: self.spy.clone(), - call: MetadataCall::Config, - }; - return std::future::pending().await; - } - let mut configs = self.configs.lock().unwrap(); - match input { - OpType::Set(set) => { - for pair in set.pairs { - configs.insert( - pair.key, - pair.value - .ok_or_else(|| Status::invalid_argument("fake config value"))?, - ); + let mut configs = self.configs.lock().unwrap(); + match input { + OpType::Set(set) => { + for pair in set.pairs { + configs.insert( + pair.key, + pair.value + .ok_or_else(|| Status::invalid_argument("fake config value"))?, + ); + } + Ok(ConfigResult { + pairs: Vec::new(), + warnings: vec!["handled by fake backend".into()], + }) } - Ok(ConfigResult { - pairs: Vec::new(), - warnings: vec!["handled by fake backend".into()], - }) + OpType::GetOption(get) => Ok(ConfigResult { + pairs: get + .keys + .into_iter() + .map(|key| { + let value = configs.get(&key).cloned(); + proto::KeyValue { key, value } + }) + .collect(), + warnings: Vec::new(), + }), + _ => Err(Status::unimplemented("fake config operation")), } - OpType::GetOption(get) => Ok(ConfigResult { - pairs: get - .keys - .into_iter() - .map(|key| { - let value = configs.get(&key).cloned(); - proto::KeyValue { key, value } - }) - .collect(), - warnings: Vec::new(), - }), - _ => Err(Status::unimplemented("fake config operation")), - } + }) } } @@ -389,6 +457,7 @@ enum BatchMode { InvalidIpc, DifferentAnnotations, Flooded, + Chunked, PanicExecute, PanicStream, PanicDrop, @@ -438,6 +507,10 @@ impl PreparedQuery for FakeQuery { )]))])), vec![Arc::new(Int64Array::from(vec![self.value]))], ), + BatchMode::Chunked => RecordBatch::try_new( + self.schema(), + vec![Arc::new(Int64Array::from(vec![self.value; 512 * 1024]))], + ), BatchMode::Flooded => RecordBatch::try_new( self.schema(), vec![Arc::new(Int64Array::from(vec![self.value; 64 * 1024]))], @@ -1535,3 +1608,211 @@ async fn release_all_timeout_keeps_cancellation_and_operation_ownership_until_dr assert_cancelled(&mut recovered).await; server.stop().await; } + +fn identified_request(session: &str, operation: &str) -> proto::ExecutePlanRequest { + let mut request = request(session, 1); + request.operation_id = Some(operation.into()); + request +} + +fn local_analysis(session: &str) -> proto::AnalyzePlanRequest { + proto::AnalyzePlanRequest { + session_id: session.into(), + user_context: requests::user("alice"), + analyze: Some(proto::analyze_plan_request::Analyze::IsLocal( + Default::default(), + )), + ..Default::default() + } +} + +async fn collect_identified(server: &mut Server, session: &str, operation: &str) { + let mut stream = server + .client + .execute_plan(identified_request(session, operation)) + .await + .unwrap() + .into_inner(); + let output = collect(&mut stream).await; + assert!(matches!( + output.last().unwrap().response_type, + Some(ResponseType::ResultComplete(_)) + )); +} + +#[tokio::test] +async fn noncanonical_ids_are_rejected_without_creating_or_releasing_state() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let uuid = Uuid::parse_str("abcdef12-3456-4789-abcd-0123456789ab").unwrap(); + let session = uuid.to_string(); + let operation = Uuid::new_v4().to_string(); + let original = server + .client + .config(get_config(&session)) + .await + .unwrap() + .into_inner() + .server_side_session_id; + for alias in [ + uuid.simple().to_string(), + uuid.braced().to_string(), + uuid.urn().to_string(), + session.to_uppercase(), + ] { + assert_eq!( + server + .client + .config(get_config(&alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .analyze_plan(local_analysis(&alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .execute_plan(identified_request(&alias, &operation)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_execute(release(&alias, &operation)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_session(release_session(&alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .execute_plan(identified_request(&Uuid::new_v4().to_string(), &alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_execute(release(&session, &alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .config(get_config(&session)) + .await + .unwrap() + .into_inner() + .server_side_session_id, + original + ); + } + assert_eq!(spy.factory_calls.lock().unwrap().len(), 1); + assert_eq!(spy.prepares.load(Ordering::SeqCst), 0); + collect_identified(&mut server, &session, &operation).await; + server.stop().await; +} + +#[tokio::test] +async fn backend_request_panics_have_internal_status_and_leave_the_session_usable() { + for call in [Call::Prepare, Call::Analyze, Call::Config] { + for at in [PanicAt::Construction, PanicAt::Poll] { + let spy = Arc::new(Spy::default()); + *spy.request_panic.lock().unwrap() = Some((call, at)); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let operation = Uuid::new_v4().to_string(); + let status = match call { + Call::Prepare => server + .client + .execute_plan(identified_request(&session, &operation)) + .await + .unwrap_err(), + Call::Analyze => server + .client + .analyze_plan(local_analysis(&session)) + .await + .unwrap_err(), + Call::Config => server + .client + .config(get_config(&session)) + .await + .unwrap_err(), + }; + assert_eq!(status.code(), Code::Internal); + assert_eq!(status.message(), "engine request panicked"); + server.client.config(get_config(&session)).await.unwrap(); + server + .client + .analyze_plan(local_analysis(&session)) + .await + .unwrap(); + // Reuse the operation ID after a preparation panic: its reservation + // must have been released before the failed RPC returned. + collect_identified(&mut server, &session, &operation).await; + assert_eq!(spy.factory_calls.lock().unwrap().len(), 1); + server.stop().await; + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn release_cancels_an_arrow_batch_between_byte_chunks() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let mut request = request(&session, CHUNKED_RESULT); + let operation = request.operation_id.clone().unwrap(); + request.request_options = vec![chunking(true, Some(1024))]; + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + let first = stream.message().await.unwrap().unwrap(); + let Some(ResponseType::ArrowBatch(chunk)) = first.response_type else { + panic!("first Arrow chunk") + }; + assert_eq!(chunk.chunk_index, Some(0)); + assert!(chunk.num_chunks_in_batch.unwrap() > 4000); + assert_eq!(chunk.row_count, 512 * 1024); + assert_eq!(chunk.start_offset, Some(0)); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_cancelled(&mut stream).await; + wait_for_count(&spy.streams_dropped, 1).await; + assert_eq!(spy.executions.load(Ordering::SeqCst), 1); + server.stop().await; +} diff --git a/crates/spark-connect-server/tests/protocol.rs b/crates/spark-connect-server/tests/protocol.rs index 38bad66f..14c637d2 100644 --- a/crates/spark-connect-server/tests/protocol.rs +++ b/crates/spark-connect-server/tests/protocol.rs @@ -17,6 +17,8 @@ //! Verify observable Spark Connect wire behavior independently of DataFrame conveniences. +#[path = "support/arrow.rs"] +mod arrow_support; #[path = "support/requests.rs"] mod requests; #[path = "support/server.rs"] @@ -26,16 +28,25 @@ mod support; use std::collections::HashSet; use std::io::Cursor; use std::sync::Arc; +use std::time::Duration; -use arrow::array::Int64Array; +use arrow::array::{ + Array, ArrayRef, BinaryViewArray, Int64Array, LargeBinaryArray, LargeListArray, + LargeStringArray, ListArray, MapArray, StringViewArray, StructArray, +}; +use arrow::buffer::OffsetBuffer; +use arrow::datatypes::{DataType, Field, Schema}; use arrow::ipc::reader::StreamReader; +use arrow::ipc::CompressionType; +use arrow::record_batch::RecordBatch; use prost::Message; use proto::execute_plan_response::ResponseType; use spark_connect_proto as proto; +use arrow_support::{assembled_batches, chunking, encoded, DEFAULT_ARROW_CHUNK_SIZE}; use requests::request as range; use server::Server; -use support::FixtureEngine; +use support::{Batch, FixtureEngine}; use tonic::Code; use uuid::Uuid; @@ -170,109 +181,168 @@ async fn observed_session_mismatch_and_released_sessions_are_rejected() { server.stop().await; } +async fn collect_result( + server: &mut Server, + request: proto::ExecutePlanRequest, +) -> Vec { + tokio::time::timeout(Duration::from_secs(30), async { + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + let mut responses = Vec::new(); + while let Some(response) = stream.message().await.unwrap() { + responses.push(response); + } + assert!(matches!( + responses.last().unwrap().response_type, + Some(ResponseType::ResultComplete(_)) + )); + responses + }) + .await + .expect("fixture execution must terminate") +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn arrow_payloads_are_byte_limited_and_oversized_rows_fail() { - use arrow::array::{LargeStringArray, StringArray, StringViewArray}; - use arrow::datatypes::{DataType, Field, Schema}; - use arrow::record_batch::RecordBatch; +async fn negotiated_chunks_preserve_plain_and_compressed_ipc_bytes_and_offsets() { + // Incompressible values keep both compression codecs above the chunk limit. + let mut random = 1u64; + let values = Int64Array::from_iter_values((0..450_000).map(|_| { + random ^= random << 13; + random ^= random >> 7; + random ^= random << 17; + random as i64 + })); + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(values)]).unwrap(); + for compression in [ + None, + Some(CompressionType::ZSTD), + Some(CompressionType::LZ4_FRAME), + ] { + let data = encoded(&batch, compression); + assert!(data.len() > 3 * 1024 * 1024); + let tail = batch.slice(0, 3); + let tail_data = encoded(&tail, compression); + let engine = FixtureEngine::default().with_results( + 42, + schema.clone(), + vec![ + Batch::Ipc(data.clone(), batch.num_rows() as i64), + Batch::Ipc(tail_data.clone(), 3), + ], + ); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + server.client = server + .client + .clone() + .max_decoding_message_size(8 * 1024 * 1024); + for option in [ + None, + Some(chunking(false, Some(1024))), + Some(chunking(true, None)), + Some(chunking(true, Some(3 * 1024 * 1024))), + ] { + let negotiated = option.as_ref().is_some_and(|option| matches!( + &option.request_option, + Some(proto::execute_plan_request::request_option::RequestOption::ResultChunkingOptions(options)) + if options.allow_arrow_batch_chunking + )); + let mut request = range(&Uuid::new_v4().to_string(), 42); + let limited = option.as_ref().is_some_and(|option| matches!( + &option.request_option, + Some(proto::execute_plan_request::request_option::RequestOption::ResultChunkingOptions(options)) + if options.allow_arrow_batch_chunking && options.preferred_arrow_chunk_size.is_some() + )); + request.request_options = option.into_iter().collect(); + let responses = collect_result(&mut server, request).await; + let output = assembled_batches(&responses, negotiated); + assert_eq!(output.len(), 2); + assert_eq!(output[0].data.as_ref(), data.as_slice()); + assert_eq!(output[1].data.as_ref(), tail_data.as_slice()); + assert_eq!(output[0].row_count, 450_000); + assert_eq!(output[1].start_offset, Some(450_000)); + for (result, expected) in output.iter().zip([&batch, &tail]) { + let mut reader = StreamReader::try_new(result.data.as_ref(), None).unwrap(); + assert_eq!(reader.next().unwrap().unwrap(), *expected); + assert!(reader.next().is_none()); + } + let chunks = responses + .iter() + .filter(|response| { + matches!(response.response_type, Some(ResponseType::ArrowBatch(_))) + }) + .count(); + assert_eq!(chunks, if limited { 3 } else { 2 }); + } + server.stop().await; + } +} - let session = Uuid::new_v4().to_string(); - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new("s", DataType::Utf8, false), - ])); - // Skewed string sizes make initial byte-ratio slices too large, forcing - // refinement while preserving every value and row offset. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn preferred_chunk_size_uses_the_protocol_range_and_splits_inside_a_large_row() { + use arrow::array::StringArray; let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int64Array::from_iter_values(0..8192)), - Arc::new(StringArray::from_iter_values( - (0..8192).map(|id| "x".repeat(if id % 64 == 0 { 200000 } else { 1000 })), - )), - ], + Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Utf8, + false, + )])), + vec![Arc::new(StringArray::from(vec![ + "x".repeat(4 * 1024 * 1024) + ]))], ) .unwrap(); - let oversized_schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)])); - let oversized = RecordBatch::try_new( - oversized_schema.clone(), - vec![Arc::new(StringArray::from(vec!["x".repeat(4000000)]))], + let engine = FixtureEngine::default().with_batches(42, batch.schema(), vec![batch.clone()]); + let mut server = Server::start_with_options( + Arc::new(engine), + spark_connect_server::ServerOptions { + max_arrow_chunk_size: 3 * 1024 * 1024, + }, ) - .unwrap(); - let engine = FixtureEngine::default() - .with_batches(2, schema, vec![batch]) - .with_batches(3, oversized_schema, vec![oversized]); - let mut server = Server::start_with_engine(Arc::new(engine)).await; - let mut stream = server - .client - .execute_plan(range(&session, 2)) - .await - .unwrap() - .into_inner(); - let (mut rows, mut batches, mut complete) = (0, 0, false); - while let Some(response) = stream.message().await.unwrap() { - assert!(response.encoded_len() <= 3 * 1024 * 1024); - match response.response_type { - Some(ResponseType::ArrowBatch(batch)) => { - assert_eq!(batch.start_offset, Some(rows)); - let records = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); - let mut decoded = 0; - for record in records { - let record = record.unwrap(); - let ids = record - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let strings = record.column(1); - for index in 0..record.num_rows() { - let id = rows + decoded + index as i64; - assert_eq!(ids.value(index), id); - let value = match strings.data_type() { - DataType::Utf8 => strings - .as_any() - .downcast_ref::() - .unwrap() - .value(index), - DataType::LargeUtf8 => strings - .as_any() - .downcast_ref::() - .unwrap() - .value(index), - DataType::Utf8View => strings - .as_any() - .downcast_ref::() - .unwrap() - .value(index), - other => panic!("unexpected string type: {other:?}"), - }; - assert_eq!(value.len(), if id % 64 == 0 { 200000 } else { 1000 }); - assert!(value.bytes().all(|byte| byte == b'x')); - } - decoded += record.num_rows() as i64; - } - assert_eq!(decoded, batch.row_count); - rows += decoded; - batches += 1; - } - Some(ResponseType::ResultComplete(_)) => complete = true, - None => {} - other => panic!("unexpected result: {other:?}"), + .await; + for preferred in [ + Some(1024), + Some(65_537), + Some(3 * 1024 * 1024), + None, + Some(0), + Some(-1), + Some(1023), + Some(3 * 1024 * 1024 + 1), + Some(i64::MAX), + ] { + let mut request = range(&Uuid::new_v4().to_string(), 42); + request.request_options = vec![chunking(true, preferred)]; + let responses = collect_result(&mut server, request).await; + let payloads: Vec<_> = responses + .iter() + .filter_map(|response| match &response.response_type { + Some(ResponseType::ArrowBatch(batch)) => Some(batch.data.len()), + _ => None, + }) + .collect(); + assert!(payloads.len() >= 2); + if let Some(size) = preferred.filter(|size| (1024..=3 * 1024 * 1024).contains(size)) { + assert!(payloads.iter().all(|len| *len <= size as usize)); + assert_eq!(payloads[0], size as usize); + } else { + assert_eq!( + payloads.len(), + 2, + "invalid preferences use the server maximum" + ); } + let output = assembled_batches(&responses, true); + assert_eq!(output.len(), 1); + assert_eq!(output[0].row_count, 1); + let mut reader = StreamReader::try_new(output[0].data.as_ref(), None).unwrap(); + assert_eq!(reader.next().unwrap().unwrap(), batch); + assert!(reader.next().is_none()); } - assert_eq!(rows, 8192); - assert!(batches > 1); - assert!(complete); - - let mut oversized = server - .client - .execute_plan(range(&session, 3)) - .await - .unwrap() - .into_inner(); - assert!(oversized.message().await.unwrap().unwrap().schema.is_some()); - let status = oversized.message().await.unwrap_err(); - assert_eq!(status.code(), Code::ResourceExhausted); server.stop().await; } @@ -374,7 +444,7 @@ async fn deeply_nested_requests_follow_the_configured_protobuf_decoder_limit() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn view_backing_buffers_are_compacted_before_splitting_including_struct_children() { +async fn view_arrays_are_materialized_once_before_byte_chunking_including_struct_children() { use arrow::array::{BinaryArray, BinaryViewArray, StringArray, StringViewArray, StructArray}; use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatch; @@ -431,67 +501,420 @@ async fn view_backing_buffers_are_compacted_before_splitting_including_struct_ch .unwrap(); let engine = FixtureEngine::default().with_batches(42, batch.schema(), vec![batch]); let mut server = Server::start_with_engine(Arc::new(engine)).await; - let request = range(&session, 42); - let mut stream = server - .client - .execute_plan(request) + let mut request = range(&session, 42); + request.request_options = vec![chunking(true, Some(3 * 1024 * 1024))]; + let responses = collect_result(&mut server, request).await; + let output = assembled_batches(&responses, true); + assert_eq!(output.len(), 1); + assert!(output[0].num_chunks_in_batch.unwrap() > 1); + let (mut rows, mut batches) = (0, 0); + for batch in output { + assert_eq!(batch.start_offset, Some(rows)); + let records = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); + let mut decoded = 0; + for record in records { + let record = record.unwrap(); + let ids = record + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(record.column(1).data_type(), &DataType::LargeUtf8); + assert_eq!(record.column(2).data_type(), &DataType::LargeBinary); + let strings = arrow::compute::cast(record.column(1), &DataType::Utf8).unwrap(); + let strings = strings.as_any().downcast_ref::().unwrap(); + let binaries = arrow::compute::cast(record.column(2), &DataType::Binary).unwrap(); + let binaries = binaries.as_any().downcast_ref::().unwrap(); + let nested = record + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(nested.column(0).data_type(), &DataType::LargeUtf8); + let nested_strings = arrow::compute::cast(nested.column(0), &DataType::Utf8).unwrap(); + let nested_strings = nested_strings + .as_any() + .downcast_ref::() + .unwrap(); + for index in 0..record.num_rows() { + let id = rows + decoded + index as i64; + assert_eq!(ids.value(index), id); + assert_eq!(strings.value(index), text(id)); + assert_eq!(binaries.value(index), binary(id)); + assert_eq!(nested_strings.value(index), nested_text(id)); + } + decoded += record.num_rows() as i64; + } + assert_eq!(decoded, batch.row_count); + rows += decoded; + batches += 1; + } + assert_eq!(rows, 4096); + assert_eq!(batches, 1); + server.stop().await; +} + +fn integers(rows: usize) -> RecordBatch { + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), + vec![Arc::new(Int64Array::from_iter_values(0..rows as i64))], + ) + .unwrap() +} + +#[tokio::test] +async fn repeated_chunk_options_enable_if_any_and_use_the_first_preference() { + let batch = integers(3000); + let engine = FixtureEngine::default().with_batches(42, batch.schema(), vec![batch.clone()]); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + for (options, size) in [ + ( + vec![chunking(false, Some(1024)), chunking(true, None)], + 1024, + ), + ( + vec![chunking(false, None), chunking(true, Some(2048))], + 2048, + ), + ( + vec![chunking(true, Some(1023)), chunking(true, Some(1024))], + DEFAULT_ARROW_CHUNK_SIZE, + ), + ] { + let mut request = range(&Uuid::new_v4().to_string(), 42); + request.request_options = options; + let responses = collect_result(&mut server, request).await; + let payloads: Vec<_> = responses + .iter() + .filter_map(|response| match &response.response_type { + Some(ResponseType::ArrowBatch(batch)) => Some(batch.data.len()), + _ => None, + }) + .collect(); + if size < DEFAULT_ARROW_CHUNK_SIZE { + assert_eq!(payloads[0], size); + } else { + assert_eq!(payloads.len(), 1); + } + assert!(payloads.iter().all(|len| *len <= size)); + let output = assembled_batches(&responses, true); + assert_eq!(output.len(), 1); + let mut reader = StreamReader::try_new(output[0].data.as_ref(), None).unwrap(); + assert_eq!(reader.next().unwrap().unwrap(), batch); + } + server.stop().await; +} + +async fn collect_small_slices(backing: RecordBatch) -> Vec { + let batches = (0..20) + .map(|index| backing.slice(index * 1000, 1000)) + .collect(); + let engine = FixtureEngine::default().with_batches(42, backing.schema(), batches); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let output = collect_result(&mut server, range(&Uuid::new_v4().to_string(), 42)).await; + let (mut batches, mut rows) = (Vec::new(), 0); + for message in output { + assert!(message.encoded_len() <= 3 * 1024 * 1024); + if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + message.response_type + { + assert_eq!(batch.start_offset, Some(rows)); + assert_eq!(batch.row_count, 1000); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + let decoded = reader.next().unwrap().unwrap(); + assert_eq!(decoded.num_rows() as i64, batch.row_count); + batches.push(decoded); + assert!(reader.next().is_none()); + rows += batch.row_count; + } + } + assert_eq!(batches.len(), 20); + assert_eq!(rows, 20_000); + server.stop().await; + batches +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn slices_of_a_large_allocation_remain_one_response_per_small_batch() { + for (index, batch) in collect_small_slices(integers(10_000_000)) .await - .unwrap() - .into_inner(); - let (mut rows, mut batches, mut complete) = (0, 0, false); - while let Some(response) = stream.message().await.unwrap() { - assert!(response.encoded_len() <= 3 * 1024 * 1024); - match response.response_type { - Some(ResponseType::ArrowBatch(batch)) => { - assert_eq!(batch.start_offset, Some(rows)); - let records = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); - let mut decoded = 0; - for record in records { - let record = record.unwrap(); - let ids = record + .iter() + .enumerate() + { + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let start = index as i64 * 1000; + assert!(values.values().iter().copied().eq(start..start + 1000)); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn nested_slices_preserve_referenced_child_values() { + // The parent needs only 20,000 rows to retain the full 80 MB child. Exercise + // each offset width and maps sequentially, sharing the same allocation. + let child: ArrayRef = Arc::new(Int64Array::from_iter_values(0..10_000_000)); + for kind in 0..3 { + let item = Arc::new(Field::new("item", DataType::Int64, false)); + let column: ArrayRef = match kind { + 0 => Arc::new(ListArray::new( + item, + OffsetBuffer::new((0..=20_000i32).collect::>().into()), + child.clone(), + None, + )), + 1 => Arc::new(LargeListArray::new( + item, + OffsetBuffer::new((0..=20_000i64).collect::>().into()), + child.clone(), + None, + )), + _ => { + let fields = vec![ + Field::new("key", DataType::Int64, false), + Field::new("value", DataType::Int64, false), + ]; + let entries = StructArray::new( + fields.clone().into(), + vec![child.clone(), child.clone()], + None, + ); + Arc::new(MapArray::new( + Arc::new(Field::new( + "entries", + DataType::Struct(fields.into()), + false, + )), + OffsetBuffer::new((0..=20_000i32).collect::>().into()), + entries, + None, + false, + )) + } + }; + let schema = Arc::new(Schema::new(vec![Field::new( + "nested", + column.data_type().clone(), + false, + )])); + let backing = RecordBatch::try_new(schema, vec![column]).unwrap(); + let batches = collect_small_slices(backing).await; + let mut row = 0i64; + for batch in batches { + for index in 0..batch.num_rows() { + let value = match kind { + 0 => batch .column(0) .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(record.column(1).data_type(), &DataType::LargeUtf8); - assert_eq!(record.column(2).data_type(), &DataType::LargeBinary); - let strings = arrow::compute::cast(record.column(1), &DataType::Utf8).unwrap(); - let strings = strings.as_any().downcast_ref::().unwrap(); - let binaries = - arrow::compute::cast(record.column(2), &DataType::Binary).unwrap(); - let binaries = binaries.as_any().downcast_ref::().unwrap(); - let nested = record - .column(3) + .downcast_ref::() + .unwrap() + .value(index), + 1 => batch + .column(0) .as_any() - .downcast_ref::() + .downcast_ref::() + .unwrap() + .value(index), + _ => { + let entry = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(index); + assert_eq!(entry.len(), 1); + let key = entry + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(key.value(0), row); + entry.column(1).clone() + } + }; + let value = value.as_any().downcast_ref::().unwrap(); + assert_eq!(value.len(), 1); + assert_eq!(value.value(0), row); + row += 1; + } + } + assert_eq!(row, 20_000); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_slices_preserve_current_values() { + fn text(row: usize) -> String { + format!("{row:010}{}", "y".repeat(30)) + } + + let values = StringViewArray::from_iter_values((0..1_000_000).map(text)); + let (views, buffers, nulls) = values.into_parts(); + for binary in [false, true] { + let column: ArrayRef = if binary { + Arc::new(BinaryViewArray::new( + views.clone(), + buffers.clone(), + nulls.clone(), + )) + } else { + Arc::new(StringViewArray::new( + views.clone(), + buffers.clone(), + nulls.clone(), + )) + }; + let schema = Arc::new(Schema::new(vec![Field::new( + "view", + column.data_type().clone(), + false, + )])); + let backing = RecordBatch::try_new(schema, vec![column]).unwrap(); + let batches = collect_small_slices(backing).await; + let mut row = 0; + for batch in batches { + for index in 0..batch.num_rows() { + let expected = text(row); + if binary { + let values = batch + .column(0) + .as_any() + .downcast_ref::() .unwrap(); - assert_eq!(nested.column(0).data_type(), &DataType::LargeUtf8); - let nested_strings = - arrow::compute::cast(nested.column(0), &DataType::Utf8).unwrap(); - let nested_strings = nested_strings + assert_eq!(values.value(index), expected.as_bytes()); + } else { + let values = batch + .column(0) .as_any() - .downcast_ref::() + .downcast_ref::() .unwrap(); - for index in 0..record.num_rows() { - let id = rows + decoded + index as i64; - assert_eq!(ids.value(index), id); - assert_eq!(strings.value(index), text(id)); - assert_eq!(binaries.value(index), binary(id)); - assert_eq!(nested_strings.value(index), nested_text(id)); - } - decoded += record.num_rows() as i64; + assert_eq!(values.value(index), expected); } - assert_eq!(decoded, batch.row_count); - rows += decoded; - batches += 1; + row += 1; } - Some(ResponseType::ResultComplete(_)) => complete = true, - None => {} - other => panic!("unexpected result: {other:?}"), } + assert_eq!(row, 20_000); + } +} + +#[cfg(target_pointer_width = "64")] +#[tokio::test] +async fn unrepresentable_engine_batches_preserve_preceding_results_and_fail_cleanly() { + use arrow::array::NullArray; + use arrow::record_batch::RecordBatchOptions; + + let rows = i64::MAX as usize + 1; + for null_column in [false, true] { + let oversized = if null_column { + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("nulls", DataType::Null, true)])), + vec![Arc::new(NullArray::new(rows))], + ) + .unwrap() + } else { + RecordBatch::try_new_with_options( + Arc::new(Schema::empty()), + vec![], + &RecordBatchOptions::new().with_row_count(Some(rows)), + ) + .unwrap() + }; + let engine = FixtureEngine::default().with_batches( + 42, + oversized.schema(), + vec![oversized.slice(0, 3), oversized], + ); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let mut stream = server + .client + .execute_plan(range(&Uuid::new_v4().to_string(), 42)) + .await + .unwrap() + .into_inner(); + let (status, counts) = tokio::time::timeout(Duration::from_secs(15), async { + let mut counts = Vec::new(); + loop { + match stream.message().await { + Ok(Some(message)) => match message.response_type { + Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) => { + assert_eq!(batch.start_offset, Some(0)); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + assert_eq!(reader.next().unwrap().unwrap().num_rows(), 3); + assert!(reader.next().is_none()); + counts.push(batch.row_count); + } + Some(proto::execute_plan_response::ResponseType::ResultComplete(_)) => { + panic!("unrepresentable output must not complete successfully"); + } + _ => {} + }, + Err(status) => break (status, counts), + Ok(None) => panic!("expected row-count overflow status"), + } + } + }) + .await + .unwrap(); + assert_eq!(status.code(), Code::ResourceExhausted); + assert_eq!(status.message(), "result batch row count exceeds i64"); + assert_eq!(counts, vec![3]); + server.stop().await; + } +} + +#[tokio::test] +async fn configured_maximum_bounds_payloads_and_zero_disables_chunking() { + assert_eq!( + spark_connect_server::ServerOptions::default().max_arrow_chunk_size, + DEFAULT_ARROW_CHUNK_SIZE + ); + let batch = integers(5000); + let bytes = encoded(&batch, None); + for maximum in [4096, 0] { + let engine = FixtureEngine::default().with_results( + 42, + batch.schema(), + vec![Batch::Ipc(bytes.clone(), 5000)], + ); + let mut server = Server::start_with_options( + Arc::new(engine), + spark_connect_server::ServerOptions { + max_arrow_chunk_size: maximum, + }, + ) + .await; + for preferred in [None, Some(2048), Some(4096), Some(4097)] { + let mut request = range(&Uuid::new_v4().to_string(), 42); + request.request_options = vec![chunking(true, preferred)]; + let responses = collect_result(&mut server, request).await; + let output = assembled_batches(&responses, maximum != 0); + assert_eq!(output.len(), 1); + assert_eq!(output[0].data.as_ref(), bytes.as_slice()); + let chunks: Vec<_> = responses + .iter() + .filter_map(|response| match &response.response_type { + Some(ResponseType::ArrowBatch(batch)) => { + Some((response.encoded_len(), batch.data.len())) + } + _ => None, + }) + .collect(); + if maximum == 0 { + assert_eq!(chunks.len(), 1); + } else { + let limit = preferred + .filter(|size| (1024..=maximum as i64).contains(size)) + .unwrap_or(maximum as i64) as usize; + assert!(chunks.iter().all(|(_, payload)| *payload <= limit)); + assert_eq!(chunks[0].1, limit); + assert!( + chunks[0].0 > limit, + "the chunk limit excludes the protobuf envelope" + ); + } + } + server.stop().await; } - assert_eq!(rows, 4096); - assert!(batches > 1); - assert!(complete); - server.stop().await; } diff --git a/crates/spark-connect-server/tests/review_regressions.rs b/crates/spark-connect-server/tests/review_regressions.rs deleted file mode 100644 index 43f46c99..00000000 --- a/crates/spark-connect-server/tests/review_regressions.rs +++ /dev/null @@ -1,652 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Regressions for sliced output, protocol IDs and backend panic boundaries. - -#[path = "support/server.rs"] -mod server; - -use std::future::Future; -use std::pin::Pin; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::Arc; -use std::time::Duration; - -use arrow::array::{ - Array, ArrayRef, BinaryViewArray, Int64Array, LargeBinaryArray, LargeListArray, - LargeStringArray, ListArray, MapArray, StringViewArray, StructArray, -}; -use arrow::buffer::OffsetBuffer; -use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; -use arrow::ipc::reader::StreamReader; -use arrow::record_batch::RecordBatch; -use prost::Message; -use server::Server; -use spark_connect_proto as proto; -use spark_connect_server::engine::{ - BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, - ResultBatch, SessionContext, -}; -use tonic::{Code, Status}; -use uuid::Uuid; - -#[derive(Clone, Copy, PartialEq, Eq)] -enum Call { - Prepare, - Analyze, - Config, -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum PanicAt { - Construction, - Poll, -} - -struct State { - batches: Vec, - panic: Option<(Call, PanicAt)>, - panicked: AtomicBool, - sessions: AtomicUsize, - prepares: AtomicUsize, -} - -impl State { - fn new(batches: Vec, panic: Option<(Call, PanicAt)>) -> Arc { - Arc::new(Self { - batches, - panic, - panicked: AtomicBool::new(false), - sessions: AtomicUsize::new(0), - prepares: AtomicUsize::new(0), - }) - } - - fn panic_once(&self, call: Call) -> Option { - self.panic - .filter(|(target, _)| *target == call) - .filter(|_| !self.panicked.swap(true, Ordering::SeqCst)) - .map(|(_, at)| at) - } -} - -struct Fixture(Arc); - -impl Engine for Fixture { - fn create_session(&self, _: SessionContext) -> Result, Status> { - self.0.sessions.fetch_add(1, Ordering::SeqCst); - Ok(Arc::new(Self(self.0.clone()))) - } -} - -type EngineFuture<'a, T> = Pin> + Send + 'a>>; - -// Implement the expanded async-trait signatures directly so the fixture can -// panic both before returning a future and while that future is polled. -impl EngineSession for Fixture { - fn prepare<'life0, 'async_trait>( - &'life0 self, - _: proto::plan::OpType, - ) -> EngineFuture<'async_trait, PreparedOperation> - where - 'life0: 'async_trait, - Self: 'async_trait, - { - self.0.prepares.fetch_add(1, Ordering::SeqCst); - let panic = self.0.panic_once(Call::Prepare); - assert!(panic != Some(PanicAt::Construction), "prepare construction"); - Box::pin(async move { - assert!(panic != Some(PanicAt::Poll), "prepare poll"); - Ok(PreparedOperation::Query(Box::new(Query( - self.0.batches.clone(), - )))) - }) - } - - fn analyze<'life0, 'async_trait>( - &'life0 self, - _: proto::analyze_plan_request::Analyze, - ) -> EngineFuture<'async_trait, proto::analyze_plan_response::Result> - where - 'life0: 'async_trait, - Self: 'async_trait, - { - let panic = self.0.panic_once(Call::Analyze); - assert!(panic != Some(PanicAt::Construction), "analyze construction"); - Box::pin(async move { - assert!(panic != Some(PanicAt::Poll), "analyze poll"); - Ok(proto::analyze_plan_response::Result::SparkVersion( - proto::analyze_plan_response::SparkVersion { - version: "fixture".into(), - }, - )) - }) - } - - fn config<'life0, 'async_trait>( - &'life0 self, - _: proto::config_request::operation::OpType, - ) -> EngineFuture<'async_trait, ConfigResult> - where - 'life0: 'async_trait, - Self: 'async_trait, - { - let panic = self.0.panic_once(Call::Config); - assert!(panic != Some(PanicAt::Construction), "config construction"); - Box::pin(async move { - assert!(panic != Some(PanicAt::Poll), "config poll"); - Ok(ConfigResult::default()) - }) - } -} - -struct Query(Vec); - -#[tonic::async_trait] -impl PreparedQuery for Query { - fn schema(&self) -> SchemaRef { - self.0[0].schema() - } - - async fn execute(self: Box) -> Result { - Ok(Box::pin(futures::stream::iter( - self.0 - .into_iter() - .map(|batch| Ok(ResultBatch::Arrow(batch))), - ))) - } -} - -fn integers(rows: usize) -> RecordBatch { - RecordBatch::try_new( - Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), - vec![Arc::new(Int64Array::from_iter_values(0..rows as i64))], - ) - .unwrap() -} - -fn execute(session: &str, operation: &str) -> proto::ExecutePlanRequest { - proto::ExecutePlanRequest { - session_id: session.into(), - operation_id: Some(operation.into()), - plan: Some(proto::Plan { - op_type: Some(proto::plan::OpType::Root(proto::Relation::default())), - }), - ..Default::default() - } -} - -fn config(session: &str) -> proto::ConfigRequest { - proto::ConfigRequest { - session_id: session.into(), - operation: Some(proto::config_request::Operation { - op_type: Some(proto::config_request::operation::OpType::GetAll( - Default::default(), - )), - }), - ..Default::default() - } -} - -fn analyze(session: &str) -> proto::AnalyzePlanRequest { - proto::AnalyzePlanRequest { - session_id: session.into(), - analyze: Some(proto::analyze_plan_request::Analyze::SparkVersion( - Default::default(), - )), - ..Default::default() - } -} - -fn release(session: &str, operation: &str) -> proto::ReleaseExecuteRequest { - proto::ReleaseExecuteRequest { - session_id: session.into(), - operation_id: operation.into(), - release: Some(proto::release_execute_request::Release::ReleaseAll( - Default::default(), - )), - ..Default::default() - } -} - -async fn collect( - server: &mut Server, - session: &str, - operation: &str, -) -> Vec { - tokio::time::timeout(Duration::from_secs(15), async { - let mut stream = server - .client - .execute_plan(execute(session, operation)) - .await - .unwrap() - .into_inner(); - let mut output = Vec::new(); - while let Some(message) = stream.message().await.unwrap() { - output.push(message); - } - assert!(matches!( - output.last().unwrap().response_type.as_ref(), - Some(proto::execute_plan_response::ResponseType::ResultComplete( - _ - )) - )); - output - }) - .await - .unwrap() -} - -async fn collect_small_slices(backing: RecordBatch) -> Vec { - let batches = (0..20) - .map(|index| backing.slice(index * 1000, 1000)) - .collect(); - let state = State::new(batches, None); - let mut server = Server::start_with_engine(Arc::new(Fixture(state))).await; - let output = collect( - &mut server, - &Uuid::new_v4().to_string(), - &Uuid::new_v4().to_string(), - ) - .await; - let (mut batches, mut rows) = (Vec::new(), 0); - for message in output { - assert!(message.encoded_len() <= 3 * 1024 * 1024); - if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = - message.response_type - { - assert_eq!(batch.start_offset, Some(rows)); - assert_eq!(batch.row_count, 1000); - let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); - let decoded = reader.next().unwrap().unwrap(); - assert_eq!(decoded.num_rows() as i64, batch.row_count); - batches.push(decoded); - assert!(reader.next().is_none()); - rows += batch.row_count; - } - } - assert_eq!(batches.len(), 20); - assert_eq!(rows, 20_000); - server.stop().await; - batches -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn slices_of_a_large_allocation_remain_one_response_per_small_batch() { - for (index, batch) in collect_small_slices(integers(10_000_000)) - .await - .iter() - .enumerate() - { - let values = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let start = index as i64 * 1000; - assert!(values.values().iter().copied().eq(start..start + 1000)); - } -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn nested_slices_only_budget_the_referenced_child_range() { - // The parent needs only 20,000 rows to retain the full 80 MB child. Exercise - // each offset width and maps sequentially, sharing the same allocation. - let child: ArrayRef = Arc::new(Int64Array::from_iter_values(0..10_000_000)); - for kind in 0..3 { - let item = Arc::new(Field::new("item", DataType::Int64, false)); - let column: ArrayRef = match kind { - 0 => Arc::new(ListArray::new( - item, - OffsetBuffer::new((0..=20_000i32).collect::>().into()), - child.clone(), - None, - )), - 1 => Arc::new(LargeListArray::new( - item, - OffsetBuffer::new((0..=20_000i64).collect::>().into()), - child.clone(), - None, - )), - _ => { - let fields = vec![ - Field::new("key", DataType::Int64, false), - Field::new("value", DataType::Int64, false), - ]; - let entries = StructArray::new( - fields.clone().into(), - vec![child.clone(), child.clone()], - None, - ); - Arc::new(MapArray::new( - Arc::new(Field::new( - "entries", - DataType::Struct(fields.into()), - false, - )), - OffsetBuffer::new((0..=20_000i32).collect::>().into()), - entries, - None, - false, - )) - } - }; - let schema = Arc::new(Schema::new(vec![Field::new( - "nested", - column.data_type().clone(), - false, - )])); - let backing = RecordBatch::try_new(schema, vec![column]).unwrap(); - let batches = collect_small_slices(backing).await; - let mut row = 0i64; - for batch in batches { - for index in 0..batch.num_rows() { - let value = match kind { - 0 => batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap() - .value(index), - 1 => batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap() - .value(index), - _ => { - let entry = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap() - .value(index); - assert_eq!(entry.len(), 1); - let key = entry - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(key.value(0), row); - entry.column(1).clone() - } - }; - let value = value.as_any().downcast_ref::().unwrap(); - assert_eq!(value.len(), 1); - assert_eq!(value.value(0), row); - row += 1; - } - } - assert_eq!(row, 20_000); - } -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn view_slices_only_budget_the_current_values() { - fn text(row: usize) -> String { - format!("{row:010}{}", "y".repeat(30)) - } - - let values = StringViewArray::from_iter_values((0..1_000_000).map(text)); - let (views, buffers, nulls) = values.into_parts(); - for binary in [false, true] { - let column: ArrayRef = if binary { - Arc::new(BinaryViewArray::new( - views.clone(), - buffers.clone(), - nulls.clone(), - )) - } else { - Arc::new(StringViewArray::new( - views.clone(), - buffers.clone(), - nulls.clone(), - )) - }; - let schema = Arc::new(Schema::new(vec![Field::new( - "view", - column.data_type().clone(), - false, - )])); - let backing = RecordBatch::try_new(schema, vec![column]).unwrap(); - let batches = collect_small_slices(backing).await; - let mut row = 0; - for batch in batches { - for index in 0..batch.num_rows() { - let expected = text(row); - if binary { - let values = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(values.value(index), expected.as_bytes()); - } else { - let values = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(values.value(index), expected); - } - row += 1; - } - } - assert_eq!(row, 20_000); - } -} - -#[cfg(target_pointer_width = "64")] -#[tokio::test] -async fn unrepresentable_engine_batches_preserve_preceding_results_and_fail_cleanly() { - use arrow::array::NullArray; - use arrow::record_batch::RecordBatchOptions; - - let rows = i64::MAX as usize + 1; - for null_column in [false, true] { - let oversized = if null_column { - RecordBatch::try_new( - Arc::new(Schema::new(vec![Field::new("nulls", DataType::Null, true)])), - vec![Arc::new(NullArray::new(rows))], - ) - .unwrap() - } else { - RecordBatch::try_new_with_options( - Arc::new(Schema::empty()), - vec![], - &RecordBatchOptions::new().with_row_count(Some(rows)), - ) - .unwrap() - }; - let state = State::new(vec![oversized.slice(0, 3), oversized], None); - let mut server = Server::start_with_engine(Arc::new(Fixture(state))).await; - let mut stream = server - .client - .execute_plan(execute( - &Uuid::new_v4().to_string(), - &Uuid::new_v4().to_string(), - )) - .await - .unwrap() - .into_inner(); - let (status, counts) = tokio::time::timeout(Duration::from_secs(15), async { - let mut counts = Vec::new(); - loop { - match stream.message().await { - Ok(Some(message)) => match message.response_type { - Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) => { - assert_eq!(batch.start_offset, Some(0)); - let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); - assert_eq!(reader.next().unwrap().unwrap().num_rows(), 3); - assert!(reader.next().is_none()); - counts.push(batch.row_count); - } - Some(proto::execute_plan_response::ResponseType::ResultComplete(_)) => { - panic!("unrepresentable output must not complete successfully"); - } - _ => {} - }, - Err(status) => break (status, counts), - Ok(None) => panic!("expected row-count overflow status"), - } - } - }) - .await - .unwrap(); - assert_eq!(status.code(), Code::ResourceExhausted); - assert_eq!(status.message(), "result batch row count exceeds i64"); - assert_eq!(counts, vec![3]); - server.stop().await; - } -} - -#[tokio::test] -async fn noncanonical_ids_are_rejected_without_creating_or_releasing_state() { - let state = State::new(vec![integers(1)], None); - let mut server = Server::start_with_engine(Arc::new(Fixture(state.clone()))).await; - let uuid = Uuid::parse_str("abcdef12-3456-4789-abcd-0123456789ab").unwrap(); - let session = uuid.to_string(); - let operation = Uuid::new_v4().to_string(); - let original = server - .client - .config(config(&session)) - .await - .unwrap() - .into_inner() - .server_side_session_id; - for alias in [ - uuid.simple().to_string(), - uuid.braced().to_string(), - uuid.urn().to_string(), - session.to_uppercase(), - ] { - assert_eq!( - server - .client - .config(config(&alias)) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - assert_eq!( - server - .client - .analyze_plan(analyze(&alias)) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - assert_eq!( - server - .client - .execute_plan(execute(&alias, &operation)) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - assert_eq!( - server - .client - .release_execute(release(&alias, &operation)) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - assert_eq!( - server - .client - .release_session(proto::ReleaseSessionRequest { - session_id: alias.clone(), - ..Default::default() - }) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - assert_eq!( - server - .client - .execute_plan(execute(&Uuid::new_v4().to_string(), &alias)) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - assert_eq!( - server - .client - .release_execute(release(&session, &alias)) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - assert_eq!( - server - .client - .config(config(&session)) - .await - .unwrap() - .into_inner() - .server_side_session_id, - original - ); - } - assert_eq!(state.sessions.load(Ordering::SeqCst), 1); - assert_eq!(state.prepares.load(Ordering::SeqCst), 0); - collect(&mut server, &session, &operation).await; - server.stop().await; -} - -#[tokio::test] -async fn backend_request_panics_have_internal_status_and_leave_the_session_usable() { - for call in [Call::Prepare, Call::Analyze, Call::Config] { - for at in [PanicAt::Construction, PanicAt::Poll] { - let state = State::new(vec![integers(1)], Some((call, at))); - let mut server = Server::start_with_engine(Arc::new(Fixture(state.clone()))).await; - let session = Uuid::new_v4().to_string(); - let operation = Uuid::new_v4().to_string(); - let status = match call { - Call::Prepare => server - .client - .execute_plan(execute(&session, &operation)) - .await - .unwrap_err(), - Call::Analyze => server - .client - .analyze_plan(analyze(&session)) - .await - .unwrap_err(), - Call::Config => server.client.config(config(&session)).await.unwrap_err(), - }; - assert_eq!(status.code(), Code::Internal); - assert_eq!(status.message(), "engine request panicked"); - server.client.config(config(&session)).await.unwrap(); - server.client.analyze_plan(analyze(&session)).await.unwrap(); - // Reuse the operation ID after a preparation panic: its reservation - // must have been released before the failed RPC returned. - collect(&mut server, &session, &operation).await; - assert_eq!(state.sessions.load(Ordering::SeqCst), 1); - server.stop().await; - } - } -} diff --git a/crates/spark-connect-server/tests/schema_consistency.rs b/crates/spark-connect-server/tests/schema_consistency.rs index 24f9bde5..82ed77bd 100644 --- a/crates/spark-connect-server/tests/schema_consistency.rs +++ b/crates/spark-connect-server/tests/schema_consistency.rs @@ -17,6 +17,8 @@ //! Result schemas remain consumable as one Arrow table across output batches. +#[path = "support/arrow.rs"] +mod arrow_support; #[path = "support/requests.rs"] mod requests; #[path = "support/server.rs"] @@ -41,6 +43,7 @@ use spark_connect_proto as proto; use tonic::{Code, Status}; use uuid::Uuid; +use arrow_support::{assembled_batches, chunking}; use server::Server; use support::{Batch, FixtureEngine}; @@ -64,7 +67,8 @@ fn ipc(batch: &RecordBatch) -> Vec { async fn execute(engine: BatchCase) -> (Vec, Option) { let engine = FixtureEngine::default().with_results(1, engine.schema, engine.batches); let mut server = Server::start_with_engine(Arc::new(engine)).await; - let request = requests::request(&Uuid::new_v4().to_string(), 1); + let mut request = requests::request(&Uuid::new_v4().to_string(), 1); + request.request_options = vec![chunking(true, Some(RESULT_LIMIT as i64))]; let mut stream = server .client .execute_plan(request) @@ -76,7 +80,14 @@ async fn execute(engine: BatchCase) -> (Vec, Option< loop { match stream.message().await { Ok(Some(response)) => { - assert!(response.encoded_len() <= RESULT_LIMIT); + if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + &response.response_type + { + assert!(batch.data.len() <= RESULT_LIMIT); + assert!(response.encoded_len() <= batch.data.len() + 512); + } else { + assert!(response.encoded_len() <= RESULT_LIMIT); + } responses.push(response); } Ok(None) => break (responses, None), @@ -91,22 +102,16 @@ async fn execute(engine: BatchCase) -> (Vec, Option< } fn output_batches(responses: &[proto::ExecutePlanResponse]) -> Vec { - let mut offset = 0; - let mut batches = Vec::new(); - for response in responses { - if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = - &response.response_type - { - assert_eq!(batch.start_offset, Some(offset)); - let mut reader = StreamReader::try_new(Cursor::new(&batch.data), None).unwrap(); + assembled_batches(responses, true) + .into_iter() + .map(|batch| { + let mut reader = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); let decoded = reader.next().unwrap().unwrap(); assert_eq!(decoded.num_rows() as i64, batch.row_count); assert!(reader.next().is_none()); - offset += batch.row_count; - batches.push(decoded); - } - } - batches + decoded + }) + .collect() } fn assert_schema_error(responses: &[proto::ExecutePlanResponse], status: Option) { @@ -234,7 +239,7 @@ fn views(rows: usize) -> RecordBatch { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn view_ipc_is_unchanged_when_small_and_materialized_when_oversized() { +async fn view_ipc_remains_unchanged_for_small_and_oversized_batches() { for rows in [2, LARGE_ROWS] { let batch = views(rows); let data = ipc(&batch); @@ -246,78 +251,44 @@ async fn view_ipc_is_unchanged_when_small_and_materialized_when_oversized() { .await; assert!(status.is_none(), "{status:?}"); assert!(completed(&responses)); + let output = assembled_batches(&responses, true); + assert_eq!(output.len(), 1); + assert_eq!(output[0].data.as_ref(), data.as_slice()); let batches = output_batches(&responses); - if rows == 2 { - let payload = responses - .iter() - .find_map(|response| match &response.response_type { - Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) => { - Some(&batch.data) - } - _ => None, - }) - .unwrap(); - assert_eq!(payload.as_ref(), data.as_slice()); - assert_eq!( - batches[0].schema().field(0).data_type(), - &DataType::Utf8View - ); - } else { - assert!(batches.len() > 1); - let mut offset = 0; - for batch in batches { - assert_eq!(batch.schema().field(0).data_type(), &DataType::LargeUtf8); - let values = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - for row in 0..batch.num_rows() { - assert_eq!(values.value(row), text(offset + row)); - } - offset += batch.num_rows(); - } - assert_eq!(offset, rows); - } + assert_eq!(batches, vec![batch]); + assert_eq!( + batches[0].schema().field(0).data_type(), + &DataType::Utf8View + ); } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn mixing_small_and_oversized_view_ipc_rejects_the_changed_output_schema() { +async fn mixing_small_and_oversized_view_ipc_preserves_one_output_schema() { let small = views(2); let large = views(LARGE_ROWS); - let small = Batch::Ipc(ipc(&small), 2); - let large = Batch::Ipc(ipc(&large), LARGE_ROWS as i64); for large_first in [false, true] { + let expected = if large_first { + vec![large.clone(), small.clone()] + } else { + vec![small.clone(), large.clone()] + }; let (responses, status) = execute(BatchCase { - schema: views(0).schema(), - batches: if large_first { - vec![large.clone(), small.clone()] - } else { - vec![small.clone(), large.clone()] - }, + schema: small.schema(), + batches: expected + .iter() + .map(|batch| Batch::Ipc(ipc(batch), batch.num_rows() as i64)) + .collect(), }) .await; - assert_schema_error(&responses, status); - let batches = output_batches(&responses); - let expected_type = if large_first { - DataType::LargeUtf8 - } else { - DataType::Utf8View - }; - let expected_rows = if large_first { LARGE_ROWS } else { 2 }; - assert_eq!( - batches.iter().map(RecordBatch::num_rows).sum::(), - expected_rows - ); - for batch in batches { - assert_eq!(batch.schema().field(0).data_type(), &expected_type); - } + assert!(status.is_none(), "{status:?}"); + assert!(completed(&responses)); + assert_eq!(output_batches(&responses), expected); } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn view_children_in_lists_large_lists_and_maps_are_materialized_when_split() { +async fn arrow_view_children_are_materialized_before_byte_chunking() { let values: ArrayRef = Arc::new(StringViewArray::from_iter_values((0..LARGE_ROWS).map(text))); let offsets = OffsetBuffer::new((0..=LARGE_ROWS as i32).collect::>().into()); let item = Arc::new(Field::new("item", DataType::Utf8View, false)); @@ -368,7 +339,13 @@ async fn view_children_in_lists_large_lists_and_maps_are_materialized_when_split assert!(status.is_none(), "{status:?}"); assert!(completed(&responses)); let batches = output_batches(&responses); - assert!(batches.len() > 1); + assert_eq!(batches.len(), 1); + assert!( + assembled_batches(&responses, true)[0] + .num_chunks_in_batch + .unwrap() + > 1 + ); let mut offset = 0; for batch in batches { let lists = batch diff --git a/crates/spark-connect-server/tests/support/arrow.rs b/crates/spark-connect-server/tests/support/arrow.rs new file mode 100644 index 00000000..c9fd6979 --- /dev/null +++ b/crates/spark-connect-server/tests/support/arrow.rs @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Wire chunk negotiation, reassembly and IPC encoding shared by protocol tests. + +#![allow(dead_code)] + +use std::collections::HashSet; + +use arrow::ipc::writer::{IpcWriteOptions, StreamWriter}; +use arrow::ipc::CompressionType; +use arrow::record_batch::RecordBatch; +use prost::Message; +use proto::execute_plan_response::ResponseType; +use spark_connect_proto as proto; + +pub const DEFAULT_ARROW_CHUNK_SIZE: usize = 128 * 1024 * 1024 * 9 / 10; + +pub fn chunking(allow: bool, preferred: Option) -> proto::execute_plan_request::RequestOption { + proto::execute_plan_request::RequestOption { + request_option: Some( + proto::execute_plan_request::request_option::RequestOption::ResultChunkingOptions( + proto::ResultChunkingOptions { + allow_arrow_batch_chunking: allow, + preferred_arrow_chunk_size: preferred, + }, + ), + ), + } +} + +// Reassemble before Arrow decoding, just as Spark clients do. Every chunk counts +// the whole batch's rows and has its own response ID; row offsets advance once. +pub fn assembled_batches( + responses: &[proto::ExecutePlanResponse], + negotiated: bool, +) -> Vec { + let mut output = Vec::new(); + let mut pending: Option = None; + let (mut offset, mut index) = (0, 0); + let mut data = Vec::new(); + let mut response_ids = HashSet::new(); + for response in responses { + assert!(response_ids.insert(&response.response_id)); + let Some(ResponseType::ArrowBatch(chunk)) = &response.response_type else { + continue; + }; + // The negotiated limit covers IPC payload bytes, matching the JVM. The + // protobuf envelope adds identifiers, field tags and chunk metadata. + assert!(response.encoded_len() <= chunk.data.len() + 512); + if negotiated { + assert!(chunk.data.len() <= DEFAULT_ARROW_CHUNK_SIZE); + } + assert_eq!(chunk.start_offset, Some(offset)); + if !negotiated { + assert_eq!(chunk.chunk_index, None); + assert_eq!(chunk.num_chunks_in_batch, None); + offset += chunk.row_count; + output.push(chunk.clone()); + continue; + } + assert_eq!(chunk.chunk_index, Some(index)); + let count = chunk.num_chunks_in_batch.unwrap(); + assert!(count > index); + if let Some(batch) = &mut pending { + assert_eq!(batch.row_count, chunk.row_count); + assert_eq!(batch.num_chunks_in_batch, Some(count)); + } else { + pending = Some(chunk.clone()); + } + data.extend_from_slice(&chunk.data); + index += 1; + if index == count { + let mut batch = pending.take().unwrap(); + batch.data = std::mem::take(&mut data).into(); + offset += batch.row_count; + output.push(batch); + index = 0; + } + } + assert!(pending.is_none(), "complete must not truncate a batch"); + output +} + +pub fn encoded(batch: &RecordBatch, compression: Option) -> Vec { + let options = IpcWriteOptions::default() + .try_with_compression(compression) + .unwrap(); + let mut data = Vec::new(); + { + let mut writer = + StreamWriter::try_new_with_options(&mut data, batch.schema().as_ref(), options) + .unwrap(); + writer.write(batch).unwrap(); + writer.finish().unwrap(); + } + data +} diff --git a/crates/spark-connect-server/tests/support/server.rs b/crates/spark-connect-server/tests/support/server.rs index 0ec88720..74602859 100644 --- a/crates/spark-connect-server/tests/support/server.rs +++ b/crates/spark-connect-server/tests/support/server.rs @@ -36,11 +36,18 @@ pub struct Server { impl Server { pub async fn start_with_engine(engine: Arc) -> Self { + Self::start_with_options(engine, spark_connect_server::ServerOptions::default()).await + } + + pub async fn start_with_options( + engine: Arc, + options: spark_connect_server::ServerOptions, + ) -> Self { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let (shutdown, stopped) = oneshot::channel(); let task = tokio::spawn(async move { - spark_connect_server::serve_with_engine(listener, engine, async { + spark_connect_server::serve_with_engine_and_options(listener, engine, options, async { let _ = stopped.await; }) .await From 8fcf31b758888253109ff4baa7a82aa1ff18739b Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 21:02:47 -0800 Subject: [PATCH 15/17] Clarify chunk limits and simplify Arrow encoding --- .github/workflows/server-frontend.yml | 15 ++- crates/spark-connect-server/README.md | 6 +- crates/spark-connect-server/src/lib.rs | 4 + crates/spark-connect-server/src/schema.rs | 6 +- crates/spark-connect-server/src/service.rs | 69 +++++------- crates/spark-connect-server/tests/protocol.rs | 102 ++++++++---------- .../tests/schema_consistency.rs | 11 +- 7 files changed, 99 insertions(+), 114 deletions(-) diff --git a/.github/workflows/server-frontend.yml b/.github/workflows/server-frontend.yml index 6b77267f..85cc57e9 100644 --- a/.github/workflows/server-frontend.yml +++ b/.github/workflows/server-frontend.yml @@ -22,7 +22,7 @@ name: Server frontend CI on: pull_request: branches: [master, "branch-*"] - paths: &frontend-paths + paths: - ".github/workflows/server-frontend.yml" - "Cargo.toml" - "Cargo.lock" @@ -32,7 +32,14 @@ on: - "crates/spark-connect-proto/**" push: branches: [master, "branch-*"] - paths: *frontend-paths + paths: + - ".github/workflows/server-frontend.yml" + - "Cargo.toml" + - "Cargo.lock" + - ".cargo/**" + - "rust-toolchain*" + - "crates/spark-connect-server/**" + - "crates/spark-connect-proto/**" workflow_dispatch: permissions: @@ -89,5 +96,5 @@ jobs: - name: Check protocol feature combinations run: | - cargo check --locked -p apache-spark-connect-proto --no-default-features - cargo check --locked -p apache-spark-connect-proto --no-default-features --features server + cargo check --locked -p apache-spark-connect-proto + cargo check --locked -p apache-spark-connect-proto --features server diff --git a/crates/spark-connect-server/README.md b/crates/spark-connect-server/README.md index 3b4ac6db..ce2a327b 100644 --- a/crates/spark-connect-server/README.md +++ b/crates/spark-connect-server/README.md @@ -79,8 +79,10 @@ preferred chunk payload size between 1 KiB and the configured maximum is accepte otherwise the maximum is used. `ServerOptions.max_arrow_chunk_size` defaults to 120,795,955 bytes (90% of 128 MiB), matching the JVM server's default chunk payload limit. It limits IPC payload bytes, excluding the protobuf envelope. Setting it -to zero disables chunking even when the client opts in. Clients must allow room -for the envelope in their gRPC receive limits. +to zero disables chunking even when the client opts in. An opted-in client must +accept the configured maximum plus its protobuf envelope, or request a smaller +chunk size. A 128 MiB receive limit accommodates the default maximum; raw tonic's +4 MiB default does not accommodate chunks above 4 MiB. Without client opt-in, or when chunking is disabled, each IPC batch is sent whole. Engines remain responsible for bounding batch production, including fitting the diff --git a/crates/spark-connect-server/src/lib.rs b/crates/spark-connect-server/src/lib.rs index f42f1895..195d5842 100644 --- a/crates/spark-connect-server/src/lib.rs +++ b/crates/spark-connect-server/src/lib.rs @@ -27,10 +27,14 @@ pub mod grpc { /// Transport options for the experimental frontend. #[derive(Clone, Debug)] +#[non_exhaustive] pub struct ServerOptions { /// Maximum Arrow chunk payload in bytes, excluding the protobuf envelope. /// Defaults to 90% of 128 MiB, as in Spark's JVM server. A client can request /// a smaller size. Zero disables chunking, even when the client opts in. + /// Clients must accept this payload plus the protobuf envelope: a 128 MiB + /// receive limit accommodates the default; tonic's 4 MiB default does not. + /// Alternatively, clients can request a smaller chunk size. /// Whole batches sent without chunking are not bounded by this option. pub max_arrow_chunk_size: usize, } diff --git a/crates/spark-connect-server/src/schema.rs b/crates/spark-connect-server/src/schema.rs index 76ab1b15..daddedbb 100644 --- a/crates/spark-connect-server/src/schema.rs +++ b/crates/spark-connect-server/src/schema.rs @@ -24,7 +24,7 @@ use prost::Message; use spark_connect_proto as proto; use tonic::Status; -pub(crate) const MAX_RESULT_BYTES: usize = 3 * 1024 * 1024; +pub(crate) const MAX_METADATA_RESPONSE_BYTES: usize = 3 * 1024 * 1024; pub(crate) fn result_schema(schema: &Schema) -> Result { let schema = spark_schema(schema)?; @@ -33,9 +33,9 @@ pub(crate) fn result_schema(schema: &Schema) -> Result } pub(crate) fn validate_schema_size(schema: &proto::DataType) -> Result<(), Status> { - if schema.encoded_len() > MAX_RESULT_BYTES { + if schema.encoded_len() > MAX_METADATA_RESPONSE_BYTES { return Err(Status::resource_exhausted( - "Spark schema exceeds the 3 MiB result limit", + "Spark schema exceeds the 3 MiB schema/command response limit", )); } Ok(()) diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index f2191251..93a97017 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -20,7 +20,7 @@ use std::panic::AssertUnwindSafe; use std::pin::Pin; use std::sync::{Arc, Mutex}; -use arrow::datatypes::Schema; +use arrow::datatypes::{Schema, SchemaRef}; use arrow::ipc::{reader::StreamReader, writer::StreamWriter}; use arrow::record_batch::RecordBatch; use futures::{FutureExt, Stream, StreamExt}; @@ -33,11 +33,10 @@ use uuid::Uuid; use crate::engine::{Engine, EngineSession, PreparedOperation, ResultBatch, SessionContext}; use crate::grpc::spark_connect_service_server::SparkConnectService; -use crate::schema::{self, result_schema}; +use crate::schema::{self, result_schema, MAX_METADATA_RESPONSE_BYTES}; type Output = Pin> + Send>>; type SessionKey = (String, String); -const MAX_RESULT_BYTES: usize = crate::schema::MAX_RESULT_BYTES; const MAX_RELEASED_SESSIONS: usize = 4096; const RELEASE_WAIT: std::time::Duration = std::time::Duration::from_secs(2); @@ -290,9 +289,9 @@ fn response( fn checked_response( message: proto::ExecutePlanResponse, ) -> Result { - if message.encoded_len() > MAX_RESULT_BYTES { + if message.encoded_len() > MAX_METADATA_RESPONSE_BYTES { return Err(Status::resource_exhausted( - "ExecutePlan response exceeds the 3 MiB result limit", + "Schema/command response exceeds the 3 MiB limit", )); } Ok(message) @@ -309,7 +308,7 @@ fn complete(id: &str, session: &Session, operation: &str) -> proto::ExecutePlanR ) } -fn encode_batch(batch: &RecordBatch) -> Result, Status> { +fn encode_batch(batch: &RecordBatch) -> Result<(Vec, SchemaRef), Status> { use arrow::datatypes::DataType; use arrow::record_batch::RecordBatchOptions; @@ -373,7 +372,7 @@ fn encode_batch(batch: &RecordBatch) -> Result, Status> { .finish() .map_err(|e| Status::internal(e.to_string()))?; } - Ok(bytes) + Ok((bytes, batch.schema())) } // PySpark assembles one table from all IPC batches. Its physical types and @@ -396,15 +395,6 @@ fn check_output_schema(schema: &Schema, expected: &mut Option) -> Result } } -fn checked_arrow_row_count(rows: usize, offset: i64) -> Result { - let rows = i64::try_from(rows) - .map_err(|_| Status::resource_exhausted("result batch row count exceeds i64"))?; - offset - .checked_add(rows) - .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; - Ok(rows) -} - // Keep backend ownership until the batch is dropped, including on encoder panic. struct EncodingBatch { batch: RecordBatch, @@ -412,7 +402,7 @@ struct EncodingBatch { } impl EncodingBatch { - fn encode(self) -> Result, Status> { + fn encode(self) -> Result<(Vec, SchemaRef), Status> { encode_batch(&self.batch) } } @@ -423,22 +413,23 @@ fn arrow_chunk_size( ) -> Option { use proto::execute_plan_request::request_option::RequestOption; - let mut options = options + let mut chunking = options .iter() .filter_map(|option| match &option.request_option { Some(RequestOption::ResultChunkingOptions(options)) => Some(options), _ => None, }); if maximum == 0 - || !options + || !chunking .clone() .any(|options| options.allow_arrow_batch_chunking) { return None; } Some( - options + chunking .find_map(|options| options.preferred_arrow_chunk_size) + // Validate i64 directly; Scala's toInt can wrap an invalid preference. .and_then(|bytes| usize::try_from(bytes).ok()) .filter(|bytes| (1024..=maximum).contains(bytes)) .unwrap_or(maximum), @@ -458,7 +449,8 @@ async fn send_batch( ) -> Result { // Reject unrepresentable counts/offsets before encoding or sending any chunk. let row_count = match &batch { - ResultBatch::Arrow(batch) => checked_arrow_row_count(batch.num_rows(), *offset)?, + ResultBatch::Arrow(batch) => i64::try_from(batch.num_rows()) + .map_err(|_| Status::resource_exhausted("result batch row count exceeds i64"))?, ResultBatch::Ipc { row_count, .. } if *row_count < 0 => { return Err(Status::internal("negative engine IPC row count")); } @@ -475,12 +467,9 @@ async fn send_batch( batch, _guard: guard.clone(), }; - let data = tokio::task::spawn_blocking(move || encoding.encode()) + let (data, schema) = tokio::task::spawn_blocking(move || encoding.encode()) .await .map_err(|_| Status::internal("Arrow result encoding task failed"))??; - let schema = StreamReader::try_new(data.as_slice(), None) - .map_err(|e| Status::internal(format!("invalid output IPC: {e}")))? - .schema(); check_output_schema(schema.as_ref(), output_schema)?; data.into() } @@ -584,11 +573,7 @@ impl SparkConnectService for ConnectService { checked_response(message) }) .collect::, Status>>()?; - messages.push(checked_response(complete( - &req.session_id, - &session, - &operation, - ))?); + messages.push(complete(&req.session_id, &session, &operation)); return Ok(Response::new(Box::pin(tokio_stream::iter( messages.into_iter().map(Ok), )))); @@ -672,11 +657,7 @@ impl SparkConnectService for ConnectService { } } let _ = sender - .send(Ok(checked_response(complete( - &req.session_id, - &session, - &operation, - ))?)) + .send(Ok(complete(&req.session_id, &session, &operation))) .await; Ok::<_, Status>(()) }; @@ -886,6 +867,8 @@ impl SparkConnectService for ConnectService { mod tests { use super::*; + const CHUNK_SIZE: usize = 1024 * 1024; + struct RegistryEngine; struct RegistrySession; @@ -945,13 +928,13 @@ mod tests { let schema = proto::DataType { kind: Some(proto::data_type::Kind::Struct(proto::data_type::Struct { fields: vec![proto::data_type::StructField { - name: "x".repeat(MAX_RESULT_BYTES - 64), + name: "x".repeat(MAX_METADATA_RESPONSE_BYTES - 64), ..Default::default() }], ..Default::default() })), }; - assert!(schema.encoded_len() < MAX_RESULT_BYTES); + assert!(schema.encoded_len() < MAX_METADATA_RESPONSE_BYTES); let mut header = response( &Uuid::new_v4().to_string(), &session, @@ -999,7 +982,7 @@ mod tests { &mut offset, &mut output_schema, &guard, - Some(MAX_RESULT_BYTES), + Some(CHUNK_SIZE), ) .await .unwrap_err(); @@ -1047,13 +1030,13 @@ mod tests { &mut offset, &mut output_schema, &guard, - Some(MAX_RESULT_BYTES), + Some(CHUNK_SIZE), ) .await .unwrap(); let batch = if ipc { - let data = encode_batch(&oversized).unwrap(); - assert!(data.len() > MAX_RESULT_BYTES); + let (data, _) = encode_batch(&oversized).unwrap(); + assert!(data.len() > MAX_METADATA_RESPONSE_BYTES); ResultBatch::Ipc { data, row_count: 2 } } else { oversized.clone().into() @@ -1067,7 +1050,7 @@ mod tests { &mut offset, &mut output_schema, &guard, - Some(MAX_RESULT_BYTES), + Some(CHUNK_SIZE), ) .await .unwrap_err(); @@ -1093,7 +1076,7 @@ mod tests { &mut offset, &mut output_schema, &guard, - Some(MAX_RESULT_BYTES), + Some(CHUNK_SIZE), ) .await .unwrap(); diff --git a/crates/spark-connect-server/tests/protocol.rs b/crates/spark-connect-server/tests/protocol.rs index 14c637d2..5cf1e886 100644 --- a/crates/spark-connect-server/tests/protocol.rs +++ b/crates/spark-connect-server/tests/protocol.rs @@ -297,13 +297,9 @@ async fn preferred_chunk_size_uses_the_protocol_range_and_splits_inside_a_large_ ) .unwrap(); let engine = FixtureEngine::default().with_batches(42, batch.schema(), vec![batch.clone()]); - let mut server = Server::start_with_options( - Arc::new(engine), - spark_connect_server::ServerOptions { - max_arrow_chunk_size: 3 * 1024 * 1024, - }, - ) - .await; + let mut options = spark_connect_server::ServerOptions::default(); + options.max_arrow_chunk_size = 3 * 1024 * 1024; + let mut server = Server::start_with_options(Arc::new(engine), options).await; for preferred in [ Some(1024), Some(65_537), @@ -444,7 +440,7 @@ async fn deeply_nested_requests_follow_the_configured_protobuf_decoder_limit() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn view_arrays_are_materialized_once_before_byte_chunking_including_struct_children() { +async fn view_arrays_are_materialized_before_byte_chunking_including_struct_children() { use arrow::array::{BinaryArray, BinaryViewArray, StringArray, StringViewArray, StructArray}; use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatch; @@ -461,7 +457,7 @@ async fn view_arrays_are_materialized_once_before_byte_chunking_including_struct let binaries = BinaryViewArray::from_iter_values((0..4096).map(binary)); let nested_strings = StringViewArray::from_iter_values((0..4096).map(nested_text)); // A one-row slice of these view arrays still references backing buffers - // larger than the response limit, although each individual value is 1 KiB. + // larger than the chunk preference, although each individual value is 1 KiB. for size in [ strings .data_buffers() @@ -507,50 +503,45 @@ async fn view_arrays_are_materialized_once_before_byte_chunking_including_struct let output = assembled_batches(&responses, true); assert_eq!(output.len(), 1); assert!(output[0].num_chunks_in_batch.unwrap() > 1); - let (mut rows, mut batches) = (0, 0); - for batch in output { - assert_eq!(batch.start_offset, Some(rows)); - let records = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); - let mut decoded = 0; - for record in records { - let record = record.unwrap(); - let ids = record - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(record.column(1).data_type(), &DataType::LargeUtf8); - assert_eq!(record.column(2).data_type(), &DataType::LargeBinary); - let strings = arrow::compute::cast(record.column(1), &DataType::Utf8).unwrap(); - let strings = strings.as_any().downcast_ref::().unwrap(); - let binaries = arrow::compute::cast(record.column(2), &DataType::Binary).unwrap(); - let binaries = binaries.as_any().downcast_ref::().unwrap(); - let nested = record - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(nested.column(0).data_type(), &DataType::LargeUtf8); - let nested_strings = arrow::compute::cast(nested.column(0), &DataType::Utf8).unwrap(); - let nested_strings = nested_strings - .as_any() - .downcast_ref::() - .unwrap(); - for index in 0..record.num_rows() { - let id = rows + decoded + index as i64; - assert_eq!(ids.value(index), id); - assert_eq!(strings.value(index), text(id)); - assert_eq!(binaries.value(index), binary(id)); - assert_eq!(nested_strings.value(index), nested_text(id)); - } - decoded += record.num_rows() as i64; + let batch = &output[0]; + assert_eq!(batch.start_offset, Some(0)); + let records = StreamReader::try_new(Cursor::new(&batch.data), None).unwrap(); + let mut rows = 0; + for record in records { + let record = record.unwrap(); + let ids = record + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(record.column(1).data_type(), &DataType::LargeUtf8); + assert_eq!(record.column(2).data_type(), &DataType::LargeBinary); + let strings = arrow::compute::cast(record.column(1), &DataType::Utf8).unwrap(); + let strings = strings.as_any().downcast_ref::().unwrap(); + let binaries = arrow::compute::cast(record.column(2), &DataType::Binary).unwrap(); + let binaries = binaries.as_any().downcast_ref::().unwrap(); + let nested = record + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(nested.column(0).data_type(), &DataType::LargeUtf8); + let nested_strings = arrow::compute::cast(nested.column(0), &DataType::Utf8).unwrap(); + let nested_strings = nested_strings + .as_any() + .downcast_ref::() + .unwrap(); + for index in 0..record.num_rows() { + let id = rows + index as i64; + assert_eq!(ids.value(index), id); + assert_eq!(strings.value(index), text(id)); + assert_eq!(binaries.value(index), binary(id)); + assert_eq!(nested_strings.value(index), nested_text(id)); } - assert_eq!(decoded, batch.row_count); - rows += decoded; - batches += 1; + rows += record.num_rows() as i64; } + assert_eq!(rows, batch.row_count); assert_eq!(rows, 4096); - assert_eq!(batches, 1); server.stop().await; } @@ -614,6 +605,7 @@ async fn collect_small_slices(backing: RecordBatch) -> Vec { let output = collect_result(&mut server, range(&Uuid::new_v4().to_string(), 42)).await; let (mut batches, mut rows) = (Vec::new(), 0); for message in output { + // A small slice must not serialize the much larger retained backing allocation. assert!(message.encoded_len() <= 3 * 1024 * 1024); if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = message.response_type @@ -878,13 +870,9 @@ async fn configured_maximum_bounds_payloads_and_zero_disables_chunking() { batch.schema(), vec![Batch::Ipc(bytes.clone(), 5000)], ); - let mut server = Server::start_with_options( - Arc::new(engine), - spark_connect_server::ServerOptions { - max_arrow_chunk_size: maximum, - }, - ) - .await; + let mut options = spark_connect_server::ServerOptions::default(); + options.max_arrow_chunk_size = maximum; + let mut server = Server::start_with_options(Arc::new(engine), options).await; for preferred in [None, Some(2048), Some(4096), Some(4097)] { let mut request = range(&Uuid::new_v4().to_string(), 42); request.request_options = vec![chunking(true, preferred)]; diff --git a/crates/spark-connect-server/tests/schema_consistency.rs b/crates/spark-connect-server/tests/schema_consistency.rs index 82ed77bd..11af295a 100644 --- a/crates/spark-connect-server/tests/schema_consistency.rs +++ b/crates/spark-connect-server/tests/schema_consistency.rs @@ -47,7 +47,7 @@ use arrow_support::{assembled_batches, chunking}; use server::Server; use support::{Batch, FixtureEngine}; -const RESULT_LIMIT: usize = 3 * 1024 * 1024; +const CHUNK_PREFERENCE: usize = 3 * 1024 * 1024; const LARGE_ROWS: usize = 4096; struct BatchCase { @@ -68,7 +68,7 @@ async fn execute(engine: BatchCase) -> (Vec, Option< let engine = FixtureEngine::default().with_results(1, engine.schema, engine.batches); let mut server = Server::start_with_engine(Arc::new(engine)).await; let mut request = requests::request(&Uuid::new_v4().to_string(), 1); - request.request_options = vec![chunking(true, Some(RESULT_LIMIT as i64))]; + request.request_options = vec![chunking(true, Some(CHUNK_PREFERENCE as i64))]; let mut stream = server .client .execute_plan(request) @@ -83,10 +83,11 @@ async fn execute(engine: BatchCase) -> (Vec, Option< if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = &response.response_type { - assert!(batch.data.len() <= RESULT_LIMIT); + assert!(batch.data.len() <= CHUNK_PREFERENCE); assert!(response.encoded_len() <= batch.data.len() + 512); } else { - assert!(response.encoded_len() <= RESULT_LIMIT); + // Schema and command envelopes retain their independent 3 MiB cap. + assert!(response.encoded_len() <= 3 * 1024 * 1024); } responses.push(response); } @@ -243,7 +244,7 @@ async fn view_ipc_remains_unchanged_for_small_and_oversized_batches() { for rows in [2, LARGE_ROWS] { let batch = views(rows); let data = ipc(&batch); - assert_eq!(data.len() > RESULT_LIMIT, rows == LARGE_ROWS); + assert_eq!(data.len() > CHUNK_PREFERENCE, rows == LARGE_ROWS); let (responses, status) = execute(BatchCase { schema: batch.schema(), batches: vec![Batch::Ipc(data.clone(), rows as i64)], From c07e3c8e4c3283b0569843620ddb3e2a7d433a21 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 21:03:19 -0800 Subject: [PATCH 16/17] Keep overflow fixture independent of metadata limits --- crates/spark-connect-server/src/service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index 93a97017..71d309db 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -1036,7 +1036,7 @@ mod tests { .unwrap(); let batch = if ipc { let (data, _) = encode_batch(&oversized).unwrap(); - assert!(data.len() > MAX_METADATA_RESPONSE_BYTES); + assert!(data.len() > CHUNK_SIZE); ResultBatch::Ipc { data, row_count: 2 } } else { oversized.clone().into() From 56640d306ec510252d962b77396f7f64ac7484ae Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 21:14:08 -0800 Subject: [PATCH 17/17] Clarify frontend configuration and validation documentation --- crates/spark-connect-server/src/lib.rs | 1 + crates/spark-connect-server/src/service.rs | 2 +- crates/spark-connect-server/tests/protocol.rs | 4 ++-- dev/design/ci.md | 3 ++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/spark-connect-server/src/lib.rs b/crates/spark-connect-server/src/lib.rs index 195d5842..195c0bc6 100644 --- a/crates/spark-connect-server/src/lib.rs +++ b/crates/spark-connect-server/src/lib.rs @@ -26,6 +26,7 @@ pub mod grpc { } /// Transport options for the experimental frontend. +/// Start from [`ServerOptions::default()`] and assign the desired fields. #[derive(Clone, Debug)] #[non_exhaustive] pub struct ServerOptions { diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index 71d309db..00c81c3f 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -291,7 +291,7 @@ fn checked_response( ) -> Result { if message.encoded_len() > MAX_METADATA_RESPONSE_BYTES { return Err(Status::resource_exhausted( - "Schema/command response exceeds the 3 MiB limit", + "schema/command response exceeds the 3 MiB limit", )); } Ok(message) diff --git a/crates/spark-connect-server/tests/protocol.rs b/crates/spark-connect-server/tests/protocol.rs index 5cf1e886..d5f9bfb5 100644 --- a/crates/spark-connect-server/tests/protocol.rs +++ b/crates/spark-connect-server/tests/protocol.rs @@ -456,8 +456,8 @@ async fn view_arrays_are_materialized_before_byte_chunking_including_struct_chil let strings = StringViewArray::from_iter_values((0..4096).map(text)); let binaries = BinaryViewArray::from_iter_values((0..4096).map(binary)); let nested_strings = StringViewArray::from_iter_values((0..4096).map(nested_text)); - // A one-row slice of these view arrays still references backing buffers - // larger than the chunk preference, although each individual value is 1 KiB. + // These view columns exceed the chunk preference, so their encoded batch + // spans several byte chunks. for size in [ strings .data_buffers() diff --git a/dev/design/ci.md b/dev/design/ci.md index d30dcf3b..531267cb 100644 --- a/dev/design/ci.md +++ b/dev/design/ci.md @@ -75,7 +75,8 @@ This job should always pass; failures indicate incomplete implementation or test This focused workflow runs on frontend, protocol, workspace manifest, lockfile, Cargo configuration, toolchain and workflow changes. Each stable Rust and Rust -1.88.0 job has a 45-minute limit and runs `cargo test --locked -p spark-connect-server`. +1.88.0 job has a 45-minute limit, runs `cargo test --locked -p spark-connect-server`, +and checks the protocol crate with its `server` feature both off and on. The existing workspace lint job covers the frontend with its pinned Rust toolchain; test jobs do not promote warnings from a floating or older compiler to errors. Ubuntu 24.04 supplies a protobuf compiler with