diff --git a/.github/workflows/server-frontend.yml b/.github/workflows/server-frontend.yml new file mode 100644 index 00000000..85cc57e9 --- /dev/null +++ b/.github/workflows/server-frontend.yml @@ -0,0 +1,100 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Server frontend CI + +on: + pull_request: + branches: [master, "branch-*"] + paths: + - ".github/workflows/server-frontend.yml" + - "Cargo.toml" + - "Cargo.lock" + - ".cargo/**" + - "rust-toolchain*" + - "crates/spark-connect-server/**" + - "crates/spark-connect-proto/**" + push: + branches: [master, "branch-*"] + paths: + - ".github/workflows/server-frontend.yml" + - "Cargo.toml" + - "Cargo.lock" + - ".cargo/**" + - "rust-toolchain*" + - "crates/spark-connect-server/**" + - "crates/spark-connect-proto/**" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CARGO_BUILD_JOBS: "2" + CARGO_PROFILE_TEST_DEBUG: "0" + # Strict linting uses the pinned toolchain in lint.yml. + RUSTFLAGS: "" + +jobs: + frontend: + name: Frontend (Rust ${{ matrix.rust-version }}) + runs-on: ubuntu-24.04 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + rust-version: [stable, "1.88.0"] + steps: + - uses: actions/checkout@v4 + + - name: Install Rust toolchain + # Use runner-provided rustup to keep actions limited to actions/*. + run: | + rustup toolchain install ${{ matrix.rust-version }} --profile minimal + rustup default ${{ matrix.rust-version }} + + - name: Cache cargo and target + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-${{ runner.arch }}-cargo-server-frontend-${{ matrix.rust-version }}-${{ hashFiles('Cargo.lock') }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-cargo-server-frontend-${{ matrix.rust-version }}- + + - name: Install protobuf compiler + # Ubuntu 24.04 provides protoc with proto3 optional support (3.15+). + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + protoc --version + + - name: Test frontend + # Protocol and injected-engine tests use local fixtures without an execution backend. + run: cargo test --locked -p spark-connect-server + + - name: Check protocol feature combinations + run: | + cargo check --locked -p apache-spark-connect-proto + cargo check --locked -p apache-spark-connect-proto --features server diff --git a/Cargo.lock b/Cargo.lock index 40cf5dab..41a9b1b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4048,6 +4048,21 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spark-connect-server" +version = "0.1.0" +dependencies = [ + "apache-spark-connect-proto", + "arrow", + "futures", + "h2", + "prost", + "tokio", + "tokio-stream", + "tonic", + "uuid", +] + [[package]] name = "sqlparser" version = "0.60.0" diff --git a/Cargo.toml b/Cargo.toml index fb6261a1..189b8bab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ members = [ "crates/spark-connect-build", "crates/pyspark-rs", "examples", + "crates/spark-connect-server", ] exclude = [ "examples/wasm-udfs", @@ -44,6 +45,8 @@ exclude = [ # (so `cargo test --workspace` stays green without it). Build them explicitly with # `cargo build --manifest-path examples/wasm-udfs/Cargo.toml` (etc) in CI or when # the target is installed locally. +# The experimental protocol frontend is opt-in; build or test it with +# `-p spark-connect-server`. It is included in workspace lint checks. default-members = [ "crates/spark-connect-proto", "crates/spark-connect-core", @@ -87,11 +90,13 @@ should_implement_trait = "allow" tonic = { version = "0.14", default-features = false, features = ["channel", "codegen"] } tonic-prost = "0.14" tonic-prost-build = "0.14" +h2 = "0.4" prost = "0.14" prost-types = "0.14" # async runtime tokio = { version = "1", features = ["rt-multi-thread", "sync", "macros", "time"] } +tokio-stream = "0.1" futures = "0.3" # Arrow (result decoding). diff --git a/README.md b/README.md index d75af986..fcc755db 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,17 @@ fn main() -> Result<(), Box> { See the [WASM UDF guide](https://apache.github.io/spark-connect-rust/udfs/) for the full setup, SQL registration, supported types, and more. +## Experimental server frontend + +The unpublished [`spark-connect-server`](crates/spark-connect-server/README.md) +library provides a Rust Spark Connect protocol frontend. It manages protocol sessions, +operations and Arrow IPC responses, delegating planning and execution to an +application-provided `Engine`. The library includes no execution backend or standalone +server executable; its protocol tests use injected engine fixtures. + +The frontend is intended for local, trusted experiments. Read its +[security model](SECURITY.md#experimental-server-frontend) before embedding it. + ## Contributing Issues are tracked in ASF JIRA under diff --git a/SECURITY.md b/SECURITY.md index 80e53253..d80df8b4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -25,3 +25,22 @@ material, TLS validation) inherit from that model. Automated security scanning agents should consult that document for the project's in-scope / out-of-scope declarations before reporting issues. + +## Experimental server frontend + +The optional `spark-connect-server` library is an experimental protocol +frontend for local, trusted workloads. It does not provide authentication, +authorization or TLS. Session IDs, operation IDs and client-provided user +context identify protocol state; they do not establish a caller's identity +or access rights. + +Bind its listener only to loopback. The embedding application supplies the +listener and the `Engine` implementation; the library does not enforce the +listener's network scope. Any process that can reach it can submit requests +to that engine. Protocol limits and session isolation are resource and +correctness guards, not a security boundary between mutually untrusted users. + +This frontend does not include an execution backend or a standalone server. +Its injected-engine tests establish frontend protocol behavior, not safe +exposure of an execution engine to untrusted clients. Apache Spark's operator +security guidance remains relevant to any application that connects an engine. diff --git a/crates/spark-connect-core/Cargo.toml b/crates/spark-connect-core/Cargo.toml index a34fb898..9d4e6c9d 100644 --- a/crates/spark-connect-core/Cargo.toml +++ b/crates/spark-connect-core/Cargo.toml @@ -32,7 +32,7 @@ prost-types.workspace = true crc32fast = "1" sha2 = "0.10" # Transport errors that tonic keeps as a status's source, for `RetryPolicy::can_retry`. -h2 = "0.4" +h2.workspace = true hyper = "1" # The Basic credential and the percent-decoded userinfo of a proxy URI, already built for tonic. base64 = "0.22" diff --git a/crates/spark-connect-proto/Cargo.toml b/crates/spark-connect-proto/Cargo.toml index efc4c833..330fdfa0 100644 --- a/crates/spark-connect-proto/Cargo.toml +++ b/crates/spark-connect-proto/Cargo.toml @@ -15,6 +15,9 @@ workspace = true [lib] name = "spark_connect_proto" +[features] +server = [] + [dependencies] tonic.workspace = true tonic-prost.workspace = true diff --git a/crates/spark-connect-proto/build.rs b/crates/spark-connect-proto/build.rs index 048ae15f..dd85bafe 100644 --- a/crates/spark-connect-proto/build.rs +++ b/crates/spark-connect-proto/build.rs @@ -35,8 +35,10 @@ fn main() -> Result<(), Box> { let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").ok_or("OUT_DIR is not set")?); let full_set = out_dir.join("spark_connect_descriptor_set_full.bin"); tonic_prost_build::configure() - .build_server(false) + .build_server(cfg!(feature = "server")) .build_client(true) + // async_trait repeats Future's must_use on generated server methods. + .trait_attribute("SparkConnectService", "#[allow(clippy::double_must_use)]") .bytes(".") .file_descriptor_set_path(&full_set) .compile_protos(&protos, &[proto_root])?; diff --git a/crates/spark-connect-proto/src/lib.rs b/crates/spark-connect-proto/src/lib.rs index 35689ba4..58eb65a9 100644 --- a/crates/spark-connect-proto/src/lib.rs +++ b/crates/spark-connect-proto/src/lib.rs @@ -3,6 +3,8 @@ //! All messages live in the `spark.connect` protobuf package and are re-exported //! here at the crate root. The generated gRPC client is //! [`spark_connect_service_client::SparkConnectServiceClient`]. +//! Enable the optional `server` feature for `spark_connect_service_server` +//! bindings. Embeddings configure tonic's transport server features separately. //! //! Baseline protocol version: see `proto/PROTO_VERSION.txt` / `proto/SPARK_SHA.txt`. diff --git a/crates/spark-connect-server/Cargo.toml b/crates/spark-connect-server/Cargo.toml new file mode 100644 index 00000000..96adcd4a --- /dev/null +++ b/crates/spark-connect-server/Cargo.toml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "spark-connect-server" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false +description = "Experimental Spark Connect server frontend with an injectable execution engine" + +[lints] +workspace = true + +[dependencies] +spark-connect-proto = { package = "apache-spark-connect-proto", path = "../spark-connect-proto", features = ["server"] } +arrow.workspace = true +futures.workspace = true +prost.workspace = true +tokio = { workspace = true, features = ["net"] } +tokio-stream.workspace = true +tonic = { workspace = true, features = ["server", "router"] } +uuid.workspace = true + +[dev-dependencies] +h2.workspace = true diff --git a/crates/spark-connect-server/README.md b/crates/spark-connect-server/README.md new file mode 100644 index 00000000..ce2a327b --- /dev/null +++ b/crates/spark-connect-server/README.md @@ -0,0 +1,145 @@ + + +# Experimental Spark Connect server frontend + +This unpublished library serves a subset of the Spark Connect protocol using the +workspace's `apache-spark-connect-proto` messages and its optional `server` +bindings. The caller supplies an +`Engine` implementation and a bound Tokio TCP listener to `serve_with_engine`. +Use `serve_with_engine_and_options` with `ServerOptions` to configure Arrow +chunking. +No execution engine or standalone server executable is included. + +Rust owns protocol sessions, operation IDs, response framing and result-stream +lifetimes. Each protocol session receives an `EngineSession` that owns its +planning, catalog and configuration state. Its `prepare` method returns a +`PreparedQuery` or command response payloads; the frontend adds protocol IDs and +`ResultComplete`. The engine decides which plans, commands and metadata requests +it can execute and reports unsupported inputs explicitly. + +## Protocol subset + +The frontend routes `ExecutePlan`, `AnalyzePlan` and `Config` to the engine. It +implements `ReleaseExecute` and `ReleaseSession`, including operation cancellation +through `ReleaseAll` and session recreation when `allow_reconnect` permits it. +Completed responses are not retained. Reattachment, artifact upload/status, +`Interrupt`, enriched error details, session cloning and status queries return +`UNIMPLEMENTED`. + +The frontend requires lowercase, hyphenated UUID session and operation IDs, +rejects duplicate active operation IDs within a session, and checks +client-observed server session IDs. +Client user IDs distinguish session keys but are not authenticated identities. +Protobuf nesting follows the selected Prost features; dependency feature +unification can disable its recursion limit, and the frontend adds no independent +nesting guard. + +## Arrow results + +A prepared query declares an Arrow schema before execution. The frontend converts +supported types to the Spark schema representation and checks each batch against +that declaration. Batches may use Spark-equivalent Arrow representations and +nullability different from the declaration. Within one result, emitted IPC +batches must share physical types, field names and nullability, so clients can +assemble them into one Arrow table. Metadata may differ between batches. +Declared field metadata and unsupported types fail explicitly. + +Engines may yield Arrow record batches or complete Arrow IPC streams containing +one batch and its actual nonnegative row count. The frontend encodes each record +batch once and forwards existing IPC bytes unchanged, including compressed IPC. +It reads the IPC schema for validation and trusts the producer's batch contents +and declared row count. Record-batch encoding materializes string/binary view +arrays, including nested children, to `LargeUtf8`/`LargeBinary`. Existing IPC +retains its physical types. The frontend reports `INTERNAL` before sending a +batch whose output schema differs from earlier batches; engines mixing record +batches with IPC must use consistent output representations. + +When the client sets `ResultChunkingOptions.allow_arrow_batch_chunking`, the +frontend divides serialized IPC bytes into chunks and sets `chunk_index` and +`num_chunks_in_batch`. Boundaries need not align with rows or Arrow buffers; the +client concatenates a batch's chunks before decoding or decompressing it. A +preferred chunk payload size between 1 KiB and the configured maximum is accepted; +otherwise the maximum is used. `ServerOptions.max_arrow_chunk_size` defaults to +120,795,955 bytes (90% of 128 MiB), matching the JVM server's default chunk payload +limit. It limits IPC payload bytes, excluding the protobuf envelope. Setting it +to zero disables chunking even when the client opts in. An opted-in client must +accept the configured maximum plus its protobuf envelope, or request a smaller +chunk size. A 128 MiB receive limit accommodates the default maximum; raw tonic's +4 MiB default does not accommodate chunks above 4 MiB. + +Without client opt-in, or when chunking is disabled, each IPC batch is sent whole. +Engines remain responsible for bounding batch production, including fitting the +client's gRPC receive limit when batches are sent whole. The frontend applies no +128 MiB batch-production cap. Schema and command response envelopes have a +separate 3 MiB limit and fail with `RESOURCE_EXHAUSTED` when oversized. Each batch +carries a cumulative row offset, and empty results still include an empty batch. + +## Lifetime and trust requirements + +Engine methods and streams must cancel their owned work when dropped. The +frontend cancels them when a client disconnects, an operation is released, a +session is released, server shutdown begins or the serving future is dropped. +Results pass through a bounded +channel, so unread responses eventually stop polling the engine. Blocking Arrow +encoding retains operation ownership until its worker finishes. IPC chunks share +their batch's backing allocation, which remains live while any chunk references +it. Larger batches and chunk limits can therefore increase retained memory even +though the response channel is bounded. + +`ReleaseAll` waits up to two seconds for teardown and reports `DEADLINE_EXCEEDED` +if it is still draining. Shutdown cancels sessions and waits up to two seconds +for transport drain; returning from `serve_with_engine` does not prove that an +engine's independently scheduled or blocking work has finished. The caller must +honor the engine's cleanup contract before releasing its resources. + +`ReleaseSession(allow_reconnect=true)` permits a new backend session with the +same protocol ID while the previous incarnation still drains. Engines must keep +the two incarnations' resources independent. Commands may commit effects before +frontend response validation fails, for example on an oversized payload; that +failure provides no rollback or safe-retry guarantee. + +With panic unwinding enabled, panics in `prepare`, `analyze`, `config` and query +execution are translated to `INTERNAL`. Synchronous session creation and query +schema access are outside that boundary and must not panic. Resource cleanup +must not panic either. + +This is an experiment for trusted local clients and trusted engine outputs. +Keep the supplied listener on loopback: this frontend supplies no authentication, +authorization or TLS. Requests can be up to 64 MiB. Active sessions and operations +have no admission limit or idle expiry; only released-session tombstones are +bounded, to the most recent 4096 distinct IDs. An evicted ID can create a new +session. These limits do not provide isolation or complete memory and CPU quotas. +Execution permissions and resource access are the engine's responsibility. + +## Build and test + +Use the workspace Rust toolchain and a `protoc` compiler with proto3 optional +support. From the repository root: + +```bash +cargo test --locked -p spark-connect-server +cargo clippy --locked -p spark-connect-server --all-targets --no-deps -- -D warnings +``` + +The tests run a real gRPC frontend against deterministic injected engines. They +cover wire framing, schema and IPC handling, backend error propagation, +backpressure, operation ownership, cancellation and session lifetime. Fixture +relations select predefined outputs; they do not implement Spark operators or +establish query semantics or Spark server compatibility. diff --git a/crates/spark-connect-server/src/engine/mod.rs b/crates/spark-connect-server/src/engine/mod.rs new file mode 100644 index 00000000..edfadfd4 --- /dev/null +++ b/crates/spark-connect-server/src/engine/mod.rs @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Native engine boundary for Spark Connect planning and execution. +//! +//! The frontend owns protocol IDs, operation lifetime, framing and backpressure. +//! Engines own their catalog, configuration and native plans. This Rust API is +//! an initial in-process contract for an injected execution engine. + +use std::pin::Pin; +use std::sync::Arc; + +use arrow::datatypes::SchemaRef; +use arrow::record_batch::RecordBatch; +use futures::Stream; +use spark_connect_proto as proto; +use tonic::Status; + +/// Convert an engine Arrow schema into the Spark Connect schema representation. +pub use crate::schema::spark_schema; + +/// Protocol context for a new backend session. +#[derive(Clone, Debug)] +pub struct SessionContext { + pub session_id: String, + /// Client-supplied metadata, not an authenticated identity. + pub user_context: Option, +} + +/// Factory for sessions bound to one native engine. +pub trait Engine: Send + Sync { + /// Allocate local session state without blocking, performing I/O or panicking. + /// Defer asynchronous initialization to the session methods. The frontend + /// serializes creation to ensure one backend session per active incarnation. + /// A reconnecting session release permits a replacement with the same + /// `session_id` while references to the previous incarnation are still + /// draining. The engine must keep those incarnations' resources independent. + fn create_session(&self, context: SessionContext) -> Result, Status>; +} + +/// Backend-owned catalog, configuration and planning state. +/// +/// Dropping a method future must cancel its work. Dropping the final session +/// reference releases its resources after in-flight operations have finished. +/// Resource cleanup must not panic. +/// With unwinding enabled, panics in these methods become `INTERNAL` errors; +/// the frontend keeps the session available for later requests. Implementations +/// must leave their state usable when unwinding. +// async_trait repeats Future's must_use annotation on generated trait methods. +#[allow(clippy::double_must_use)] +#[tonic::async_trait] +pub trait EngineSession: Send + Sync { + /// Prepare a query, or perform an eager protocol command. The frontend + /// reserves the operation ID before calling this method. + /// If cancelled, a command must leave its state change fully committed or + /// rolled back, with no partially applied changes. + async fn prepare(&self, operation: proto::plan::OpType) -> Result; + + async fn analyze( + &self, + input: proto::analyze_plan_request::Analyze, + ) -> Result; + + async fn config( + &self, + input: proto::config_request::operation::OpType, + ) -> Result; +} + +/// Prepared native query or command-specific response payloads. +pub enum PreparedOperation { + Query(Box), + /// The frontend adds envelope IDs and the final ResultComplete message. + /// Eager command effects may already be committed when later frontend + /// response validation fails, for example when a payload exceeds the size + /// limit. Such a failure does not roll back effects or make retries safe. + Command(Vec), +} + +/// Query execution starts after the frontend validates and sends its schema. +// async_trait repeats Future's must_use annotation on generated trait methods. +#[allow(clippy::double_must_use)] +#[tonic::async_trait] +pub trait PreparedQuery: Send { + /// The declared Spark schema. This synchronous accessor must not panic. + /// Batches must have the same field names and Spark-visible types; + /// equivalent Arrow representations are accepted. + /// Batch nullability and metadata may differ from this declaration, but + /// emitted IPC batches in one result must share physical types, field names + /// and nullability. Metadata may differ between batches. Record-batch encoding + /// materializes view arrays to large-offset types; existing IPC retains its + /// physical types. Mixing incompatible output representations fails before + /// the changed output schema is sent. Declared fields must not contain + /// metadata, which Spark schema conversion rejects. + fn schema(&self) -> SchemaRef; + + /// Dropping this future or its returned stream must cancel backend work; + /// implementations must not leave detached execution tasks running. + async fn execute(self: Box) -> Result; +} + +/// Engine output, optionally already serialized for Spark Connect. +/// +/// Engines must bound batch production. Optional byte chunking bounds network +/// payloads, not the complete batch's allocation. When chunking is disabled or +/// the client does not opt in, each batch must fit the client's gRPC receive limit. +pub enum ResultBatch { + /// Encoded once as a complete IPC stream before optional byte chunking. + Arrow(RecordBatch), + /// A complete IPC stream containing its schema, exactly one record batch, + /// and EOS. The producer must supply that batch's actual nonnegative row count. + /// The frontend validates the schema and trusts the producer's batch contents + /// and declared row count. IPC bytes, including compression, are preserved. + /// Clients that opt in receive byte chunks for reassembly before decoding; + /// otherwise the whole stream is sent, so producers must respect the client's + /// gRPC receive limit when selecting batch sizes. + Ipc { data: Vec, row_count: i64 }, +} + +impl From for ResultBatch { + fn from(batch: RecordBatch) -> Self { + Self::Arrow(batch) + } +} + +/// Pull-based results: the frontend polls only as client backpressure allows. +pub type BatchStream = Pin> + Send>>; + +/// Engine configuration output without frontend-owned protocol IDs. +#[derive(Default)] +pub struct ConfigResult { + pub pairs: Vec, + pub warnings: Vec, +} diff --git a/crates/spark-connect-server/src/lib.rs b/crates/spark-connect-server/src/lib.rs new file mode 100644 index 00000000..195c0bc6 --- /dev/null +++ b/crates/spark-connect-server/src/lib.rs @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod engine; +mod schema; +mod service; + +/// Generated server bindings for custom embedding. This experimental API follows +/// the protocol schema and may change with the protocol. +pub mod grpc { + pub use spark_connect_proto::spark_connect_service_server; +} + +/// Transport options for the experimental frontend. +/// Start from [`ServerOptions::default()`] and assign the desired fields. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct ServerOptions { + /// Maximum Arrow chunk payload in bytes, excluding the protobuf envelope. + /// Defaults to 90% of 128 MiB, as in Spark's JVM server. A client can request + /// a smaller size. Zero disables chunking, even when the client opts in. + /// Clients must accept this payload plus the protobuf envelope: a 128 MiB + /// receive limit accommodates the default; tonic's 4 MiB default does not. + /// Alternatively, clients can request a smaller chunk size. + /// Whole batches sent without chunking are not bounded by this option. + pub max_arrow_chunk_size: usize, +} + +impl Default for ServerOptions { + fn default() -> Self { + Self { + max_arrow_chunk_size: 128 * 1024 * 1024 * 9 / 10, + } + } +} + +struct CancelOnDrop(std::sync::Arc); + +impl Drop for CancelOnDrop { + fn drop(&mut self) { + self.0.cancel_all(); + } +} + +fn tcp_incoming(listener: tokio::net::TcpListener) -> tonic::transport::server::TcpIncoming { + // Builder socket options do not apply to serve_with_incoming_shutdown. + tonic::transport::server::TcpIncoming::from(listener).with_nodelay(Some(true)) +} + +/// Serve with an injected execution engine and a fresh session registry. +/// +/// The frontend binds each protocol session to the engine session created for it. +/// The caller owns the engine and signals shutdown before releasing its resources. +/// Dropping this future also cancels registered engine work, without waiting for +/// the bounded transport drain used when shutdown is signalled. +pub async fn serve_with_engine( + listener: tokio::net::TcpListener, + engine: std::sync::Arc, + shutdown: impl std::future::Future + Send + 'static, +) -> Result<(), tonic::transport::Error> { + serve_with_engine_and_options(listener, engine, ServerOptions::default(), shutdown).await +} + +/// Serve with an injected engine and explicit transport options. +/// +/// Session lifetime and shutdown follow [`serve_with_engine`]. Producers should +/// bound engine batches independently of this transport chunk size. Clients must +/// allow protobuf overhead in their gRPC receive limits. +pub async fn serve_with_engine_and_options( + listener: tokio::net::TcpListener, + engine: std::sync::Arc, + options: ServerOptions, + shutdown: impl std::future::Future + Send + 'static, +) -> Result<(), tonic::transport::Error> { + let service = std::sync::Arc::new(service::ConnectService::new(engine, options)); + let _cancel_on_drop = CancelOnDrop(service.clone()); + let router = tonic::transport::Server::builder().add_service( + grpc::spark_connect_service_server::SparkConnectServiceServer::from_arc(service.clone()) + .max_decoding_message_size(64 * 1024 * 1024), + ); + let (draining, drain_started) = tokio::sync::oneshot::channel(); + let server = router.serve_with_incoming_shutdown(tcp_incoming(listener), async move { + shutdown.await; + service.cancel_all(); + let _ = draining.send(()); + }); + tokio::pin!(server); + tokio::select! { + result = &mut server => result, + _ = drain_started => { + // A peer that stops reading must not prevent local process shutdown. + tokio::time::timeout(std::time::Duration::from_secs(2), &mut server) + .await.unwrap_or(Ok(())) + } + } +} + +#[cfg(test)] +mod tests { + use futures::StreamExt; + + #[tokio::test] + async fn accepted_connections_disable_nagle() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let mut incoming = super::tcp_incoming(listener); + let _client = tokio::net::TcpStream::connect(address).await.unwrap(); + let accepted = incoming.next().await.unwrap().unwrap(); + assert!(accepted.nodelay().unwrap()); + } +} diff --git a/crates/spark-connect-server/src/schema.rs b/crates/spark-connect-server/src/schema.rs new file mode 100644 index 00000000..daddedbb --- /dev/null +++ b/crates/spark-connect-server/src/schema.rs @@ -0,0 +1,231 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! The supported Spark schema boundary. Unsupported Arrow types fail explicitly. + +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, FieldRef, Fields, Schema, TimeUnit}; +use prost::Message; +use spark_connect_proto as proto; +use tonic::Status; + +pub(crate) const MAX_METADATA_RESPONSE_BYTES: usize = 3 * 1024 * 1024; + +pub(crate) fn result_schema(schema: &Schema) -> Result { + let schema = spark_schema(schema)?; + validate_schema_size(&schema)?; + Ok(schema) +} + +pub(crate) fn validate_schema_size(schema: &proto::DataType) -> Result<(), Status> { + if schema.encoded_len() > MAX_METADATA_RESPONSE_BYTES { + return Err(Status::resource_exhausted( + "Spark schema exceeds the 3 MiB schema/command response limit", + )); + } + Ok(()) +} + +/// Transform fields after recursively transforming their supported child fields. +pub(crate) fn map_fields( + fields: &Fields, + transform: &impl Fn(&FieldRef, DataType) -> FieldRef, +) -> Fields { + fn field(input: &FieldRef, transform: &impl Fn(&FieldRef, DataType) -> FieldRef) -> FieldRef { + let data_type = match input.data_type() { + DataType::List(child) => DataType::List(field(child, transform)), + DataType::LargeList(child) => DataType::LargeList(field(child, transform)), + DataType::Struct(children) => DataType::Struct(map_fields(children, transform)), + DataType::Map(entries, sorted) => DataType::Map(field(entries, transform), *sorted), + other => other.clone(), + }; + transform(input, data_type) + } + + fields.iter().map(|input| field(input, transform)).collect() +} + +/// Compare Spark-visible fields and types, allowing equivalent Arrow physical +/// representations and differences in nullability or field/schema metadata. +pub(crate) fn comparable_schema(schema: &Schema) -> Result { + let fields = map_fields(schema.fields(), &|field, data_type| { + Arc::new(Field::new(field.name(), data_type, true)) + }); + spark_schema(&Schema::new(fields)) +} + +pub fn spark_schema(schema: &Schema) -> Result { + spark_type(&DataType::Struct(schema.fields().clone())) +} + +fn validate_field_metadata(field: &Field) -> Result<(), Status> { + if !field.metadata().is_empty() { + return Err(Status::unimplemented( + "Arrow field metadata in Spark schemas", + )); + } + Ok(()) +} + +fn spark_field(field: &Field) -> Result { + validate_field_metadata(field)?; + Ok(proto::data_type::StructField { + name: field.name().clone(), + data_type: Some(spark_type(field.data_type())?), + nullable: field.is_nullable(), + metadata: None, + }) +} + +fn spark_type(data_type: &DataType) -> Result { + use proto::data_type::{self as pb, Kind}; + let kind = match data_type { + DataType::Null => Kind::Null(pb::Null::default()), + DataType::Boolean => Kind::Boolean(pb::Boolean::default()), + DataType::Int8 => Kind::Byte(pb::Byte::default()), + DataType::Int16 => Kind::Short(pb::Short::default()), + DataType::Int32 => Kind::Integer(pb::Integer::default()), + DataType::Int64 => Kind::Long(pb::Long::default()), + DataType::Float32 => Kind::Float(pb::Float::default()), + DataType::Float64 => Kind::Double(pb::Double::default()), + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Kind::String(pb::String { + collation: "UTF8_BINARY".into(), + ..Default::default() + }), + DataType::Binary | DataType::LargeBinary | DataType::BinaryView => { + Kind::Binary(pb::Binary::default()) + } + DataType::Date32 => Kind::Date(pb::Date::default()), + DataType::Timestamp(TimeUnit::Microsecond, None) => { + Kind::TimestampNtz(pb::TimestampNtz::default()) + } + DataType::Timestamp(TimeUnit::Microsecond, Some(_)) => { + Kind::Timestamp(pb::Timestamp::default()) + } + DataType::Decimal128(precision, scale) => Kind::Decimal(pb::Decimal { + precision: Some(i32::from(*precision)), + scale: Some(i32::from(*scale)), + ..Default::default() + }), + DataType::List(field) | DataType::LargeList(field) => { + validate_field_metadata(field)?; + Kind::Array(Box::new(pb::Array { + element_type: Some(Box::new(spark_type(field.data_type())?)), + contains_null: field.is_nullable(), + ..Default::default() + })) + } + DataType::Struct(fields) => Kind::Struct(pb::Struct { + fields: fields + .iter() + .map(|field| spark_field(field)) + .collect::>()?, + ..Default::default() + }), + DataType::Map(entries, _) => { + validate_field_metadata(entries)?; + let DataType::Struct(fields) = entries.data_type() else { + return Err(Status::invalid_argument( + "Arrow map entries must be a struct", + )); + }; + if fields.len() != 2 { + return Err(Status::invalid_argument( + "Arrow map entries need key and value fields", + )); + } + validate_field_metadata(&fields[0])?; + validate_field_metadata(&fields[1])?; + Kind::Map(Box::new(pb::Map { + key_type: Some(Box::new(spark_type(fields[0].data_type())?)), + value_type: Some(Box::new(spark_type(fields[1].data_type())?)), + value_contains_null: fields[1].is_nullable(), + ..Default::default() + })) + } + other => { + return Err(Status::unimplemented(format!( + "Arrow result type {other:?}" + ))) + } + }; + Ok(proto::DataType { kind: Some(kind) }) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + fn annotated(name: &str, data_type: DataType, nullable: bool) -> Field { + Field::new(name, data_type, nullable).with_metadata(HashMap::from([( + "producer.note".into(), + "cannot be represented in Spark schemas".into(), + )])) + } + + #[test] + fn declared_metadata_is_rejected_on_list_and_every_map_child() { + let entries = |key: Field, value: Field| { + Field::new("entries", DataType::Struct(vec![key, value].into()), false) + }; + let types = [ + DataType::List(Arc::new(annotated("item", DataType::Int64, true))), + DataType::LargeList(Arc::new(annotated("item", DataType::Int64, true))), + DataType::Map( + Arc::new(annotated( + "entries", + DataType::Struct( + vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Int64, true), + ] + .into(), + ), + false, + )), + false, + ), + DataType::Map( + Arc::new(entries( + annotated("key", DataType::Utf8, false), + Field::new("value", DataType::Int64, true), + )), + false, + ), + DataType::Map( + Arc::new(entries( + Field::new("key", DataType::Utf8, false), + annotated("value", DataType::Int64, true), + )), + false, + ), + ]; + for data_type in types { + let schema = Schema::new(vec![Field::new("nested", data_type, true)]); + assert_eq!( + result_schema(&schema).unwrap_err().code(), + tonic::Code::Unimplemented + ); + // Batch annotations remain acceptable when comparing the declared + // Spark-visible shape, including all nested child fields. + comparable_schema(&schema).unwrap(); + } + } +} diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs new file mode 100644 index 00000000..00c81c3f --- /dev/null +++ b/crates/spark-connect-server/src/service.rs @@ -0,0 +1,1199 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::{HashMap, VecDeque}; +use std::panic::AssertUnwindSafe; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; + +use arrow::datatypes::{Schema, SchemaRef}; +use arrow::ipc::{reader::StreamReader, writer::StreamWriter}; +use arrow::record_batch::RecordBatch; +use futures::{FutureExt, Stream, StreamExt}; +use prost::Message; +use spark_connect_proto as proto; +use tokio::sync::{mpsc, watch}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{Request, Response, Status}; +use uuid::Uuid; + +use crate::engine::{Engine, EngineSession, PreparedOperation, ResultBatch, SessionContext}; +use crate::grpc::spark_connect_service_server::SparkConnectService; +use crate::schema::{self, result_schema, MAX_METADATA_RESPONSE_BYTES}; + +type Output = Pin> + Send>>; +type SessionKey = (String, String); +const MAX_RELEASED_SESSIONS: usize = 4096; +const RELEASE_WAIT: std::time::Duration = std::time::Duration::from_secs(2); + +fn require_canonical_uuid(value: &str, name: &str) -> Result<(), Status> { + if Uuid::parse_str(value).is_ok_and(|id| id.hyphenated().to_string() == value) { + Ok(()) + } else { + Err(Status::invalid_argument(format!( + "{name} must be a canonical lowercase hyphenated UUID" + ))) + } +} + +async fn catch_engine_panic( + action: impl std::future::Future> + Send, +) -> Result { + AssertUnwindSafe(action) + .catch_unwind() + .await + .unwrap_or_else(|_| Err(Status::internal("engine request panicked"))) +} + +struct Session { + id: String, + backend: Arc, + cancelled: watch::Sender, + operations: Mutex>>, +} + +struct Operation { + cancelled: watch::Sender, + done: watch::Sender, +} + +impl Session { + fn new(backend: Arc) -> Self { + Self { + id: Uuid::new_v4().to_string(), + backend, + cancelled: watch::channel(false).0, + operations: Mutex::new(HashMap::new()), + } + } + + async fn while_open( + &self, + action: impl std::future::Future> + Send, + ) -> Result { + let mut cancelled = self.cancelled.subscribe(); + if *cancelled.borrow() { + return Err(Status::cancelled("session released")); + } + tokio::select! { + biased; + _ = cancelled.changed() => Err(Status::cancelled("session released")), + result = action => result, + } + } + + fn operation( + self: &Arc, + id: &str, + ) -> Result<(Arc, watch::Receiver), Status> { + // Subscribe before publishing: ReleaseAll can cancel as soon as the + // operation appears in the registry, before preparation is first polled. + let (cancelled, cancellation) = watch::channel(false); + let running = Arc::new(Operation { + cancelled, + done: watch::channel(false).0, + }); + let mut operations = self + .operations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if operations.contains_key(id) { + return Err(Status::already_exists("operation is already running")); + } + operations.insert(id.to_owned(), running.clone()); + Ok(( + Arc::new(OperationGuard { + session: self.clone(), + id: id.to_owned(), + running, + }), + cancellation, + )) + } +} + +// This guard spans preparation and execution, including eager commands. Drop the +// backend future/stream before the guard so ReleaseAll observes completed teardown. +struct OperationGuard { + session: Arc, + id: String, + running: Arc, +} + +impl Drop for OperationGuard { + fn drop(&mut self) { + self.session + .operations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&self.id); + self.running.done.send_replace(true); + } +} + +#[derive(Default)] +struct SessionRegistry { + // None is a recently released-session tombstone, retained until eviction. + entries: HashMap>>, + // Remember the most recent distinct closed IDs in release order. Repeated + // releases do not refresh retention. After eviction, an old ID may create a + // fresh session; active sessions never count toward this finite budget. + released: VecDeque, +} + +impl SessionRegistry { + fn release(&mut self, key: SessionKey, allow_reconnect: bool) -> Option> { + // Releasing a missing or already closed session is a no-op. In + // particular, a later reconnecting release cannot erase a tombstone. + let session = self.entries.get_mut(&key)?.take()?; + if allow_reconnect { + self.entries.remove(&key); + } else { + self.released.push_back(key); + if self.released.len() > MAX_RELEASED_SESSIONS { + if let Some(oldest) = self.released.pop_front() { + self.entries.remove(&oldest); + } + } + } + Some(session) + } +} + +pub(crate) struct ConnectService { + engine: Arc, + options: crate::ServerOptions, + sessions: Mutex, + closing: std::sync::atomic::AtomicBool, +} + +impl ConnectService { + pub(crate) fn new(engine: Arc, options: crate::ServerOptions) -> Self { + Self { + engine, + options, + sessions: Mutex::new(SessionRegistry::default()), + closing: std::sync::atomic::AtomicBool::new(false), + } + } + + fn key(id: &str, user: Option<&proto::UserContext>) -> Result { + require_canonical_uuid(id, "session_id")?; + Ok(( + user.map(|u| u.user_id.clone()).unwrap_or_default(), + id.into(), + )) + } + + fn session( + &self, + id: &str, + user: Option<&proto::UserContext>, + observed: Option<&str>, + ) -> Result, Status> { + self.find_session(id, user, observed, true) + } + + fn find_session( + &self, + id: &str, + user: Option<&proto::UserContext>, + observed: Option<&str>, + create: bool, + ) -> Result, Status> { + let key = Self::key(id, user)?; + let mut sessions = self + .sessions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.closing.load(std::sync::atomic::Ordering::SeqCst) { + return Err(Status::unavailable("server is shutting down")); + } + let session = match sessions.entries.get(&key) { + Some(Some(session)) => session.clone(), + Some(None) => return Err(Status::failed_precondition("session was released")), + None if !create => return Err(Status::not_found("session does not exist")), + // An observed ID refers to a previous incarnation; a new session + // cannot satisfy it. Reject before allocating backend resources. + None if observed.is_some() => { + return Err(Status::failed_precondition( + "server-side session ID changed", + )); + } + None => { + let backend = self.engine.create_session(SessionContext { + session_id: id.to_owned(), + user_context: user.cloned(), + })?; + let session = Arc::new(Session::new(backend)); + sessions.entries.insert(key, Some(session.clone())); + session + } + }; + if observed.is_some_and(|id| id != session.id) { + return Err(Status::failed_precondition( + "server-side session ID changed", + )); + } + Ok(session) + } + + pub(crate) fn cancel_all(&self) { + self.closing + .store(true, std::sync::atomic::Ordering::SeqCst); + // Stalled HTTP/2 connections can retain the service after the bounded + // transport drain. Remove its registry ownership now so backend sessions + // can close once their cancelled operations finish. + let sessions = std::mem::take( + &mut *self + .sessions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + ); + for session in sessions.entries.into_values().flatten() { + session.cancelled.send_replace(true); + } + } +} + +fn response( + session_id: &str, + session: &Session, + operation_id: &str, + response_type: Option, +) -> proto::ExecutePlanResponse { + proto::ExecutePlanResponse { + session_id: session_id.into(), + server_side_session_id: session.id.clone(), + operation_id: operation_id.into(), + response_id: Uuid::new_v4().to_string(), + response_type, + ..Default::default() + } +} + +fn checked_response( + message: proto::ExecutePlanResponse, +) -> Result { + if message.encoded_len() > MAX_METADATA_RESPONSE_BYTES { + return Err(Status::resource_exhausted( + "schema/command response exceeds the 3 MiB limit", + )); + } + Ok(message) +} + +fn complete(id: &str, session: &Session, operation: &str) -> proto::ExecutePlanResponse { + response( + id, + session, + operation, + Some(proto::execute_plan_response::ResponseType::ResultComplete( + proto::execute_plan_response::ResultComplete {}, + )), + ) +} + +fn encode_batch(batch: &RecordBatch) -> Result<(Vec, SchemaRef), Status> { + use arrow::datatypes::DataType; + use arrow::record_batch::RecordBatchOptions; + + // Materialize view arrays, including nested children, into the large-offset + // string/binary representation used for encoded RecordBatch results. + let source_schema = batch.schema(); + let fields = schema::map_fields(source_schema.fields(), &|field, data_type| { + let data_type = match data_type { + DataType::Utf8View => DataType::LargeUtf8, + DataType::BinaryView => DataType::LargeBinary, + other => other, + }; + if data_type == *field.data_type() { + field.clone() + } else { + Arc::new(field.as_ref().clone().with_data_type(data_type)) + } + }); + let changed = fields + .iter() + .zip(source_schema.fields()) + .any(|(target, source)| target.data_type() != source.data_type()); + let materialized = if changed { + let columns = batch + .columns() + .iter() + .zip(&fields) + .map(|(column, field)| { + if column.data_type() == field.data_type() { + Ok(column.clone()) + } else { + arrow::compute::cast(column.as_ref(), field.data_type()) + } + }) + .collect::, _>>() + .map_err(|error| Status::internal(error.to_string()))?; + let target_schema = Arc::new(Schema::new_with_metadata( + fields, + source_schema.metadata().clone(), + )); + Some( + RecordBatch::try_new_with_options( + target_schema, + columns, + &RecordBatchOptions::new().with_row_count(Some(batch.num_rows())), + ) + .map_err(|error| Status::internal(error.to_string()))?, + ) + } else { + None + }; + let batch = materialized.as_ref().unwrap_or(batch); + let mut bytes = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut bytes, batch.schema().as_ref()) + .map_err(|e| Status::internal(e.to_string()))?; + writer + .write(batch) + .map_err(|e| Status::internal(e.to_string()))?; + writer + .finish() + .map_err(|e| Status::internal(e.to_string()))?; + } + Ok((bytes, batch.schema())) +} + +// PySpark assembles one table from all IPC batches. Its physical types and +// nullability must agree across batches, while field/schema metadata may differ. +fn check_output_schema(schema: &Schema, expected: &mut Option) -> Result<(), Status> { + use arrow::datatypes::Field; + + let actual = Schema::new(schema::map_fields(schema.fields(), &|field, data_type| { + Arc::new(Field::new(field.name(), data_type, field.is_nullable())) + })); + match expected { + Some(expected) if *expected != actual => Err(Status::internal( + "Arrow output schema changed within one result", + )), + Some(_) => Ok(()), + None => { + *expected = Some(actual); + Ok(()) + } + } +} + +// Keep backend ownership until the batch is dropped, including on encoder panic. +struct EncodingBatch { + batch: RecordBatch, + _guard: Arc, +} + +impl EncodingBatch { + fn encode(self) -> Result<(Vec, SchemaRef), Status> { + encode_batch(&self.batch) + } +} + +fn arrow_chunk_size( + options: &[proto::execute_plan_request::RequestOption], + maximum: usize, +) -> Option { + use proto::execute_plan_request::request_option::RequestOption; + + let mut chunking = options + .iter() + .filter_map(|option| match &option.request_option { + Some(RequestOption::ResultChunkingOptions(options)) => Some(options), + _ => None, + }); + if maximum == 0 + || !chunking + .clone() + .any(|options| options.allow_arrow_batch_chunking) + { + return None; + } + Some( + chunking + .find_map(|options| options.preferred_arrow_chunk_size) + // Validate i64 directly; Scala's toInt can wrap an invalid preference. + .and_then(|bytes| usize::try_from(bytes).ok()) + .filter(|bytes| (1024..=maximum).contains(bytes)) + .unwrap_or(maximum), + ) +} + +async fn send_batch( + sender: &mpsc::Sender>, + id: &str, + operation: &str, + batch: ResultBatch, + batch_schema: &Schema, + offset: &mut i64, + output_schema: &mut Option, + guard: &Arc, + chunk_size: Option, +) -> Result { + // Reject unrepresentable counts/offsets before encoding or sending any chunk. + let row_count = match &batch { + ResultBatch::Arrow(batch) => i64::try_from(batch.num_rows()) + .map_err(|_| Status::resource_exhausted("result batch row count exceeds i64"))?, + ResultBatch::Ipc { row_count, .. } if *row_count < 0 => { + return Err(Status::internal("negative engine IPC row count")); + } + ResultBatch::Ipc { row_count, .. } => *row_count, + }; + let next_offset = offset + .checked_add(row_count) + .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; + let data: prost::bytes::Bytes = match batch { + ResultBatch::Arrow(batch) => { + // A blocking encoder cannot be cancelled once started. Retain engine + // ownership until its batch is dropped, even if the caller goes away. + let encoding = EncodingBatch { + batch, + _guard: guard.clone(), + }; + let (data, schema) = tokio::task::spawn_blocking(move || encoding.encode()) + .await + .map_err(|_| Status::internal("Arrow result encoding task failed"))??; + check_output_schema(schema.as_ref(), output_schema)?; + data.into() + } + ResultBatch::Ipc { data, .. } => { + check_output_schema(batch_schema, output_schema)?; + data.into() + } + }; + let mut payload = proto::execute_plan_response::ArrowBatch { + row_count, + start_offset: Some(*offset), + ..Default::default() + }; + let Some(chunk_bytes) = chunk_size else { + // As on the JVM, clients without chunking opt-in receive the original + // whole IPC batch. Their gRPC receive limit must accommodate it. + payload.data = data; + let message = response( + id, + &guard.session, + operation, + Some(proto::execute_plan_response::ResponseType::ArrowBatch( + payload, + )), + ); + *offset = next_offset; + return Ok(sender.send(Ok(message)).await.is_ok()); + }; + let chunks = data.len().div_ceil(chunk_bytes).max(1); + payload.num_chunks_in_batch = Some( + i64::try_from(chunks) + .map_err(|_| Status::resource_exhausted("Arrow chunk count exceeds i64"))?, + ); + for index in 0..chunks { + let from = index * chunk_bytes; + let to = from.saturating_add(chunk_bytes).min(data.len()); + // Bytes slices share the original IPC allocation, including compressed + // buffers. Clients concatenate the chunks before decoding Arrow. + payload.data = data.slice(from..to); + payload.chunk_index = Some(index as i64); + let message = response( + id, + &guard.session, + operation, + Some(proto::execute_plan_response::ResponseType::ArrowBatch( + payload.clone(), + )), + ); + if sender.send(Ok(message)).await.is_err() { + return Ok(false); + } + } + *offset = next_offset; + Ok(true) +} + +#[tonic::async_trait] +impl SparkConnectService for ConnectService { + type ExecutePlanStream = Output; + type ReattachExecuteStream = Output; + + async fn execute_plan( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let chunk_size = arrow_chunk_size(&req.request_options, self.options.max_arrow_chunk_size); + let operation = req + .operation_id + .unwrap_or_else(|| Uuid::new_v4().to_string()); + require_canonical_uuid(&operation, "operation_id")?; + let op = req + .plan + .and_then(|p| p.op_type) + .ok_or_else(|| Status::invalid_argument("missing plan"))?; + let session = self.session( + &req.session_id, + req.user_context.as_ref(), + req.client_observed_server_side_session_id.as_deref(), + )?; + let mut cancelled = session.cancelled.subscribe(); + if *cancelled.borrow() { + return Err(Status::cancelled("session released")); + } + // Reserve before calling the backend: preparing a command can mutate state. + let (guard, mut operation_cancelled) = session.operation(&operation)?; + let prepared = tokio::select! { + biased; + _ = cancelled.changed() => return Err(Status::cancelled("session released")), + _ = operation_cancelled.changed() => return Err(Status::cancelled("operation released")), + prepared = catch_engine_panic(async { session.backend.prepare(op).await }) => prepared?, + }; + let query = match prepared { + PreparedOperation::Query(query) => query, + PreparedOperation::Command(payloads) => { + let mut messages = payloads + .into_iter() + .map(|payload| { + let message = + response(&req.session_id, &session, &operation, Some(payload)); + checked_response(message) + }) + .collect::, Status>>()?; + messages.push(complete(&req.session_id, &session, &operation)); + return Ok(Response::new(Box::pin(tokio_stream::iter( + messages.into_iter().map(Ok), + )))); + } + }; + let arrow_schema = query.schema(); + let spark_schema = result_schema(&arrow_schema)?; + let expected_batch_schema = schema::comparable_schema(&arrow_schema)?; + let (sender, receiver) = mpsc::channel(2); + tokio::spawn(async move { + let run = async { + let mut header = response(&req.session_id, &session, &operation, None); + header.schema = Some(spark_schema); + let header = checked_response(header)?; + if sender.send(Ok(header)).await.is_err() { + return Ok(()); + } + let mut batches = query.execute().await?; + let mut offset = 0; + let mut output_schema = None; + let mut sent = false; + while let Some(batch) = batches.next().await { + let batch = batch?; + let batch_schema = match &batch { + ResultBatch::Arrow(batch) => batch.schema(), + ResultBatch::Ipc { data, .. } => { + StreamReader::try_new(data.as_slice(), None) + .map_err(|e| Status::internal(format!("invalid engine IPC: {e}")))? + .schema() + } + }; + let actual_schema = + schema::comparable_schema(batch_schema.as_ref()).map_err(|error| { + Status::internal(format!( + "invalid engine batch schema: {}", + error.message() + )) + })?; + if actual_schema != expected_batch_schema { + return Err(Status::internal( + "engine batch schema differs from its declared schema", + )); + } + if !send_batch( + &sender, + &req.session_id, + &operation, + batch, + batch_schema.as_ref(), + &mut offset, + &mut output_schema, + &guard, + chunk_size, + ) + .await? + { + return Ok(()); + } + sent = true; + } + // Teardown can itself panic. Complete only after the backend + // stream has been dropped within the panic boundary. + drop(batches); + if !sent { + let batch = RecordBatch::new_empty(arrow_schema); + let batch_schema = batch.schema(); + if !send_batch( + &sender, + &req.session_id, + &operation, + batch.into(), + batch_schema.as_ref(), + &mut offset, + &mut output_schema, + &guard, + chunk_size, + ) + .await? + { + return Ok(()); + } + } + let _ = sender + .send(Ok(complete(&req.session_id, &session, &operation))) + .await; + Ok::<_, Status>(()) + }; + let result = AssertUnwindSafe(async { + tokio::select! { + biased; + _ = cancelled.changed() => Err(Status::cancelled("session released")), + _ = operation_cancelled.changed() => Err(Status::cancelled("operation released")), + _ = sender.closed() => Ok(()), + result = run => result, + } + }) + .catch_unwind() + .await + .unwrap_or_else(|_| Err(Status::internal("engine execution panicked"))); + drop(guard); + // A slow peer can keep this terminal send pending after cancellation. + // Backend lifetime must end before waiting for that response slot. + drop(session); + if let Err(error) = result { + let _ = sender.send(Err(error)).await; + } + }); + Ok(Response::new(Box::pin(ReceiverStream::new(receiver)))) + } + + async fn analyze_plan( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let input = req + .analyze + .ok_or_else(|| Status::invalid_argument("missing analysis"))?; + let session = self.session( + &req.session_id, + req.user_context.as_ref(), + req.client_observed_server_side_session_id.as_deref(), + )?; + let result = session + .while_open(catch_engine_panic(async { + session.backend.analyze(input).await + })) + .await?; + if let proto::analyze_plan_response::Result::Schema(result) = &result { + if let Some(schema) = &result.schema { + schema::validate_schema_size(schema)?; + } + } + Ok(Response::new(proto::AnalyzePlanResponse { + session_id: req.session_id, + server_side_session_id: session.id.clone(), + result: Some(result), + })) + } + + async fn config( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let input = req + .operation + .and_then(|operation| operation.op_type) + .ok_or_else(|| Status::invalid_argument("missing config operation"))?; + let session = self.session( + &req.session_id, + req.user_context.as_ref(), + req.client_observed_server_side_session_id.as_deref(), + )?; + let result = session + .while_open(catch_engine_panic(async { + session.backend.config(input).await + })) + .await?; + Ok(Response::new(proto::ConfigResponse { + session_id: req.session_id, + server_side_session_id: session.id.clone(), + pairs: result.pairs, + warnings: result.warnings, + })) + } + + async fn release_execute( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + require_canonical_uuid(&req.operation_id, "operation_id")?; + let session = self.find_session( + &req.session_id, + req.user_context.as_ref(), + req.client_observed_server_side_session_id.as_deref(), + false, + )?; + // Completed responses are not retained; release is idempotent for absent operations. + let running = session + .operations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&req.operation_id) + .cloned(); + let operation_id = running.as_ref().map(|_| req.operation_id.clone()); + if matches!( + req.release, + Some(proto::release_execute_request::Release::ReleaseAll(_)) + ) { + if let Some(running) = running { + let mut done = running.done.subscribe(); + running.cancelled.send_replace(true); + tokio::time::timeout(RELEASE_WAIT, async { + while !*done.borrow_and_update() { + if done.changed().await.is_err() { + break; + } + } + }) + .await + .map_err(|_| { + Status::deadline_exceeded("operation cancellation is still draining") + })?; + } + } + Ok(Response::new(proto::ReleaseExecuteResponse { + session_id: req.session_id, + server_side_session_id: session.id.clone(), + operation_id, + })) + } + + async fn release_session( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let key = Self::key(&req.session_id, req.user_context.as_ref())?; + let session = { + let mut sessions = self + .sessions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + sessions.release(key, req.allow_reconnect) + }; + let server_id = session.as_ref().map(|s| s.id.clone()).unwrap_or_default(); + if let Some(session) = session { + session.cancelled.send_replace(true); + } + Ok(Response::new(proto::ReleaseSessionResponse { + session_id: req.session_id, + server_side_session_id: server_id, + })) + } + + async fn reattach_execute( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "execution reattachment is not supported", + )) + } + async fn add_artifacts( + &self, + _: Request>, + ) -> Result, Status> { + Err(Status::unimplemented("artifact uploads are not supported")) + } + async fn artifact_status( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("artifact status is not supported")) + } + async fn interrupt( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "operation interruption is not supported", + )) + } + async fn fetch_error_details( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "enriched error details are not supported", + )) + } + async fn clone_session( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "server-side session cloning is not supported", + )) + } + async fn get_status( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("status queries are not supported")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const CHUNK_SIZE: usize = 1024 * 1024; + + struct RegistryEngine; + struct RegistrySession; + + impl Engine for RegistryEngine { + fn create_session( + &self, + _context: SessionContext, + ) -> Result, Status> { + Ok(Arc::new(RegistrySession)) + } + } + + #[tonic::async_trait] + impl EngineSession for RegistrySession { + async fn prepare( + &self, + _operation: proto::plan::OpType, + ) -> Result { + Err(Status::unimplemented( + "registry fixture does not execute queries", + )) + } + + async fn analyze( + &self, + _input: proto::analyze_plan_request::Analyze, + ) -> Result { + Err(Status::unimplemented( + "registry fixture does not analyze queries", + )) + } + + async fn config( + &self, + _input: proto::config_request::operation::OpType, + ) -> Result { + Err(Status::unimplemented( + "registry fixture has no configuration", + )) + } + } + + async fn release(service: &ConnectService, id: &str, allow_reconnect: bool) { + service + .release_session(Request::new(proto::ReleaseSessionRequest { + session_id: id.into(), + allow_reconnect, + ..Default::default() + })) + .await + .unwrap(); + } + + #[test] + fn schema_response_limit_includes_protocol_envelope() { + let session = Session::new(Arc::new(RegistrySession)); + let schema = proto::DataType { + kind: Some(proto::data_type::Kind::Struct(proto::data_type::Struct { + fields: vec![proto::data_type::StructField { + name: "x".repeat(MAX_METADATA_RESPONSE_BYTES - 64), + ..Default::default() + }], + ..Default::default() + })), + }; + assert!(schema.encoded_len() < MAX_METADATA_RESPONSE_BYTES); + let mut header = response( + &Uuid::new_v4().to_string(), + &session, + &Uuid::new_v4().to_string(), + None, + ); + header.schema = Some(schema); + assert_eq!( + checked_response(header).unwrap_err().code(), + tonic::Code::ResourceExhausted + ); + } + + #[cfg(target_pointer_width = "64")] + #[tokio::test] + async fn unrepresentable_arrow_row_counts_fail_before_encoding() { + use arrow::array::NullArray; + use arrow::datatypes::{DataType, Field, Schema}; + use arrow::record_batch::RecordBatchOptions; + + let rows = i64::MAX as usize + 1; + let empty = RecordBatch::try_new_with_options( + Arc::new(Schema::empty()), + vec![], + &RecordBatchOptions::new().with_row_count(Some(rows)), + ) + .unwrap(); + let nulls = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("nulls", DataType::Null, true)])), + vec![Arc::new(NullArray::new(rows))], + ) + .unwrap(); + for batch in [empty, nulls] { + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("row_count").unwrap(); + let (sender, mut receiver) = mpsc::channel(2); + let mut offset = 7; + let mut output_schema = None; + let error = send_batch( + &sender, + "session", + "row_count", + batch.clone().into(), + batch.schema().as_ref(), + &mut offset, + &mut output_schema, + &guard, + Some(CHUNK_SIZE), + ) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::ResourceExhausted); + assert_eq!(error.message(), "result batch row count exceeds i64"); + assert_eq!(offset, 7); + assert!(output_schema.is_none()); + assert!(receiver.try_recv().is_err()); + } + } + + #[tokio::test] + async fn overflowing_engine_batches_do_not_partially_emit() { + use arrow::array::LargeStringArray; + use arrow::datatypes::{DataType, Field, Schema}; + + let schema = Arc::new(Schema::new(vec![Field::new( + "text", + DataType::LargeUtf8, + false, + )])); + let small = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(LargeStringArray::from(vec![""]))], + ) + .unwrap(); + let large = "z".repeat(2 * 1024 * 1024); + let oversized = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(LargeStringArray::from(vec![large.as_str(); 2]))], + ) + .unwrap(); + for ipc in [false, true] { + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("row_offset").unwrap(); + let (sender, mut receiver) = mpsc::channel(4); + let mut offset = i64::MAX - 2; + let mut output_schema = None; + send_batch( + &sender, + "session", + "row_offset", + small.clone().into(), + schema.as_ref(), + &mut offset, + &mut output_schema, + &guard, + Some(CHUNK_SIZE), + ) + .await + .unwrap(); + let batch = if ipc { + let (data, _) = encode_batch(&oversized).unwrap(); + assert!(data.len() > CHUNK_SIZE); + ResultBatch::Ipc { data, row_count: 2 } + } else { + oversized.clone().into() + }; + let error = send_batch( + &sender, + "session", + "row_offset", + batch, + schema.as_ref(), + &mut offset, + &mut output_schema, + &guard, + Some(CHUNK_SIZE), + ) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::ResourceExhausted); + assert_eq!(error.message(), "result row offset exceeds i64"); + assert_eq!(offset, i64::MAX - 1); + let previous = receiver.try_recv().unwrap().unwrap(); + let Some(proto::execute_plan_response::ResponseType::ArrowBatch(previous)) = + previous.response_type + else { + panic!("expected the preceding valid Arrow response"); + }; + assert_eq!(previous.start_offset, Some(i64::MAX - 2)); + assert_eq!(previous.row_count, 1); + assert!(receiver.try_recv().is_err()); + // A one-row batch still fits exactly at the signed boundary. + send_batch( + &sender, + "session", + "row_offset", + small.clone().into(), + schema.as_ref(), + &mut offset, + &mut output_schema, + &guard, + Some(CHUNK_SIZE), + ) + .await + .unwrap(); + assert_eq!(offset, i64::MAX); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn cancelled_encoding_wait_retains_operation_until_blocking_task_finishes() { + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let operation = Uuid::new_v4().to_string(); + let (guard, _) = session.operation(&operation).unwrap(); + let mut done = guard.running.done.subscribe(); + let encoding = EncodingBatch { + batch: RecordBatch::new_empty(Arc::new(arrow::datatypes::Schema::empty())), + _guard: guard.clone(), + }; + let (started, ready) = tokio::sync::oneshot::channel(); + let (release, resume) = std::sync::mpsc::channel(); + let waiting = tokio::spawn(async move { + tokio::task::spawn_blocking(move || { + let _ = started.send(()); + resume + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + encoding.encode() + }) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(5), ready) + .await + .unwrap() + .unwrap(); + waiting.abort(); + let _ = waiting.await; + drop(guard); + assert!(!*done.borrow()); + assert!(session.operations.lock().unwrap().contains_key(&operation)); + release.send(()).unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while !*done.borrow_and_update() { + done.changed().await.unwrap(); + } + }) + .await + .unwrap(); + assert!(!session.operations.lock().unwrap().contains_key(&operation)); + } + + #[tokio::test] + async fn released_session_retention_is_bounded_without_evicting_active_sessions() { + let service = + ConnectService::new(Arc::new(RegistryEngine), crate::ServerOptions::default()); + let active_id = Uuid::new_v4().to_string(); + let active = service.session(&active_id, None, None).unwrap(); + + let oldest = Uuid::new_v4().to_string(); + let previous = service.session(&oldest, None, None).unwrap(); + release(&service, &oldest, false).await; + assert!(*previous.cancelled.borrow()); + release(&service, &oldest, false).await; + release(&service, &oldest, true).await; + assert!(matches!(service.session(&oldest, None, None), + Err(status) if status.code() == tonic::Code::FailedPrecondition)); + { + let registry = service.sessions.lock().unwrap(); + assert_eq!(registry.entries.len(), 2); + assert_eq!(registry.released.len(), 1); + } + let mut newest = oldest.clone(); + for _ in 1..MAX_RELEASED_SESSIONS { + newest = Uuid::new_v4().to_string(); + service.session(&newest, None, None).unwrap(); + release(&service, &newest, false).await; + } + release(&service, &oldest, false).await; + assert!(matches!( + service.session(&oldest, None, None), + Err(status) if status.code() == tonic::Code::FailedPrecondition + )); + { + let registry = service.sessions.lock().unwrap(); + assert_eq!(registry.released.len(), MAX_RELEASED_SESSIONS); + assert_eq!(registry.entries.len(), MAX_RELEASED_SESSIONS + 1); + } + + // Unknown releases neither allocate tombstones nor evict closed IDs. + let unknown = Uuid::new_v4().to_string(); + release(&service, &unknown, false).await; + release(&service, &unknown, true).await; + assert!(matches!(service.session(&oldest, None, None), + Err(status) if status.code() == tonic::Code::FailedPrecondition)); + service.session(&unknown, None, None).unwrap(); + release(&service, &unknown, false).await; + assert!(service.session(&oldest, None, None).is_ok()); + assert!(matches!( + service.session(&newest, None, None), + Err(status) if status.code() == tonic::Code::FailedPrecondition + )); + assert!(Arc::ptr_eq( + &active, + &service.session(&active_id, None, None).unwrap() + )); + let registry = service.sessions.lock().unwrap(); + assert_eq!(registry.released.len(), MAX_RELEASED_SESSIONS); + assert_eq!(registry.entries.len(), MAX_RELEASED_SESSIONS + 2); + } + + #[tokio::test] + async fn reconnecting_release_of_live_session_allows_a_new_backend() { + let service = + ConnectService::new(Arc::new(RegistryEngine), crate::ServerOptions::default()); + let id = Uuid::new_v4().to_string(); + let previous = service.session(&id, None, None).unwrap(); + release(&service, &id, true).await; + assert!(*previous.cancelled.borrow()); + let current = service.session(&id, None, None).unwrap(); + assert!(!Arc::ptr_eq(¤t, &previous)); + } +} diff --git a/crates/spark-connect-server/tests/engine_injection.rs b/crates/spark-connect-server/tests/engine_injection.rs new file mode 100644 index 00000000..661d0450 --- /dev/null +++ b/crates/spark-connect-server/tests/engine_injection.rs @@ -0,0 +1,1818 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Real gRPC tests of the injected backend boundary. The fake is independent of Spark execution. + +#[path = "support/arrow.rs"] +mod arrow_support; +#[path = "support/requests.rs"] +mod requests; +#[path = "support/server.rs"] +mod server; + +use std::collections::HashMap; +use std::future::Future; +use std::io::Cursor; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::task::{Context, Poll}; +use std::time::Duration; + +use arrow::array::{Int64Array, StringArray}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::ipc::reader::StreamReader; +use arrow::ipc::writer::StreamWriter; +use arrow::record_batch::RecordBatch; +use arrow_support::chunking; +use futures::{Stream, StreamExt}; +use prost::Message; +use proto::execute_plan_response::ResponseType; +use proto::spark_connect_service_client::SparkConnectServiceClient; +use requests::{ + assert_cancelled, config_request, relation, release, release_session, request, user, +}; +use server::Server; +use spark_connect_proto as proto; +use spark_connect_server::engine::{ + BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, + ResultBatch, SessionContext, +}; +use tonic::{Code, Status}; +use uuid::Uuid; + +const LIVE: i64 = 1_000_000; +const BLOCKED_PREPARE: i64 = 2_000_000; +const MISMATCHED_BATCH: i64 = 3_000_000; +const ANNOTATED_BATCH: i64 = 4_000_000; +const FLOODED_RESULT: i64 = 5_000_000; +const PANICKING_EXECUTION: i64 = 6_000_000; +const PANICKING_STREAM: i64 = 7_000_000; +const PANICKING_STREAM_DROP: i64 = 8_000_000; +const MISMATCHED_IPC_BATCH: i64 = 9_000_000; +const INVALID_IPC_BATCH: i64 = 10_000_000; +const CHUNKED_RESULT: i64 = 11_000_000; + +#[derive(Default)] +struct Spy { + factory_calls: Mutex>, + fail_next: AtomicBool, + sessions_dropped: AtomicUsize, + prepares: AtomicUsize, + commands: AtomicUsize, + analyses: AtomicUsize, + configs: AtomicUsize, + executions: AtomicUsize, + streams_dropped: AtomicUsize, + batches_polled: AtomicUsize, + blocked_prepares_dropped: AtomicUsize, + blocked_analyses_dropped: AtomicUsize, + blocked_configs_dropped: AtomicUsize, + metadata_drop_observations: Mutex>, + slow_drop: Mutex)>>, + stream_drop_gate: Mutex>>, + request_panic: Mutex>, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Call { + Prepare, + Analyze, + Config, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum PanicAt { + Construction, + Poll, +} + +impl Spy { + fn panic_once(&self, call: Call) -> Option { + let mut next = self.request_panic.lock().unwrap(); + if next.as_ref().is_some_and(|(target, _)| *target == call) { + next.take().map(|(_, at)| at) + } else { + None + } + } +} + +type EngineFuture<'a, T> = Pin> + Send + 'a>>; + +#[derive(Default)] +struct DropGate { + released: Mutex, + condition: Condvar, + entered: AtomicUsize, + exited: AtomicBool, + timed_out: AtomicBool, +} + +impl DropGate { + fn wait(&self) { + let released = self.released.lock().unwrap(); + self.entered.fetch_add(1, Ordering::SeqCst); + let (released, timeout) = self + .condition + .wait_timeout_while(released, Duration::from_secs(10), |released| !*released) + .unwrap(); + self.timed_out + .store(timeout.timed_out() && !*released, Ordering::SeqCst); + self.exited.store(true, Ordering::SeqCst); + } + + fn release(&self) { + *self.released.lock().unwrap() = true; + self.condition.notify_all(); + } +} + +// Even a failing assertion must never leave a worker blocked in the fake Drop. +struct UnblockDrop(Arc); + +impl Drop for UnblockDrop { + fn drop(&mut self) { + self.0.release(); + } +} + +struct FakeEngine(Arc); + +impl Engine for FakeEngine { + fn create_session(&self, context: SessionContext) -> Result, Status> { + let drop_gate = self + .0 + .slow_drop + .lock() + .unwrap() + .as_ref() + .filter(|(id, _)| id == &context.session_id) + .map(|(_, gate)| gate.clone()); + let user = context + .user_context + .map(|user| user.user_id) + .unwrap_or_default(); + self.0 + .factory_calls + .lock() + .unwrap() + .push((user, context.session_id)); + if self.0.fail_next.swap(false, Ordering::SeqCst) { + return Err(Status::unavailable( + "fake factory is temporarily unavailable", + )); + } + Ok(Arc::new(FakeSession { + spy: self.0.clone(), + configs: Mutex::new(HashMap::new()), + views: Mutex::new(HashMap::new()), + drop_gate, + })) + } +} + +struct FakeSession { + spy: Arc, + configs: Mutex>, + views: Mutex>, + drop_gate: Option>, +} + +impl Drop for FakeSession { + fn drop(&mut self) { + if let Some(gate) = &self.drop_gate { + // Keep Drop synchronous, while allowing Tokio to replace this worker + // so the test measures registry contention rather than timer starvation. + tokio::task::block_in_place(|| gate.wait()); + } + self.spy.metadata_drop_observations.lock().unwrap().push(( + self.spy.blocked_analyses_dropped.load(Ordering::SeqCst), + self.spy.blocked_configs_dropped.load(Ordering::SeqCst), + )); + self.spy.sessions_dropped.fetch_add(1, Ordering::SeqCst); + } +} + +struct BlockedPrepare(Arc); + +impl Drop for BlockedPrepare { + fn drop(&mut self) { + self.0 + .blocked_prepares_dropped + .fetch_add(1, Ordering::SeqCst); + } +} + +enum MetadataCall { + Analysis, + Config, +} + +struct BlockedMetadata { + spy: Arc, + call: MetadataCall, +} + +impl Drop for BlockedMetadata { + fn drop(&mut self) { + let counter = match self.call { + MetadataCall::Analysis => &self.spy.blocked_analyses_dropped, + MetadataCall::Config => &self.spy.blocked_configs_dropped, + }; + counter.fetch_add(1, Ordering::SeqCst); + } +} + +// Explicit async-trait signatures also exercise a panic before a future is returned. +impl EngineSession for FakeSession { + fn prepare<'life0, 'async_trait>( + &'life0 self, + op: proto::plan::OpType, + ) -> EngineFuture<'async_trait, PreparedOperation> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + let panic = self.spy.panic_once(Call::Prepare); + assert!(panic != Some(PanicAt::Construction), "prepare construction"); + Box::pin(async move { + assert!(panic != Some(PanicAt::Poll), "prepare poll"); + self.spy.prepares.fetch_add(1, Ordering::SeqCst); + let (value, live, batch_mode) = match op { + proto::plan::OpType::Root(relation) => match relation.rel_type { + Some(proto::relation::RelType::Range(range)) => { + if range.end == BLOCKED_PREPARE { + let _dropped = BlockedPrepare(self.spy.clone()); + std::future::pending::<()>().await; + } + if range.end == -1 { + return Err(Status::invalid_argument("fake rejected the plan")); + } + // Deliberately unlike range semantics: this proves the injected + // backend, rather than the default planner, executed the request. + let batch_mode = match range.end { + MISMATCHED_BATCH => BatchMode::WrongType, + MISMATCHED_IPC_BATCH => BatchMode::WrongTypeIpc, + INVALID_IPC_BATCH => BatchMode::InvalidIpc, + ANNOTATED_BATCH => BatchMode::DifferentAnnotations, + FLOODED_RESULT => BatchMode::Flooded, + CHUNKED_RESULT => BatchMode::Chunked, + PANICKING_EXECUTION => BatchMode::PanicExecute, + PANICKING_STREAM => BatchMode::PanicStream, + PANICKING_STREAM_DROP => BatchMode::PanicDrop, + _ => BatchMode::Declared, + }; + (999, range.end == LIVE, batch_mode) + } + Some(proto::relation::RelType::Read(read)) => { + let Some(proto::read::ReadType::NamedTable(table)) = read.read_type else { + return Err(Status::unimplemented("fake read")); + }; + let value = self + .views + .lock() + .unwrap() + .get(&table.unparsed_identifier) + .copied() + .ok_or_else(|| Status::not_found("fake view not found"))?; + (value, false, BatchMode::Declared) + } + _ => return Err(Status::unimplemented("fake relation")), + }, + proto::plan::OpType::Command(command) => { + self.spy.commands.fetch_add(1, Ordering::SeqCst); + match command.command_type { + Some(proto::command::CommandType::CreateDataframeView(view)) => { + let Some(proto::relation::RelType::Range(input)) = + view.input.and_then(|input| input.rel_type) + else { + return Err(Status::invalid_argument( + "fake view requires range input", + )); + }; + let mut views = self.views.lock().unwrap(); + if !view.replace && views.contains_key(&view.name) { + return Err(Status::already_exists("fake view already exists")); + } + views.insert(view.name, input.end); + return Ok(PreparedOperation::Command(Vec::new())); + } + Some(proto::command::CommandType::SqlCommand(command)) => { + return Ok(PreparedOperation::Command(vec![ + ResponseType::SqlCommandResult( + proto::execute_plan_response::SqlCommandResult { + relation: command.input, + }, + ), + ])); + } + _ => return Err(Status::unimplemented("fake command")), + } + } + _ => return Err(Status::unimplemented("fake operation")), + }; + Ok(PreparedOperation::Query(Box::new(FakeQuery { + spy: self.spy.clone(), + value, + live, + batch_mode, + }))) + }) + } + + fn analyze<'life0, 'async_trait>( + &'life0 self, + input: proto::analyze_plan_request::Analyze, + ) -> EngineFuture<'async_trait, proto::analyze_plan_response::Result> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + let panic = self.spy.panic_once(Call::Analyze); + assert!(panic != Some(PanicAt::Construction), "analyze construction"); + Box::pin(async move { + assert!(panic != Some(PanicAt::Poll), "analyze poll"); + self.spy.analyses.fetch_add(1, Ordering::SeqCst); + match input { + proto::analyze_plan_request::Analyze::Schema(_) => { + Ok(proto::analyze_plan_response::Result::Schema( + proto::analyze_plan_response::Schema { + schema: Some(proto::DataType { + kind: Some(proto::data_type::Kind::Struct( + proto::data_type::Struct { + fields: vec![proto::data_type::StructField { + name: "x".repeat(3 * 1024 * 1024 + 1), + data_type: Some(proto::DataType { + kind: Some(proto::data_type::Kind::Long( + proto::data_type::Long::default(), + )), + }), + nullable: false, + metadata: None, + }], + ..Default::default() + }, + )), + }), + }, + )) + } + proto::analyze_plan_request::Analyze::IsStreaming(_) => { + let _dropped = BlockedMetadata { + spy: self.spy.clone(), + call: MetadataCall::Analysis, + }; + std::future::pending().await + } + proto::analyze_plan_request::Analyze::IsLocal(_) => { + Ok(proto::analyze_plan_response::Result::IsLocal( + proto::analyze_plan_response::IsLocal { is_local: true }, + )) + } + _ => Err(Status::unimplemented("fake analysis")), + } + }) + } + + fn config<'life0, 'async_trait>( + &'life0 self, + input: proto::config_request::operation::OpType, + ) -> EngineFuture<'async_trait, ConfigResult> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + let panic = self.spy.panic_once(Call::Config); + assert!(panic != Some(PanicAt::Construction), "config construction"); + Box::pin(async move { + assert!(panic != Some(PanicAt::Poll), "config poll"); + use proto::config_request::operation::OpType; + self.spy.configs.fetch_add(1, Ordering::SeqCst); + if matches!(&input, OpType::Get(get) if get.keys.iter().any(|key| key == "fake.block")) + { + let _dropped = BlockedMetadata { + spy: self.spy.clone(), + call: MetadataCall::Config, + }; + return std::future::pending().await; + } + let mut configs = self.configs.lock().unwrap(); + match input { + OpType::Set(set) => { + for pair in set.pairs { + configs.insert( + pair.key, + pair.value + .ok_or_else(|| Status::invalid_argument("fake config value"))?, + ); + } + Ok(ConfigResult { + pairs: Vec::new(), + warnings: vec!["handled by fake backend".into()], + }) + } + OpType::GetOption(get) => Ok(ConfigResult { + pairs: get + .keys + .into_iter() + .map(|key| { + let value = configs.get(&key).cloned(); + proto::KeyValue { key, value } + }) + .collect(), + warnings: Vec::new(), + }), + _ => Err(Status::unimplemented("fake config operation")), + } + }) + } +} + +struct FakeQuery { + spy: Arc, + value: i64, + live: bool, + batch_mode: BatchMode, +} + +enum BatchMode { + Declared, + WrongType, + WrongTypeIpc, + InvalidIpc, + DifferentAnnotations, + Flooded, + Chunked, + PanicExecute, + PanicStream, + PanicDrop, +} + +#[tonic::async_trait] +impl PreparedQuery for FakeQuery { + fn schema(&self) -> SchemaRef { + Arc::new(Schema::new(vec![Field::new( + "backend_value", + DataType::Int64, + false, + )])) + } + + async fn execute(self: Box) -> Result { + self.spy.executions.fetch_add(1, Ordering::SeqCst); + if matches!(self.batch_mode, BatchMode::PanicExecute) { + panic!("fake query execution panic"); + } + let batch = match self.batch_mode { + BatchMode::Declared + | BatchMode::InvalidIpc + | BatchMode::PanicExecute + | BatchMode::PanicStream + | BatchMode::PanicDrop => RecordBatch::try_new( + self.schema(), + vec![Arc::new(Int64Array::from(vec![self.value]))], + ), + BatchMode::WrongType | BatchMode::WrongTypeIpc => RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "backend_value", + DataType::Utf8, + false, + )])), + vec![Arc::new(StringArray::from(vec!["wrong type"]))], + ), + BatchMode::DifferentAnnotations => RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "backend_value", + DataType::Int64, + true, + ) + .with_metadata(HashMap::from([( + "backend.note".into(), + "ignored annotation".into(), + )]))])), + vec![Arc::new(Int64Array::from(vec![self.value]))], + ), + BatchMode::Chunked => RecordBatch::try_new( + self.schema(), + vec![Arc::new(Int64Array::from(vec![self.value; 512 * 1024]))], + ), + BatchMode::Flooded => RecordBatch::try_new( + self.schema(), + vec![Arc::new(Int64Array::from(vec![self.value; 64 * 1024]))], + ), + } + .unwrap(); + if matches!(self.batch_mode, BatchMode::Flooded) { + let spy = self.spy.clone(); + return Ok(Box::pin(SpyStream { + inner: Box::pin(futures::stream::repeat_with(move || { + spy.batches_polled.fetch_add(1, Ordering::SeqCst); + Ok(batch.clone().into()) + })), + spy: self.spy.clone(), + panic_on_drop: false, + })); + } + let result = match self.batch_mode { + BatchMode::WrongTypeIpc => { + let mut data = Vec::new(); + { + let mut writer = + StreamWriter::try_new(&mut data, batch.schema().as_ref()).unwrap(); + writer.write(&batch).unwrap(); + writer.finish().unwrap(); + } + ResultBatch::Ipc { data, row_count: 1 } + } + BatchMode::InvalidIpc => ResultBatch::Ipc { + data: b"invalid IPC".to_vec(), + row_count: 1, + }, + _ => batch.into(), + }; + let first = futures::stream::iter(vec![Ok(result)]); + let inner: BatchStream = if matches!(self.batch_mode, BatchMode::PanicStream) { + Box::pin(first.chain(futures::stream::poll_fn(|_| { + panic!("fake stream panic after a partial result"); + }))) + } else if self.live { + Box::pin(first.chain(futures::stream::pending())) + } else { + Box::pin(first) + }; + Ok(Box::pin(SpyStream { + inner, + spy: self.spy.clone(), + panic_on_drop: matches!(self.batch_mode, BatchMode::PanicDrop), + })) + } +} + +struct SpyStream { + inner: BatchStream, + spy: Arc, + panic_on_drop: bool, +} + +impl Stream for SpyStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + self.get_mut().inner.as_mut().poll_next(context) + } +} + +impl Drop for SpyStream { + fn drop(&mut self) { + let gate = self.spy.stream_drop_gate.lock().unwrap().clone(); + if let Some(gate) = gate { + tokio::task::block_in_place(|| gate.wait()); + } + self.spy.streams_dropped.fetch_add(1, Ordering::SeqCst); + assert!(!self.panic_on_drop, "fake stream teardown panic"); + } +} + +async fn start_server(spy: Arc) -> Server { + Server::start_with_engine(Arc::new(FakeEngine(spy))).await +} + +fn command_request( + session: &str, + command_type: proto::command::CommandType, +) -> proto::ExecutePlanRequest { + let mut request = request(session, 0); + request.plan = Some(proto::Plan { + op_type: Some(proto::plan::OpType::Command(proto::Command { + command_type: Some(command_type), + })), + }); + request +} + +fn get_config(session: &str) -> proto::ConfigRequest { + config_request( + session, + proto::config_request::operation::OpType::GetOption(proto::config_request::GetOption { + keys: vec!["spark.fake.setting".into()], + }), + ) +} + +async fn wait_for_count(counter: &AtomicUsize, expected: usize) { + tokio::time::timeout(Duration::from_secs(5), async { + while counter.load(Ordering::SeqCst) < expected { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("backend lifecycle event must arrive"); + assert_eq!(counter.load(Ordering::SeqCst), expected); +} + +async fn collect( + stream: &mut tonic::Streaming, +) -> Vec { + let mut responses = Vec::new(); + while let Some(response) = stream.message().await.unwrap() { + responses.push(response); + } + responses +} + +fn values(responses: &[proto::ExecutePlanResponse]) -> Vec { + let mut values = Vec::new(); + for response in responses { + if let Some(ResponseType::ArrowBatch(batch)) = &response.response_type { + for batch in StreamReader::try_new(Cursor::new(batch.data.clone()), None).unwrap() { + let batch = batch.unwrap(); + let array = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + values.extend((0..array.len()).map(|index| array.value(index))); + } + } + } + values +} + +async fn start_live( + server: &mut Server, + request: proto::ExecutePlanRequest, +) -> tonic::Streaming { + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + assert!(matches!( + stream.message().await.unwrap().unwrap().response_type, + Some(ResponseType::ArrowBatch(_)) + )); + stream +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn injected_backend_owns_query_analysis_config_and_catalog_state() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let mut first = server.client.clone(); + let mut second = server.client.clone(); + let (left, right) = tokio::join!( + first.config(get_config(&session)), + second.config(get_config(&session)) + ); + left.unwrap(); + right.unwrap(); + assert_eq!( + spy.factory_calls.lock().unwrap().len(), + 1, + "concurrent requests must share one backend session" + ); + + let set = config_request( + &session, + proto::config_request::operation::OpType::Set(proto::config_request::Set { + pairs: vec![proto::KeyValue { + key: "spark.fake.setting".into(), + value: Some("backend-owned".into()), + }], + silent: Some(false), + }), + ); + let configured = server.client.config(set).await.unwrap().into_inner(); + assert_eq!(configured.warnings, vec!["handled by fake backend"]); + assert_eq!( + server + .client + .config(get_config(&session)) + .await + .unwrap() + .into_inner() + .pairs[0] + .value + .as_deref(), + Some("backend-owned") + ); + let analysis = server + .client + .analyze_plan(proto::AnalyzePlanRequest { + session_id: session.clone(), + user_context: user("alice"), + analyze: Some(proto::analyze_plan_request::Analyze::IsLocal( + proto::analyze_plan_request::IsLocal { + plan: request(&session, 99).plan, + }, + )), + ..Default::default() + }) + .await + .unwrap() + .into_inner(); + assert!(matches!( + analysis.result, + Some(proto::analyze_plan_response::Result::IsLocal( + proto::analyze_plan_response::IsLocal { is_local: true } + )) + )); + + let mut stream = server + .client + .execute_plan(request(&session, 99)) + .await + .unwrap() + .into_inner(); + let responses = collect(&mut stream).await; + assert_eq!(values(&responses), vec![999]); + assert!(matches!( + responses.last().unwrap().response_type, + Some(ResponseType::ResultComplete(_)) + )); + let create_view = command_request( + &session, + proto::command::CommandType::CreateDataframeView(proto::CreateDataFrameViewCommand { + input: Some(relation(77)), + name: "fake_view".into(), + is_global: false, + replace: false, + }), + ); + let mut created = server + .client + .execute_plan(create_view) + .await + .unwrap() + .into_inner(); + collect(&mut created).await; + let mut read_view = request(&session, 0); + read_view.plan = Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(proto::Relation { + rel_type: Some(proto::relation::RelType::Read(proto::Read { + read_type: Some(proto::read::ReadType::NamedTable(proto::read::NamedTable { + unparsed_identifier: "fake_view".into(), + ..Default::default() + })), + ..Default::default() + })), + ..Default::default() + })), + }); + let mut stream = server + .client + .execute_plan(read_view.clone()) + .await + .unwrap() + .into_inner(); + assert_eq!(values(&collect(&mut stream).await), vec![77]); + let sql = command_request( + &session, + proto::command::CommandType::SqlCommand(proto::SqlCommand { + input: Some(relation(13)), + ..Default::default() + }), + ); + let mut stream = server.client.execute_plan(sql).await.unwrap().into_inner(); + let responses = collect(&mut stream).await; + assert!(matches!( + responses[0].response_type, + Some(ResponseType::SqlCommandResult(_)) + )); + + let mut other_user = get_config(&session); + other_user.user_context = user("bob"); + assert_eq!( + server + .client + .config(other_user) + .await + .unwrap() + .into_inner() + .pairs[0] + .value, + None + ); + read_view.user_context = user("bob"); + assert_eq!( + server + .client + .execute_plan(read_view) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + let other_session = Uuid::new_v4().to_string(); + assert_eq!( + server + .client + .config(get_config(&other_session)) + .await + .unwrap() + .into_inner() + .pairs[0] + .value, + None + ); + assert_eq!( + *spy.factory_calls.lock().unwrap(), + vec![ + ("alice".into(), session.clone()), + ("bob".into(), session), + ("alice".into(), other_session), + ] + ); + assert_eq!(spy.analyses.load(Ordering::SeqCst), 1); + assert_eq!(spy.commands.load(Ordering::SeqCst), 2); + assert_eq!(spy.executions.load(Ordering::SeqCst), 2); + server.stop().await; + wait_for_count(&spy.sessions_dropped, 3).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn invalid_requests_do_not_open_backends_and_factory_failures_can_retry() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + assert_eq!( + server + .client + .execute_plan(request("not-a-uuid", 0)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + let mut invalid_operation = request(&session, 0); + invalid_operation.operation_id = Some("not-a-uuid".into()); + assert_eq!( + server + .client + .execute_plan(invalid_operation) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + for plan in [None, Some(proto::Plan { op_type: None })] { + let mut request = request(&session, 0); + request.plan = plan; + assert_eq!( + server + .client + .execute_plan(request) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + } + assert_eq!( + server + .client + .analyze_plan(proto::AnalyzePlanRequest { + session_id: session.clone(), + ..Default::default() + }) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + for operation in [ + None, + Some(proto::config_request::Operation { op_type: None }), + ] { + assert_eq!( + server + .client + .config(proto::ConfigRequest { + session_id: session.clone(), + operation, + ..Default::default() + }) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + } + // A stale observed ID cannot match a session that does not exist. Repeated + // requests must not consume backend session capacity while being rejected. + for _ in 0..3 { + let mut stale = request(&session, 0); + stale.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); + assert_eq!( + server.client.execute_plan(stale).await.unwrap_err().code(), + Code::FailedPrecondition + ); + } + let mut stale = config_request( + &session, + proto::config_request::operation::OpType::GetOption(proto::config_request::GetOption { + keys: vec!["missing".into()], + }), + ); + stale.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); + assert_eq!( + server.client.config(stale).await.unwrap_err().code(), + Code::FailedPrecondition + ); + // Identifier and envelope validation must precede backend session allocation. + assert_eq!(spy.factory_calls.lock().unwrap().len(), 0); + assert_eq!( + server + .client + .release_execute(release(&session, &Uuid::new_v4().to_string())) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + assert_eq!( + server + .client + .release_execute(release("not-a-uuid", &session)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_execute(release(&session, "not-a-uuid")) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + let unknown = Uuid::new_v4().to_string(); + assert_eq!( + server + .client + .release_execute(release(&unknown, &session)) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + assert_eq!(spy.factory_calls.lock().unwrap().len(), 0); + spy.fail_next.store(true, Ordering::SeqCst); + assert_eq!( + server + .client + .config(get_config(&session)) + .await + .unwrap_err() + .code(), + Code::Unavailable + ); + server.client.config(get_config(&session)).await.unwrap(); + assert_eq!(spy.factory_calls.lock().unwrap().len(), 2); + server + .client + .release_session(release_session(&session)) + .await + .unwrap(); + wait_for_count(&spy.sessions_dropped, 1).await; + server + .client + .release_session(release_session(&session)) + .await + .unwrap(); + assert_eq!(spy.factory_calls.lock().unwrap().len(), 2); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn native_streams_drop_on_release_disconnect_session_release_and_shutdown() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let live = request(&session, LIVE); + let operation = live.operation_id.clone().unwrap(); + let mut stream = start_live(&mut server, live.clone()).await; + assert_eq!( + server + .client + .execute_plan(live.clone()) + .await + .unwrap_err() + .code(), + Code::AlreadyExists + ); + let mut duplicate = command_request( + &session, + proto::command::CommandType::SqlCommand(proto::SqlCommand { + input: Some(relation(1)), + ..Default::default() + }), + ); + duplicate.operation_id = Some(operation.clone()); + assert_eq!( + server + .client + .execute_plan(duplicate) + .await + .unwrap_err() + .code(), + Code::AlreadyExists + ); + assert_eq!(spy.commands.load(Ordering::SeqCst), 0); + assert_eq!(spy.prepares.load(Ordering::SeqCst), 1); + let mut release_request = release(&session, &operation); + release_request.client_observed_server_side_session_id = Some( + server + .client + .config(get_config(&session)) + .await + .unwrap() + .into_inner() + .server_side_session_id, + ); + let released = tokio::time::timeout( + Duration::from_secs(3), + server.client.release_execute(release_request.clone()), + ) + .await + .expect("ReleaseAll must finish without consuming the result") + .unwrap() + .into_inner(); + assert_eq!(released.operation_id.as_deref(), Some(operation.as_str())); + assert_eq!( + spy.streams_dropped.load(Ordering::SeqCst), + 1, + "release must wait for backend stream teardown" + ); + let absent = server + .client + .release_execute(release_request) + .await + .unwrap() + .into_inner(); + assert_eq!( + absent.operation_id, None, + "teardown must remove the operation" + ); + assert_cancelled(&mut stream).await; + + let mut abandoned = server + .client + .execute_plan(live.clone()) + .await + .unwrap() + .into_inner(); + assert!(abandoned.message().await.unwrap().unwrap().schema.is_some()); + // Keep the result batch unread while proving the backend stream exists. + wait_for_count(&spy.executions, 2).await; + drop(abandoned); + wait_for_count(&spy.streams_dropped, 2).await; + let mut restarted = tokio::time::timeout(Duration::from_secs(5), async { + loop { + match server.client.execute_plan(live.clone()).await { + Ok(response) => break response.into_inner(), + Err(status) if status.code() == Code::AlreadyExists => { + tokio::time::sleep(Duration::from_millis(10)).await + } + Err(status) => panic!("unexpected restart error: {status}"), + } + } + }) + .await + .expect("disconnected operation must be removed"); + assert!(restarted.message().await.unwrap().unwrap().schema.is_some()); + assert!(matches!( + restarted.message().await.unwrap().unwrap().response_type, + Some(ResponseType::ArrowBatch(_)) + )); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_cancelled(&mut restarted).await; + wait_for_count(&spy.streams_dropped, 3).await; + + let mut live = start_live(&mut server, request(&session, LIVE)).await; + server + .client + .release_session(release_session(&session)) + .await + .unwrap(); + assert_cancelled(&mut live).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert_eq!( + server + .client + .execute_plan(request(&session, 99)) + .await + .unwrap_err() + .code(), + Code::FailedPrecondition + ); + let other = Uuid::new_v4().to_string(); + let mut live = start_live(&mut server, request(&other, LIVE)).await; + server.shutdown.take().unwrap().send(()).unwrap(); + assert_cancelled(&mut live).await; + server.stop().await; + wait_for_count(&spy.streams_dropped, 5).await; + wait_for_count(&spy.sessions_dropped, 2).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shutdown_releases_backend_session_while_a_result_remains_unread() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let (mut client, connection) = h2::client::Builder::new() + .initial_window_size(1024) + .initial_connection_window_size(1024) + .handshake::<_, tonic::codegen::Bytes>( + tokio::net::TcpStream::connect(server.address) + .await + .unwrap(), + ) + .await + .unwrap(); + let connection = tokio::spawn(async move { + let _ = connection.await; + }); + let request = request(&Uuid::new_v4().to_string(), FLOODED_RESULT).encode_to_vec(); + let mut payload = vec![0]; + payload.extend_from_slice(&(request.len() as u32).to_be_bytes()); + payload.extend_from_slice(&request); + let request = tonic::codegen::http::Request::builder() + .method("POST") + .uri(format!( + "http://{}/spark.connect.SparkConnectService/ExecutePlan", + server.address + )) + .header("content-type", "application/grpc") + .body(()) + .unwrap(); + let (response, mut upload) = client.send_request(request, false).unwrap(); + upload.send_data(payload.into(), true).unwrap(); + let response = tokio::time::timeout(Duration::from_secs(5), response) + .await + .expect("response headers must arrive") + .unwrap(); + // Keep the body and HTTP/2 connection alive without reading or releasing + // capacity. Each 512 KiB batch exceeds the stream's 1 KiB receive window. + tokio::time::timeout(Duration::from_secs(5), async { + while spy.batches_polled.load(Ordering::SeqCst) < 4 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the frontend result queue must fill behind the unread peer"); + assert_eq!(spy.sessions_dropped.load(Ordering::SeqCst), 0); + server.shutdown.take().unwrap().send(()).unwrap(); + // Shutdown must release the session registry and the producer's session + // reference before waiting to send a terminal error through the full queue. + wait_for_count(&spy.streams_dropped, 1).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert!(!response.body().is_end_stream()); + drop(response); + drop(upload); + drop(client); + connection.abort(); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_reservation_precedes_prepare_and_cleans_up_after_cancellation_and_errors() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let blocked = request(&session, BLOCKED_PREPARE); + let operation = blocked.operation_id.clone().unwrap(); + let mut pending_client = server.client.clone(); + let pending = tokio::spawn(async move { pending_client.execute_plan(blocked).await }); + wait_for_count(&spy.prepares, 1).await; + let mut duplicate = command_request( + &session, + proto::command::CommandType::SqlCommand(proto::SqlCommand { + input: Some(relation(1)), + ..Default::default() + }), + ); + duplicate.operation_id = Some(operation.clone()); + assert_eq!( + server + .client + .execute_plan(duplicate) + .await + .unwrap_err() + .code(), + Code::AlreadyExists + ); + assert_eq!(spy.commands.load(Ordering::SeqCst), 0); + assert_eq!(spy.prepares.load(Ordering::SeqCst), 1); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_eq!(spy.blocked_prepares_dropped.load(Ordering::SeqCst), 1); + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), pending) + .await + .unwrap() + .unwrap() + .unwrap_err() + .code(), + Code::Cancelled + ); + + let mut retry = request(&session, 99); + retry.operation_id = Some(operation); + let mut stream = server + .client + .execute_plan(retry) + .await + .unwrap() + .into_inner(); + assert_eq!(values(&collect(&mut stream).await), vec![999]); + let rejected = request(&session, -1); + let mut retry = request(&session, 99); + retry.operation_id.clone_from(&rejected.operation_id); + assert_eq!( + server + .client + .execute_plan(rejected) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + let mut stream = server + .client + .execute_plan(retry) + .await + .unwrap() + .into_inner(); + assert_eq!(values(&collect(&mut stream).await), vec![999]); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn release_and_shutdown_cancel_backend_metadata_futures_before_dropping_sessions() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let mut analysis_client = server.client.clone(); + let analysis_session = session.clone(); + let analysis = tokio::spawn(async move { + analysis_client + .analyze_plan(proto::AnalyzePlanRequest { + session_id: analysis_session.clone(), + user_context: user("alice"), + analyze: Some(proto::analyze_plan_request::Analyze::IsStreaming( + proto::analyze_plan_request::IsStreaming { + plan: request(&analysis_session, 99).plan, + }, + )), + ..Default::default() + }) + .await + }); + wait_for_count(&spy.analyses, 1).await; + server + .client + .release_session(release_session(&session)) + .await + .unwrap(); + let status = tokio::time::timeout(Duration::from_secs(5), analysis) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert_eq!(status.code(), Code::Cancelled); + wait_for_count(&spy.blocked_analyses_dropped, 1).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert_eq!( + *spy.metadata_drop_observations.lock().unwrap(), + vec![(1, 0)] + ); + + let session = Uuid::new_v4().to_string(); + let mut config_client = server.client.clone(); + let config = tokio::spawn(async move { + config_client + .config(config_request( + &session, + proto::config_request::operation::OpType::Get(proto::config_request::Get { + keys: vec!["fake.block".into()], + }), + )) + .await + }); + wait_for_count(&spy.configs, 1).await; + server.shutdown.take().unwrap().send(()).unwrap(); + let status = tokio::time::timeout(Duration::from_secs(5), config) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert_eq!(status.code(), Code::Cancelled); + wait_for_count(&spy.blocked_configs_dropped, 1).await; + server.stop().await; + wait_for_count(&spy.sessions_dropped, 2).await; + assert_eq!( + *spy.metadata_drop_observations.lock().unwrap(), + vec![(1, 0), (1, 1)] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn slow_backend_drop_does_not_hold_the_global_session_registry_lock() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + // Channel clones share an HTTP/2 connection. Establish a new TCP connection + // before blocking Drop so the probe isolates the global registry lock. + let mut independent = SparkConnectServiceClient::connect(format!("http://{}", server.address)) + .await + .unwrap(); + let slow_session = Uuid::new_v4().to_string(); + let gate = Arc::new(DropGate::default()); + let unblock = UnblockDrop(gate.clone()); + *spy.slow_drop.lock().unwrap() = Some((slow_session.clone(), gate.clone())); + server + .client + .config(get_config(&slow_session)) + .await + .unwrap(); + let mut release_client = server.client.clone(); + let released = tokio::spawn(async move { + release_client + .release_session(release_session(&slow_session)) + .await + }); + // Store outcomes without asserting while Drop is blocked. Its own ten-second + // deadline and the RAII unblock guard also cover failures in this test. + let entered = tokio::time::timeout(Duration::from_secs(2), async { + while gate.entered.load(Ordering::SeqCst) == 0 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await; + let other = tokio::time::timeout( + Duration::from_secs(1), + independent.config(get_config(&Uuid::new_v4().to_string())), + ) + .await; + let still_blocked = !gate.exited.load(Ordering::SeqCst); + drop(unblock); + let released = tokio::time::timeout(Duration::from_secs(5), released).await; + server.stop().await; + + assert!(entered.is_ok(), "slow Drop must start"); + other + .expect("another session must remain responsive while Drop is blocked") + .unwrap(); + assert!( + still_blocked, + "the independent request must complete before Drop exits" + ); + assert!( + !gate.timed_out.load(Ordering::SeqCst), + "Drop must be explicitly unblocked" + ); + released.unwrap().unwrap().unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn frontend_rejects_oversized_analysis_and_batch_type_mismatches() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let status = server + .client + .analyze_plan(proto::AnalyzePlanRequest { + session_id: session.clone(), + user_context: user("alice"), + analyze: Some(proto::analyze_plan_request::Analyze::Schema( + proto::analyze_plan_request::Schema { + plan: request(&session, 99).plan, + }, + )), + ..Default::default() + }) + .await + .unwrap_err(); + assert_eq!(status.code(), Code::ResourceExhausted); + + let mut mismatch = server + .client + .execute_plan(request(&session, MISMATCHED_BATCH)) + .await + .unwrap() + .into_inner(); + let header = mismatch.message().await.unwrap().unwrap(); + assert!(header.schema.is_some()); + assert!(header.response_type.is_none()); + // The very next event must be the error: no inconsistent Arrow payload or + // successful completion may escape after the declared Int64 schema. + assert_eq!(mismatch.message().await.unwrap_err().code(), Code::Internal); + + let mut compatible = server + .client + .execute_plan(request(&session, ANNOTATED_BATCH)) + .await + .unwrap() + .into_inner(); + let responses = collect(&mut compatible).await; + assert_eq!(values(&responses), vec![999]); + assert!(matches!( + responses.last().unwrap().response_type, + Some(ResponseType::ResultComplete(_)) + )); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn frontend_rejects_mismatched_and_invalid_ipc_before_forwarding() { + let mut server = start_server(Arc::new(Spy::default())).await; + let session = Uuid::new_v4().to_string(); + for (selector, message) in [ + ( + MISMATCHED_IPC_BATCH, + "engine batch schema differs from its declared schema", + ), + (INVALID_IPC_BATCH, "invalid engine IPC:"), + ] { + let mut stream = server + .client + .execute_plan(request(&session, selector)) + .await + .unwrap() + .into_inner(); + let header = stream.message().await.unwrap().unwrap(); + assert!(header.schema.is_some()); + assert!(header.response_type.is_none()); + // The next event is the schema-validation error, before any IPC payload + // or successful completion can escape the declared schema header. + let status = stream.message().await.unwrap_err(); + assert_eq!(status.code(), Code::Internal); + assert!(status.message().starts_with(message), "{status}"); + } + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn panicking_execution_reports_internal_after_header_or_partial_result() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + for selector in [PANICKING_EXECUTION, PANICKING_STREAM, PANICKING_STREAM_DROP] { + let mut stream = server + .client + .execute_plan(request(&session, selector)) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + if selector != PANICKING_EXECUTION { + assert!(matches!( + stream.message().await.unwrap().unwrap().response_type, + Some(ResponseType::ArrowBatch(_)) + )); + } + let error = tokio::time::timeout(Duration::from_secs(5), stream.message()) + .await + .unwrap() + .unwrap_err(); + assert_eq!(error.code(), Code::Internal); + assert!(error.message().contains("panicked")); + } + let mut recovered = server + .client + .execute_plan(request(&session, 1)) + .await + .unwrap() + .into_inner(); + assert_eq!(values(&collect(&mut recovered).await), vec![999]); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn oversized_command_payload_fails_before_sending_any_success_message() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy).await; + let session = Uuid::new_v4().to_string(); + let command = command_request( + &session, + proto::command::CommandType::SqlCommand(proto::SqlCommand { + input: Some(proto::Relation { + rel_type: Some(proto::relation::RelType::Read(proto::Read { + read_type: Some(proto::read::ReadType::NamedTable(proto::read::NamedTable { + unparsed_identifier: "x".repeat(3 * 1024 * 1024 + 1), + ..Default::default() + })), + ..Default::default() + })), + ..Default::default() + }), + ..Default::default() + }), + ); + assert_eq!( + server + .client + .execute_plan(command) + .await + .unwrap_err() + .code(), + Code::ResourceExhausted + ); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn release_all_timeout_keeps_cancellation_and_operation_ownership_until_drain() { + let spy = Arc::new(Spy::default()); + let gate = Arc::new(DropGate::default()); + *spy.stream_drop_gate.lock().unwrap() = Some(gate.clone()); + let unblock = UnblockDrop(gate.clone()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let original = request(&session, LIVE); + let operation = original.operation_id.clone().unwrap(); + let mut stream = server + .client + .execute_plan(original.clone()) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + assert!(matches!( + stream.message().await.unwrap().unwrap().response_type, + Some(ResponseType::ArrowBatch(_)) + )); + let error = server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap_err(); + assert_eq!(error.code(), Code::DeadlineExceeded); + assert!(!gate.exited.load(Ordering::SeqCst)); + assert_eq!( + server + .client + .execute_plan(original.clone()) + .await + .unwrap_err() + .code(), + Code::AlreadyExists + ); + drop(unblock); + assert_cancelled(&mut stream).await; + assert!(!gate.timed_out.load(Ordering::SeqCst)); + *spy.stream_drop_gate.lock().unwrap() = None; + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + let mut recovered = server + .client + .execute_plan(original) + .await + .unwrap() + .into_inner(); + assert!(recovered.message().await.unwrap().unwrap().schema.is_some()); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_cancelled(&mut recovered).await; + server.stop().await; +} + +fn identified_request(session: &str, operation: &str) -> proto::ExecutePlanRequest { + let mut request = request(session, 1); + request.operation_id = Some(operation.into()); + request +} + +fn local_analysis(session: &str) -> proto::AnalyzePlanRequest { + proto::AnalyzePlanRequest { + session_id: session.into(), + user_context: requests::user("alice"), + analyze: Some(proto::analyze_plan_request::Analyze::IsLocal( + Default::default(), + )), + ..Default::default() + } +} + +async fn collect_identified(server: &mut Server, session: &str, operation: &str) { + let mut stream = server + .client + .execute_plan(identified_request(session, operation)) + .await + .unwrap() + .into_inner(); + let output = collect(&mut stream).await; + assert!(matches!( + output.last().unwrap().response_type, + Some(ResponseType::ResultComplete(_)) + )); +} + +#[tokio::test] +async fn noncanonical_ids_are_rejected_without_creating_or_releasing_state() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let uuid = Uuid::parse_str("abcdef12-3456-4789-abcd-0123456789ab").unwrap(); + let session = uuid.to_string(); + let operation = Uuid::new_v4().to_string(); + let original = server + .client + .config(get_config(&session)) + .await + .unwrap() + .into_inner() + .server_side_session_id; + for alias in [ + uuid.simple().to_string(), + uuid.braced().to_string(), + uuid.urn().to_string(), + session.to_uppercase(), + ] { + assert_eq!( + server + .client + .config(get_config(&alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .analyze_plan(local_analysis(&alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .execute_plan(identified_request(&alias, &operation)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_execute(release(&alias, &operation)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_session(release_session(&alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .execute_plan(identified_request(&Uuid::new_v4().to_string(), &alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_execute(release(&session, &alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .config(get_config(&session)) + .await + .unwrap() + .into_inner() + .server_side_session_id, + original + ); + } + assert_eq!(spy.factory_calls.lock().unwrap().len(), 1); + assert_eq!(spy.prepares.load(Ordering::SeqCst), 0); + collect_identified(&mut server, &session, &operation).await; + server.stop().await; +} + +#[tokio::test] +async fn backend_request_panics_have_internal_status_and_leave_the_session_usable() { + for call in [Call::Prepare, Call::Analyze, Call::Config] { + for at in [PanicAt::Construction, PanicAt::Poll] { + let spy = Arc::new(Spy::default()); + *spy.request_panic.lock().unwrap() = Some((call, at)); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let operation = Uuid::new_v4().to_string(); + let status = match call { + Call::Prepare => server + .client + .execute_plan(identified_request(&session, &operation)) + .await + .unwrap_err(), + Call::Analyze => server + .client + .analyze_plan(local_analysis(&session)) + .await + .unwrap_err(), + Call::Config => server + .client + .config(get_config(&session)) + .await + .unwrap_err(), + }; + assert_eq!(status.code(), Code::Internal); + assert_eq!(status.message(), "engine request panicked"); + server.client.config(get_config(&session)).await.unwrap(); + server + .client + .analyze_plan(local_analysis(&session)) + .await + .unwrap(); + // Reuse the operation ID after a preparation panic: its reservation + // must have been released before the failed RPC returned. + collect_identified(&mut server, &session, &operation).await; + assert_eq!(spy.factory_calls.lock().unwrap().len(), 1); + server.stop().await; + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn release_cancels_an_arrow_batch_between_byte_chunks() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let mut request = request(&session, CHUNKED_RESULT); + let operation = request.operation_id.clone().unwrap(); + request.request_options = vec![chunking(true, Some(1024))]; + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + let first = stream.message().await.unwrap().unwrap(); + let Some(ResponseType::ArrowBatch(chunk)) = first.response_type else { + panic!("first Arrow chunk") + }; + assert_eq!(chunk.chunk_index, Some(0)); + assert!(chunk.num_chunks_in_batch.unwrap() > 4000); + assert_eq!(chunk.row_count, 512 * 1024); + assert_eq!(chunk.start_offset, Some(0)); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_cancelled(&mut stream).await; + wait_for_count(&spy.streams_dropped, 1).await; + assert_eq!(spy.executions.load(Ordering::SeqCst), 1); + server.stop().await; +} diff --git a/crates/spark-connect-server/tests/protocol.rs b/crates/spark-connect-server/tests/protocol.rs new file mode 100644 index 00000000..d5f9bfb5 --- /dev/null +++ b/crates/spark-connect-server/tests/protocol.rs @@ -0,0 +1,908 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Verify observable Spark Connect wire behavior independently of DataFrame conveniences. + +#[path = "support/arrow.rs"] +mod arrow_support; +#[path = "support/requests.rs"] +mod requests; +#[path = "support/server.rs"] +mod server; +mod support; + +use std::collections::HashSet; +use std::io::Cursor; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::{ + Array, ArrayRef, BinaryViewArray, Int64Array, LargeBinaryArray, LargeListArray, + LargeStringArray, ListArray, MapArray, StringViewArray, StructArray, +}; +use arrow::buffer::OffsetBuffer; +use arrow::datatypes::{DataType, Field, Schema}; +use arrow::ipc::reader::StreamReader; +use arrow::ipc::CompressionType; +use arrow::record_batch::RecordBatch; +use prost::Message; +use proto::execute_plan_response::ResponseType; +use spark_connect_proto as proto; + +use arrow_support::{assembled_batches, chunking, encoded, DEFAULT_ARROW_CHUNK_SIZE}; +use requests::request as range; +use server::Server; +use support::{Batch, FixtureEngine}; +use tonic::Code; +use uuid::Uuid; + +async fn start_server() -> Server { + Server::start_with_engine(Arc::new(FixtureEngine::default())).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn arrow_stream_preserves_schema_ids_offsets_and_empty_results() { + let mut server = start_server().await; + let session = Uuid::new_v4().to_string(); + let mut server_session = None; + for end in [0, 25_000] { + let request = range(&session, end); + let operation = request.operation_id.clone().unwrap(); + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + let mut response_ids = HashSet::new(); + let (mut rows, mut batches, mut schemas, mut complete) = (0, 0, 0, false); + while let Some(response) = stream.message().await.unwrap() { + assert!(!complete, "ResultComplete must be the last response"); + assert_eq!(response.session_id, session); + assert_eq!(response.operation_id, operation); + Uuid::parse_str(&response.response_id).unwrap(); + assert!(response_ids.insert(response.response_id)); + Uuid::parse_str(&response.server_side_session_id).unwrap(); + let stable = server_session.get_or_insert(response.server_side_session_id.clone()); + assert_eq!(*stable, response.server_side_session_id); + if let Some(schema) = response.schema { + let Some(proto::data_type::Kind::Struct(schema)) = schema.kind else { + panic!("struct schema"); + }; + assert_eq!(schema.fields.len(), 1); + assert_eq!(schema.fields[0].name, "id"); + assert!(!schema.fields[0].nullable); + assert!(matches!( + schema.fields[0].data_type.as_ref().unwrap().kind, + Some(proto::data_type::Kind::Long(_)) + )); + schemas += 1; + } + match response.response_type { + Some(ResponseType::ArrowBatch(batch)) => { + assert_eq!(batch.start_offset, Some(rows)); + let reader = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); + let mut batch_rows = 0; + for record in reader { + let record = record.unwrap(); + let values = record + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for index in 0..values.len() { + assert_eq!(values.value(index), rows + batch_rows + index as i64); + } + batch_rows += record.num_rows() as i64; + } + assert_eq!(batch_rows, batch.row_count); + rows += batch_rows; + batches += 1; + } + Some(ResponseType::ResultComplete(_)) => complete = true, + None => {} + other => panic!("unexpected result: {other:?}"), + } + } + assert_eq!(rows, end); + assert_eq!(schemas, 1); + assert!(complete); + assert!(batches >= 1, "empty execution must still return Arrow data"); + if end > 0 { + // The fixture supplies multiple physical batches; all must reach the wire. + assert!(batches > 1, "large result should stream multiple batches"); + } + } + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn observed_session_mismatch_and_released_sessions_are_rejected() { + let mut server = start_server().await; + let session = Uuid::new_v4().to_string(); + let mut first = server + .client + .execute_plan(range(&session, 0)) + .await + .unwrap() + .into_inner(); + let original = first + .message() + .await + .unwrap() + .unwrap() + .server_side_session_id; + while first.message().await.unwrap().is_some() {} + let mut mismatch = range(&session, 1); + mismatch.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); + let error = server.client.execute_plan(mismatch).await.unwrap_err(); + assert_eq!(error.code(), Code::FailedPrecondition); + let release = proto::ReleaseSessionRequest { + session_id: session.clone(), + user_context: requests::user("alice"), + ..Default::default() + }; + let released = server + .client + .release_session(release.clone()) + .await + .unwrap() + .into_inner(); + assert_eq!(released.server_side_session_id, original); + server.client.release_session(release).await.unwrap(); + let error = server + .client + .execute_plan(range(&session, 1)) + .await + .unwrap_err(); + assert_eq!(error.code(), Code::FailedPrecondition); + // A separate session on the same transport remains usable. + let mut other = server + .client + .execute_plan(range(&Uuid::new_v4().to_string(), 0)) + .await + .unwrap() + .into_inner(); + while other.message().await.unwrap().is_some() {} + server.stop().await; +} + +async fn collect_result( + server: &mut Server, + request: proto::ExecutePlanRequest, +) -> Vec { + tokio::time::timeout(Duration::from_secs(30), async { + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + let mut responses = Vec::new(); + while let Some(response) = stream.message().await.unwrap() { + responses.push(response); + } + assert!(matches!( + responses.last().unwrap().response_type, + Some(ResponseType::ResultComplete(_)) + )); + responses + }) + .await + .expect("fixture execution must terminate") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn negotiated_chunks_preserve_plain_and_compressed_ipc_bytes_and_offsets() { + // Incompressible values keep both compression codecs above the chunk limit. + let mut random = 1u64; + let values = Int64Array::from_iter_values((0..450_000).map(|_| { + random ^= random << 13; + random ^= random >> 7; + random ^= random << 17; + random as i64 + })); + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(values)]).unwrap(); + for compression in [ + None, + Some(CompressionType::ZSTD), + Some(CompressionType::LZ4_FRAME), + ] { + let data = encoded(&batch, compression); + assert!(data.len() > 3 * 1024 * 1024); + let tail = batch.slice(0, 3); + let tail_data = encoded(&tail, compression); + let engine = FixtureEngine::default().with_results( + 42, + schema.clone(), + vec![ + Batch::Ipc(data.clone(), batch.num_rows() as i64), + Batch::Ipc(tail_data.clone(), 3), + ], + ); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + server.client = server + .client + .clone() + .max_decoding_message_size(8 * 1024 * 1024); + for option in [ + None, + Some(chunking(false, Some(1024))), + Some(chunking(true, None)), + Some(chunking(true, Some(3 * 1024 * 1024))), + ] { + let negotiated = option.as_ref().is_some_and(|option| matches!( + &option.request_option, + Some(proto::execute_plan_request::request_option::RequestOption::ResultChunkingOptions(options)) + if options.allow_arrow_batch_chunking + )); + let mut request = range(&Uuid::new_v4().to_string(), 42); + let limited = option.as_ref().is_some_and(|option| matches!( + &option.request_option, + Some(proto::execute_plan_request::request_option::RequestOption::ResultChunkingOptions(options)) + if options.allow_arrow_batch_chunking && options.preferred_arrow_chunk_size.is_some() + )); + request.request_options = option.into_iter().collect(); + let responses = collect_result(&mut server, request).await; + let output = assembled_batches(&responses, negotiated); + assert_eq!(output.len(), 2); + assert_eq!(output[0].data.as_ref(), data.as_slice()); + assert_eq!(output[1].data.as_ref(), tail_data.as_slice()); + assert_eq!(output[0].row_count, 450_000); + assert_eq!(output[1].start_offset, Some(450_000)); + for (result, expected) in output.iter().zip([&batch, &tail]) { + let mut reader = StreamReader::try_new(result.data.as_ref(), None).unwrap(); + assert_eq!(reader.next().unwrap().unwrap(), *expected); + assert!(reader.next().is_none()); + } + let chunks = responses + .iter() + .filter(|response| { + matches!(response.response_type, Some(ResponseType::ArrowBatch(_))) + }) + .count(); + assert_eq!(chunks, if limited { 3 } else { 2 }); + } + server.stop().await; + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn preferred_chunk_size_uses_the_protocol_range_and_splits_inside_a_large_row() { + use arrow::array::StringArray; + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Utf8, + false, + )])), + vec![Arc::new(StringArray::from(vec![ + "x".repeat(4 * 1024 * 1024) + ]))], + ) + .unwrap(); + let engine = FixtureEngine::default().with_batches(42, batch.schema(), vec![batch.clone()]); + let mut options = spark_connect_server::ServerOptions::default(); + options.max_arrow_chunk_size = 3 * 1024 * 1024; + let mut server = Server::start_with_options(Arc::new(engine), options).await; + for preferred in [ + Some(1024), + Some(65_537), + Some(3 * 1024 * 1024), + None, + Some(0), + Some(-1), + Some(1023), + Some(3 * 1024 * 1024 + 1), + Some(i64::MAX), + ] { + let mut request = range(&Uuid::new_v4().to_string(), 42); + request.request_options = vec![chunking(true, preferred)]; + let responses = collect_result(&mut server, request).await; + let payloads: Vec<_> = responses + .iter() + .filter_map(|response| match &response.response_type { + Some(ResponseType::ArrowBatch(batch)) => Some(batch.data.len()), + _ => None, + }) + .collect(); + assert!(payloads.len() >= 2); + if let Some(size) = preferred.filter(|size| (1024..=3 * 1024 * 1024).contains(size)) { + assert!(payloads.iter().all(|len| *len <= size as usize)); + assert_eq!(payloads[0], size as usize); + } else { + assert_eq!( + payloads.len(), + 2, + "invalid preferences use the server maximum" + ); + } + let output = assembled_batches(&responses, true); + assert_eq!(output.len(), 1); + assert_eq!(output[0].row_count, 1); + let mut reader = StreamReader::try_new(output[0].data.as_ref(), None).unwrap(); + assert_eq!(reader.next().unwrap().unwrap(), batch); + assert!(reader.next().is_none()); + } + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn backend_status_codes_survive_prepare_and_stream_failures() { + let engine = FixtureEngine::default() + .reject(10, Code::Unimplemented) + .reject(11, Code::InvalidArgument) + .stream_error(12, Code::ResourceExhausted); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let session = Uuid::new_v4().to_string(); + for (selector, expected) in [(10, Code::Unimplemented), (11, Code::InvalidArgument)] { + let status = server + .client + .execute_plan(range(&session, selector)) + .await + .unwrap_err(); + assert_eq!(status.code(), expected); + } + let mut stream = server + .client + .execute_plan(range(&session, 12)) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + assert_eq!( + stream.message().await.unwrap_err().code(), + Code::ResourceExhausted + ); + // Backend failures must not poison the protocol session or its transport. + let mut recovered = server + .client + .execute_plan(range(&session, 1)) + .await + .unwrap() + .into_inner(); + let (mut rows, mut complete) = (0, false); + while let Some(response) = recovered.message().await.unwrap() { + match response.response_type { + Some(ResponseType::ArrowBatch(batch)) => rows += batch.row_count, + Some(ResponseType::ResultComplete(_)) => complete = true, + None => {} + other => panic!("unexpected result: {other:?}"), + } + } + assert_eq!(rows, 1); + assert!(complete); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn deeply_nested_requests_follow_the_configured_protobuf_decoder_limit() { + let mut server = start_server().await; + let session = Uuid::new_v4().to_string(); + let mut request = range(&session, 1); + let Some(proto::plan::OpType::Root(mut relation)) = request.plan.take().unwrap().op_type else { + unreachable!() + }; + for _ in 0..60 { + relation = proto::Relation { + rel_type: Some(proto::relation::RelType::Filter(Box::new(proto::Filter { + input: Some(Box::new(relation)), + condition: Some(proto::Expression { + expr_type: Some(proto::expression::ExprType::Literal( + proto::expression::Literal { + literal_type: Some(proto::expression::literal::LiteralType::Boolean( + true, + )), + ..Default::default() + }, + )), + ..Default::default() + }), + }))), + ..Default::default() + }; + } + request.plan = Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(relation)), + }); + let decoded = proto::ExecutePlanRequest::decode(request.encode_to_vec().as_slice()); + let status = server.client.execute_plan(request).await.unwrap_err(); + match decoded { + Err(error) => { + // The default decoder rejects this before the backend can inspect it. + assert!(error.to_string().contains("recursion"), "{error}"); + assert_eq!(status.code(), Code::Internal); + assert!(status.message().contains("recursion"), "{status}"); + } + Ok(_) => { + // Other workspace crates enable prost/no-recursion-limit. In that + // configuration the unsupported Filter reaches the fixture backend. + assert_eq!(status.code(), Code::Unimplemented); + assert_eq!(status.message(), "fixture requires a range selector"); + } + } + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_arrays_are_materialized_before_byte_chunking_including_struct_children() { + use arrow::array::{BinaryArray, BinaryViewArray, StringArray, StringViewArray, StructArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use arrow::record_batch::RecordBatch; + + let session = Uuid::new_v4().to_string(); + let text = |id: i64| format!("{id:04}{}", "x".repeat(1020)); + let nested_text = |id: i64| format!("n{id:04}{}", "y".repeat(1019)); + let binary = |id: i64| { + let mut bytes = id.to_le_bytes().to_vec(); + bytes.extend(std::iter::repeat_n((id % 251) as u8, 1016)); + bytes + }; + let strings = StringViewArray::from_iter_values((0..4096).map(text)); + let binaries = BinaryViewArray::from_iter_values((0..4096).map(binary)); + let nested_strings = StringViewArray::from_iter_values((0..4096).map(nested_text)); + // These view columns exceed the chunk preference, so their encoded batch + // spans several byte chunks. + for size in [ + strings + .data_buffers() + .iter() + .map(|buffer| buffer.len()) + .sum::(), + binaries + .data_buffers() + .iter() + .map(|buffer| buffer.len()) + .sum::(), + nested_strings + .data_buffers() + .iter() + .map(|buffer| buffer.len()) + .sum::(), + ] { + assert!(size > 3 * 1024 * 1024); + } + let nested_fields: arrow::datatypes::Fields = + vec![Field::new("text", DataType::Utf8View, false)].into(); + let nested = StructArray::new(nested_fields.clone(), vec![Arc::new(nested_strings)], None); + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8View, false), + Field::new("binary", DataType::BinaryView, false), + Field::new("nested", DataType::Struct(nested_fields), false), + ])), + vec![ + Arc::new(Int64Array::from_iter_values(0..4096)), + Arc::new(strings), + Arc::new(binaries), + Arc::new(nested), + ], + ) + .unwrap(); + let engine = FixtureEngine::default().with_batches(42, batch.schema(), vec![batch]); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let mut request = range(&session, 42); + request.request_options = vec![chunking(true, Some(3 * 1024 * 1024))]; + let responses = collect_result(&mut server, request).await; + let output = assembled_batches(&responses, true); + assert_eq!(output.len(), 1); + assert!(output[0].num_chunks_in_batch.unwrap() > 1); + let batch = &output[0]; + assert_eq!(batch.start_offset, Some(0)); + let records = StreamReader::try_new(Cursor::new(&batch.data), None).unwrap(); + let mut rows = 0; + for record in records { + let record = record.unwrap(); + let ids = record + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(record.column(1).data_type(), &DataType::LargeUtf8); + assert_eq!(record.column(2).data_type(), &DataType::LargeBinary); + let strings = arrow::compute::cast(record.column(1), &DataType::Utf8).unwrap(); + let strings = strings.as_any().downcast_ref::().unwrap(); + let binaries = arrow::compute::cast(record.column(2), &DataType::Binary).unwrap(); + let binaries = binaries.as_any().downcast_ref::().unwrap(); + let nested = record + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(nested.column(0).data_type(), &DataType::LargeUtf8); + let nested_strings = arrow::compute::cast(nested.column(0), &DataType::Utf8).unwrap(); + let nested_strings = nested_strings + .as_any() + .downcast_ref::() + .unwrap(); + for index in 0..record.num_rows() { + let id = rows + index as i64; + assert_eq!(ids.value(index), id); + assert_eq!(strings.value(index), text(id)); + assert_eq!(binaries.value(index), binary(id)); + assert_eq!(nested_strings.value(index), nested_text(id)); + } + rows += record.num_rows() as i64; + } + assert_eq!(rows, batch.row_count); + assert_eq!(rows, 4096); + server.stop().await; +} + +fn integers(rows: usize) -> RecordBatch { + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), + vec![Arc::new(Int64Array::from_iter_values(0..rows as i64))], + ) + .unwrap() +} + +#[tokio::test] +async fn repeated_chunk_options_enable_if_any_and_use_the_first_preference() { + let batch = integers(3000); + let engine = FixtureEngine::default().with_batches(42, batch.schema(), vec![batch.clone()]); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + for (options, size) in [ + ( + vec![chunking(false, Some(1024)), chunking(true, None)], + 1024, + ), + ( + vec![chunking(false, None), chunking(true, Some(2048))], + 2048, + ), + ( + vec![chunking(true, Some(1023)), chunking(true, Some(1024))], + DEFAULT_ARROW_CHUNK_SIZE, + ), + ] { + let mut request = range(&Uuid::new_v4().to_string(), 42); + request.request_options = options; + let responses = collect_result(&mut server, request).await; + let payloads: Vec<_> = responses + .iter() + .filter_map(|response| match &response.response_type { + Some(ResponseType::ArrowBatch(batch)) => Some(batch.data.len()), + _ => None, + }) + .collect(); + if size < DEFAULT_ARROW_CHUNK_SIZE { + assert_eq!(payloads[0], size); + } else { + assert_eq!(payloads.len(), 1); + } + assert!(payloads.iter().all(|len| *len <= size)); + let output = assembled_batches(&responses, true); + assert_eq!(output.len(), 1); + let mut reader = StreamReader::try_new(output[0].data.as_ref(), None).unwrap(); + assert_eq!(reader.next().unwrap().unwrap(), batch); + } + server.stop().await; +} + +async fn collect_small_slices(backing: RecordBatch) -> Vec { + let batches = (0..20) + .map(|index| backing.slice(index * 1000, 1000)) + .collect(); + let engine = FixtureEngine::default().with_batches(42, backing.schema(), batches); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let output = collect_result(&mut server, range(&Uuid::new_v4().to_string(), 42)).await; + let (mut batches, mut rows) = (Vec::new(), 0); + for message in output { + // A small slice must not serialize the much larger retained backing allocation. + assert!(message.encoded_len() <= 3 * 1024 * 1024); + if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + message.response_type + { + assert_eq!(batch.start_offset, Some(rows)); + assert_eq!(batch.row_count, 1000); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + let decoded = reader.next().unwrap().unwrap(); + assert_eq!(decoded.num_rows() as i64, batch.row_count); + batches.push(decoded); + assert!(reader.next().is_none()); + rows += batch.row_count; + } + } + assert_eq!(batches.len(), 20); + assert_eq!(rows, 20_000); + server.stop().await; + batches +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn slices_of_a_large_allocation_remain_one_response_per_small_batch() { + for (index, batch) in collect_small_slices(integers(10_000_000)) + .await + .iter() + .enumerate() + { + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let start = index as i64 * 1000; + assert!(values.values().iter().copied().eq(start..start + 1000)); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn nested_slices_preserve_referenced_child_values() { + // The parent needs only 20,000 rows to retain the full 80 MB child. Exercise + // each offset width and maps sequentially, sharing the same allocation. + let child: ArrayRef = Arc::new(Int64Array::from_iter_values(0..10_000_000)); + for kind in 0..3 { + let item = Arc::new(Field::new("item", DataType::Int64, false)); + let column: ArrayRef = match kind { + 0 => Arc::new(ListArray::new( + item, + OffsetBuffer::new((0..=20_000i32).collect::>().into()), + child.clone(), + None, + )), + 1 => Arc::new(LargeListArray::new( + item, + OffsetBuffer::new((0..=20_000i64).collect::>().into()), + child.clone(), + None, + )), + _ => { + let fields = vec![ + Field::new("key", DataType::Int64, false), + Field::new("value", DataType::Int64, false), + ]; + let entries = StructArray::new( + fields.clone().into(), + vec![child.clone(), child.clone()], + None, + ); + Arc::new(MapArray::new( + Arc::new(Field::new( + "entries", + DataType::Struct(fields.into()), + false, + )), + OffsetBuffer::new((0..=20_000i32).collect::>().into()), + entries, + None, + false, + )) + } + }; + let schema = Arc::new(Schema::new(vec![Field::new( + "nested", + column.data_type().clone(), + false, + )])); + let backing = RecordBatch::try_new(schema, vec![column]).unwrap(); + let batches = collect_small_slices(backing).await; + let mut row = 0i64; + for batch in batches { + for index in 0..batch.num_rows() { + let value = match kind { + 0 => batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(index), + 1 => batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(index), + _ => { + let entry = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(index); + assert_eq!(entry.len(), 1); + let key = entry + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(key.value(0), row); + entry.column(1).clone() + } + }; + let value = value.as_any().downcast_ref::().unwrap(); + assert_eq!(value.len(), 1); + assert_eq!(value.value(0), row); + row += 1; + } + } + assert_eq!(row, 20_000); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_slices_preserve_current_values() { + fn text(row: usize) -> String { + format!("{row:010}{}", "y".repeat(30)) + } + + let values = StringViewArray::from_iter_values((0..1_000_000).map(text)); + let (views, buffers, nulls) = values.into_parts(); + for binary in [false, true] { + let column: ArrayRef = if binary { + Arc::new(BinaryViewArray::new( + views.clone(), + buffers.clone(), + nulls.clone(), + )) + } else { + Arc::new(StringViewArray::new( + views.clone(), + buffers.clone(), + nulls.clone(), + )) + }; + let schema = Arc::new(Schema::new(vec![Field::new( + "view", + column.data_type().clone(), + false, + )])); + let backing = RecordBatch::try_new(schema, vec![column]).unwrap(); + let batches = collect_small_slices(backing).await; + let mut row = 0; + for batch in batches { + for index in 0..batch.num_rows() { + let expected = text(row); + if binary { + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(values.value(index), expected.as_bytes()); + } else { + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(values.value(index), expected); + } + row += 1; + } + } + assert_eq!(row, 20_000); + } +} + +#[cfg(target_pointer_width = "64")] +#[tokio::test] +async fn unrepresentable_engine_batches_preserve_preceding_results_and_fail_cleanly() { + use arrow::array::NullArray; + use arrow::record_batch::RecordBatchOptions; + + let rows = i64::MAX as usize + 1; + for null_column in [false, true] { + let oversized = if null_column { + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("nulls", DataType::Null, true)])), + vec![Arc::new(NullArray::new(rows))], + ) + .unwrap() + } else { + RecordBatch::try_new_with_options( + Arc::new(Schema::empty()), + vec![], + &RecordBatchOptions::new().with_row_count(Some(rows)), + ) + .unwrap() + }; + let engine = FixtureEngine::default().with_batches( + 42, + oversized.schema(), + vec![oversized.slice(0, 3), oversized], + ); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let mut stream = server + .client + .execute_plan(range(&Uuid::new_v4().to_string(), 42)) + .await + .unwrap() + .into_inner(); + let (status, counts) = tokio::time::timeout(Duration::from_secs(15), async { + let mut counts = Vec::new(); + loop { + match stream.message().await { + Ok(Some(message)) => match message.response_type { + Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) => { + assert_eq!(batch.start_offset, Some(0)); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + assert_eq!(reader.next().unwrap().unwrap().num_rows(), 3); + assert!(reader.next().is_none()); + counts.push(batch.row_count); + } + Some(proto::execute_plan_response::ResponseType::ResultComplete(_)) => { + panic!("unrepresentable output must not complete successfully"); + } + _ => {} + }, + Err(status) => break (status, counts), + Ok(None) => panic!("expected row-count overflow status"), + } + } + }) + .await + .unwrap(); + assert_eq!(status.code(), Code::ResourceExhausted); + assert_eq!(status.message(), "result batch row count exceeds i64"); + assert_eq!(counts, vec![3]); + server.stop().await; + } +} + +#[tokio::test] +async fn configured_maximum_bounds_payloads_and_zero_disables_chunking() { + assert_eq!( + spark_connect_server::ServerOptions::default().max_arrow_chunk_size, + DEFAULT_ARROW_CHUNK_SIZE + ); + let batch = integers(5000); + let bytes = encoded(&batch, None); + for maximum in [4096, 0] { + let engine = FixtureEngine::default().with_results( + 42, + batch.schema(), + vec![Batch::Ipc(bytes.clone(), 5000)], + ); + let mut options = spark_connect_server::ServerOptions::default(); + options.max_arrow_chunk_size = maximum; + let mut server = Server::start_with_options(Arc::new(engine), options).await; + for preferred in [None, Some(2048), Some(4096), Some(4097)] { + let mut request = range(&Uuid::new_v4().to_string(), 42); + request.request_options = vec![chunking(true, preferred)]; + let responses = collect_result(&mut server, request).await; + let output = assembled_batches(&responses, maximum != 0); + assert_eq!(output.len(), 1); + assert_eq!(output[0].data.as_ref(), bytes.as_slice()); + let chunks: Vec<_> = responses + .iter() + .filter_map(|response| match &response.response_type { + Some(ResponseType::ArrowBatch(batch)) => { + Some((response.encoded_len(), batch.data.len())) + } + _ => None, + }) + .collect(); + if maximum == 0 { + assert_eq!(chunks.len(), 1); + } else { + let limit = preferred + .filter(|size| (1024..=maximum as i64).contains(size)) + .unwrap_or(maximum as i64) as usize; + assert!(chunks.iter().all(|(_, payload)| *payload <= limit)); + assert_eq!(chunks[0].1, limit); + assert!( + chunks[0].0 > limit, + "the chunk limit excludes the protobuf envelope" + ); + } + } + server.stop().await; + } +} diff --git a/crates/spark-connect-server/tests/schema_consistency.rs b/crates/spark-connect-server/tests/schema_consistency.rs new file mode 100644 index 00000000..11af295a --- /dev/null +++ b/crates/spark-connect-server/tests/schema_consistency.rs @@ -0,0 +1,405 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Result schemas remain consumable as one Arrow table across output batches. + +#[path = "support/arrow.rs"] +mod arrow_support; +#[path = "support/requests.rs"] +mod requests; +#[path = "support/server.rs"] +mod server; +mod support; + +use std::collections::HashMap; +use std::io::Cursor; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::{ + Array, ArrayRef, BinaryViewArray, LargeBinaryArray, LargeListArray, LargeStringArray, + ListArray, MapArray, StringArray, StringViewArray, StructArray, +}; +use arrow::buffer::OffsetBuffer; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::ipc::{reader::StreamReader, writer::StreamWriter}; +use arrow::record_batch::RecordBatch; +use prost::Message; +use spark_connect_proto as proto; +use tonic::{Code, Status}; +use uuid::Uuid; + +use arrow_support::{assembled_batches, chunking}; +use server::Server; +use support::{Batch, FixtureEngine}; + +const CHUNK_PREFERENCE: usize = 3 * 1024 * 1024; +const LARGE_ROWS: usize = 4096; + +struct BatchCase { + schema: SchemaRef, + batches: Vec, +} + +fn ipc(batch: &RecordBatch) -> Vec { + let mut data = Vec::new(); + let mut writer = StreamWriter::try_new(&mut data, batch.schema().as_ref()).unwrap(); + writer.write(batch).unwrap(); + writer.finish().unwrap(); + drop(writer); + data +} + +async fn execute(engine: BatchCase) -> (Vec, Option) { + let engine = FixtureEngine::default().with_results(1, engine.schema, engine.batches); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let mut request = requests::request(&Uuid::new_v4().to_string(), 1); + request.request_options = vec![chunking(true, Some(CHUNK_PREFERENCE as i64))]; + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + let result = tokio::time::timeout(Duration::from_secs(30), async { + let mut responses = Vec::new(); + loop { + match stream.message().await { + Ok(Some(response)) => { + if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + &response.response_type + { + assert!(batch.data.len() <= CHUNK_PREFERENCE); + assert!(response.encoded_len() <= batch.data.len() + 512); + } else { + // Schema and command envelopes retain their independent 3 MiB cap. + assert!(response.encoded_len() <= 3 * 1024 * 1024); + } + responses.push(response); + } + Ok(None) => break (responses, None), + Err(error) => break (responses, Some(error)), + } + } + }) + .await + .expect("schema fixture execution must terminate"); + server.stop().await; + result +} + +fn output_batches(responses: &[proto::ExecutePlanResponse]) -> Vec { + assembled_batches(responses, true) + .into_iter() + .map(|batch| { + let mut reader = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); + let decoded = reader.next().unwrap().unwrap(); + assert_eq!(decoded.num_rows() as i64, batch.row_count); + assert!(reader.next().is_none()); + decoded + }) + .collect() +} + +fn assert_schema_error(responses: &[proto::ExecutePlanResponse], status: Option) { + let status = status.expect("a changed output schema must fail"); + assert_eq!(status.code(), Code::Internal); + assert_eq!( + status.message(), + "Arrow output schema changed within one result" + ); + assert!(!completed(responses)); +} + +fn completed(responses: &[proto::ExecutePlanResponse]) -> bool { + responses.iter().any(|response| { + matches!( + response.response_type, + Some(proto::execute_plan_response::ResponseType::ResultComplete( + _ + )) + ) + }) +} + +fn strings(data_type: DataType, nullable: bool) -> RecordBatch { + let values: ArrayRef = match data_type { + DataType::Utf8 => Arc::new(StringArray::from(vec!["value"])), + DataType::LargeUtf8 => Arc::new(LargeStringArray::from(vec!["value"])), + _ => unreachable!(), + }; + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("value", data_type, nullable)])), + vec![values], + ) + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn changed_physical_types_and_nullability_fail_before_the_second_batch() { + let first = strings(DataType::Utf8, false); + for second in [ + strings(DataType::LargeUtf8, false), + strings(DataType::Utf8, true), + ] { + let (responses, status) = execute(BatchCase { + schema: first.schema(), + batches: vec![Batch::Arrow(first.clone()), Batch::Arrow(second)], + }) + .await; + assert_schema_error(&responses, status); + let batches = output_batches(&responses); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].schema(), first.schema()); + } +} + +fn metadata_batch(note: &str) -> RecordBatch { + let metadata = HashMap::from([("producer.note".to_string(), note.to_string())]); + let item = + Arc::new(Field::new("item", DataType::LargeUtf8, true).with_metadata(metadata.clone())); + let values = ListArray::new( + item.clone(), + OffsetBuffer::new(vec![0, 1].into()), + Arc::new(LargeStringArray::from(vec!["value"])), + None, + ); + RecordBatch::try_new( + Arc::new(Schema::new_with_metadata( + vec![Field::new("value", DataType::List(item), true).with_metadata(metadata.clone())], + metadata, + )), + vec![Arc::new(values)], + ) + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn metadata_can_change_and_batch_schema_can_differ_from_the_declaration() { + let declared = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::List(Arc::new(Field::new("item", DataType::Utf8, false))), + false, + )])); + let (responses, status) = execute(BatchCase { + schema: declared, + batches: vec![ + Batch::Arrow(metadata_batch("first")), + Batch::Arrow(metadata_batch("second")), + ], + }) + .await; + assert!(status.is_none(), "{status:?}"); + assert!(completed(&responses)); + let batches = output_batches(&responses); + assert_eq!(batches.len(), 2); + for (batch, note) in batches.iter().zip(["first", "second"]) { + let schema = batch.schema(); + assert_eq!(schema.metadata()["producer.note"], note); + assert_eq!(schema.field(0).metadata()["producer.note"], note); + let DataType::List(item) = schema.field(0).data_type() else { + panic!("expected list output"); + }; + assert_eq!(item.metadata()["producer.note"], note); + assert_eq!(item.data_type(), &DataType::LargeUtf8); + assert!(schema.field(0).is_nullable()); + assert!(item.is_nullable()); + } +} + +fn text(row: usize) -> String { + format!("{row:04}{}", "x".repeat(1020)) +} + +fn views(rows: usize) -> RecordBatch { + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Utf8View, + false, + )])), + vec![Arc::new(StringViewArray::from_iter_values( + (0..rows).map(text), + ))], + ) + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_ipc_remains_unchanged_for_small_and_oversized_batches() { + for rows in [2, LARGE_ROWS] { + let batch = views(rows); + let data = ipc(&batch); + assert_eq!(data.len() > CHUNK_PREFERENCE, rows == LARGE_ROWS); + let (responses, status) = execute(BatchCase { + schema: batch.schema(), + batches: vec![Batch::Ipc(data.clone(), rows as i64)], + }) + .await; + assert!(status.is_none(), "{status:?}"); + assert!(completed(&responses)); + let output = assembled_batches(&responses, true); + assert_eq!(output.len(), 1); + assert_eq!(output[0].data.as_ref(), data.as_slice()); + let batches = output_batches(&responses); + assert_eq!(batches, vec![batch]); + assert_eq!( + batches[0].schema().field(0).data_type(), + &DataType::Utf8View + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn mixing_small_and_oversized_view_ipc_preserves_one_output_schema() { + let small = views(2); + let large = views(LARGE_ROWS); + for large_first in [false, true] { + let expected = if large_first { + vec![large.clone(), small.clone()] + } else { + vec![small.clone(), large.clone()] + }; + let (responses, status) = execute(BatchCase { + schema: small.schema(), + batches: expected + .iter() + .map(|batch| Batch::Ipc(ipc(batch), batch.num_rows() as i64)) + .collect(), + }) + .await; + assert!(status.is_none(), "{status:?}"); + assert!(completed(&responses)); + assert_eq!(output_batches(&responses), expected); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn arrow_view_children_are_materialized_before_byte_chunking() { + let values: ArrayRef = Arc::new(StringViewArray::from_iter_values((0..LARGE_ROWS).map(text))); + let offsets = OffsetBuffer::new((0..=LARGE_ROWS as i32).collect::>().into()); + let item = Arc::new(Field::new("item", DataType::Utf8View, false)); + let lists = ListArray::new(item.clone(), offsets.clone(), values.clone(), None); + let large_lists = LargeListArray::new( + item, + OffsetBuffer::new((0..=LARGE_ROWS as i64).collect::>().into()), + values.clone(), + None, + ); + let entries = StructArray::new( + vec![ + Field::new("key", DataType::Utf8View, false), + Field::new("value", DataType::BinaryView, false), + ] + .into(), + vec![ + values, + Arc::new(BinaryViewArray::from_iter_values( + (0..LARGE_ROWS).map(|row| text(row).into_bytes()), + )), + ], + None, + ); + let maps = MapArray::new( + Arc::new(Field::new("entries", entries.data_type().clone(), false)), + offsets, + entries, + None, + false, + ); + let columns: Vec = vec![Arc::new(lists), Arc::new(large_lists), Arc::new(maps)]; + let schema = Arc::new(Schema::new( + columns + .iter() + .enumerate() + .map(|(index, column)| { + Field::new(format!("value{index}"), column.data_type().clone(), false) + }) + .collect::>(), + )); + let batch = RecordBatch::try_new(schema.clone(), columns).unwrap(); + let (responses, status) = execute(BatchCase { + schema, + batches: vec![Batch::Arrow(batch)], + }) + .await; + assert!(status.is_none(), "{status:?}"); + assert!(completed(&responses)); + let batches = output_batches(&responses); + assert_eq!(batches.len(), 1); + assert!( + assembled_batches(&responses, true)[0] + .num_chunks_in_batch + .unwrap() + > 1 + ); + let mut offset = 0; + for batch in batches { + let lists = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let large_lists = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let maps = batch.column(2).as_any().downcast_ref::().unwrap(); + for row in 0..batch.num_rows() { + let expected = text(offset + row); + let list = lists.value(row); + assert_eq!( + list.as_any() + .downcast_ref::() + .unwrap() + .value(0), + expected + ); + let large_list = large_lists.value(row); + assert_eq!( + large_list + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + expected + ); + let entries = maps.value(row); + assert_eq!( + entries + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + expected + ); + assert_eq!( + entries + .column(1) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + expected.as_bytes() + ); + } + offset += batch.num_rows(); + } + assert_eq!(offset, LARGE_ROWS); +} diff --git a/crates/spark-connect-server/tests/server_shutdown.rs b/crates/spark-connect-server/tests/server_shutdown.rs new file mode 100644 index 00000000..ac624c93 --- /dev/null +++ b/crates/spark-connect-server/tests/server_shutdown.rs @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Duration; + +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use futures::Stream; +use spark_connect_proto as proto; +use spark_connect_server::engine::{ + BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, + ResultBatch, SessionContext, +}; +use tonic::Status; + +#[derive(Default)] +struct Spy { + streams_started: AtomicUsize, + streams_dropped: AtomicUsize, + configs_started: AtomicUsize, + configs_dropped: AtomicUsize, + sessions_dropped: AtomicUsize, +} + +struct PendingEngine(Arc); + +impl Engine for PendingEngine { + fn create_session(&self, _context: SessionContext) -> Result, Status> { + Ok(Arc::new(PendingSession(self.0.clone()))) + } +} + +struct PendingSession(Arc); + +impl Drop for PendingSession { + fn drop(&mut self) { + self.0.sessions_dropped.fetch_add(1, Ordering::SeqCst); + } +} + +#[tonic::async_trait] +impl EngineSession for PendingSession { + async fn prepare(&self, _operation: proto::plan::OpType) -> Result { + Ok(PreparedOperation::Query(Box::new(PendingQuery( + self.0.clone(), + )))) + } + + async fn analyze( + &self, + _input: proto::analyze_plan_request::Analyze, + ) -> Result { + Err(Status::unimplemented("shutdown fixture does not analyze")) + } + + async fn config( + &self, + _input: proto::config_request::operation::OpType, + ) -> Result { + let _call = PendingConfig(self.0.clone()); + self.0.configs_started.fetch_add(1, Ordering::SeqCst); + std::future::pending().await + } +} + +struct PendingConfig(Arc); + +impl Drop for PendingConfig { + fn drop(&mut self) { + self.0.configs_dropped.fetch_add(1, Ordering::SeqCst); + } +} + +struct PendingQuery(Arc); + +#[tonic::async_trait] +impl PreparedQuery for PendingQuery { + fn schema(&self) -> SchemaRef { + Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])) + } + + async fn execute(self: Box) -> Result { + self.0.streams_started.fetch_add(1, Ordering::SeqCst); + Ok(Box::pin(PendingResults(self.0.clone()))) + } +} + +struct PendingResults(Arc); + +impl Stream for PendingResults { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Pending + } +} + +impl Drop for PendingResults { + fn drop(&mut self) { + self.0.streams_dropped.fetch_add(1, Ordering::SeqCst); + } +} + +async fn wait_for_count(counter: &AtomicUsize, expected: usize) { + tokio::time::timeout(Duration::from_secs(5), async { + while counter.load(Ordering::SeqCst) != expected { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("backend work must finish without disconnecting its client"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn aborting_serve_cancels_backend_work_before_clients_disconnect() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let spy = Arc::new(Spy::default()); + let server = tokio::spawn(spark_connect_server::serve_with_engine( + listener, + Arc::new(PendingEngine(spy.clone())), + std::future::pending(), + )); + let mut client = proto::spark_connect_service_client::SparkConnectServiceClient::connect( + format!("http://{address}"), + ) + .await + .unwrap(); + let session = uuid::Uuid::new_v4().to_string(); + let mut result = client + .execute_plan(proto::ExecutePlanRequest { + session_id: session.clone(), + operation_id: Some(uuid::Uuid::new_v4().to_string()), + plan: Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(proto::Relation::default())), + }), + ..Default::default() + }) + .await + .unwrap() + .into_inner(); + assert!(result.message().await.unwrap().unwrap().schema.is_some()); + wait_for_count(&spy.streams_started, 1).await; + + let mut config_client = client.clone(); + let config = tokio::spawn(async move { + config_client + .config(proto::ConfigRequest { + session_id: session, + operation: Some(proto::config_request::Operation { + op_type: Some(proto::config_request::operation::OpType::GetAll( + Default::default(), + )), + }), + ..Default::default() + }) + .await + }); + wait_for_count(&spy.configs_started, 1).await; + server.abort(); + assert!(server.await.unwrap_err().is_cancelled()); + + // Retain both the streamed response and its client connection: cancellation + // must come from dropping serve_with_engine, not from a peer disconnect. + wait_for_count(&spy.streams_dropped, 1).await; + wait_for_count(&spy.configs_dropped, 1).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert!(tokio::time::timeout(Duration::from_secs(5), config) + .await + .unwrap() + .unwrap() + .is_err()); + drop(result); + drop(client); +} diff --git a/crates/spark-connect-server/tests/support/arrow.rs b/crates/spark-connect-server/tests/support/arrow.rs new file mode 100644 index 00000000..c9fd6979 --- /dev/null +++ b/crates/spark-connect-server/tests/support/arrow.rs @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Wire chunk negotiation, reassembly and IPC encoding shared by protocol tests. + +#![allow(dead_code)] + +use std::collections::HashSet; + +use arrow::ipc::writer::{IpcWriteOptions, StreamWriter}; +use arrow::ipc::CompressionType; +use arrow::record_batch::RecordBatch; +use prost::Message; +use proto::execute_plan_response::ResponseType; +use spark_connect_proto as proto; + +pub const DEFAULT_ARROW_CHUNK_SIZE: usize = 128 * 1024 * 1024 * 9 / 10; + +pub fn chunking(allow: bool, preferred: Option) -> proto::execute_plan_request::RequestOption { + proto::execute_plan_request::RequestOption { + request_option: Some( + proto::execute_plan_request::request_option::RequestOption::ResultChunkingOptions( + proto::ResultChunkingOptions { + allow_arrow_batch_chunking: allow, + preferred_arrow_chunk_size: preferred, + }, + ), + ), + } +} + +// Reassemble before Arrow decoding, just as Spark clients do. Every chunk counts +// the whole batch's rows and has its own response ID; row offsets advance once. +pub fn assembled_batches( + responses: &[proto::ExecutePlanResponse], + negotiated: bool, +) -> Vec { + let mut output = Vec::new(); + let mut pending: Option = None; + let (mut offset, mut index) = (0, 0); + let mut data = Vec::new(); + let mut response_ids = HashSet::new(); + for response in responses { + assert!(response_ids.insert(&response.response_id)); + let Some(ResponseType::ArrowBatch(chunk)) = &response.response_type else { + continue; + }; + // The negotiated limit covers IPC payload bytes, matching the JVM. The + // protobuf envelope adds identifiers, field tags and chunk metadata. + assert!(response.encoded_len() <= chunk.data.len() + 512); + if negotiated { + assert!(chunk.data.len() <= DEFAULT_ARROW_CHUNK_SIZE); + } + assert_eq!(chunk.start_offset, Some(offset)); + if !negotiated { + assert_eq!(chunk.chunk_index, None); + assert_eq!(chunk.num_chunks_in_batch, None); + offset += chunk.row_count; + output.push(chunk.clone()); + continue; + } + assert_eq!(chunk.chunk_index, Some(index)); + let count = chunk.num_chunks_in_batch.unwrap(); + assert!(count > index); + if let Some(batch) = &mut pending { + assert_eq!(batch.row_count, chunk.row_count); + assert_eq!(batch.num_chunks_in_batch, Some(count)); + } else { + pending = Some(chunk.clone()); + } + data.extend_from_slice(&chunk.data); + index += 1; + if index == count { + let mut batch = pending.take().unwrap(); + batch.data = std::mem::take(&mut data).into(); + offset += batch.row_count; + output.push(batch); + index = 0; + } + } + assert!(pending.is_none(), "complete must not truncate a batch"); + output +} + +pub fn encoded(batch: &RecordBatch, compression: Option) -> Vec { + let options = IpcWriteOptions::default() + .try_with_compression(compression) + .unwrap(); + let mut data = Vec::new(); + { + let mut writer = + StreamWriter::try_new_with_options(&mut data, batch.schema().as_ref(), options) + .unwrap(); + writer.write(batch).unwrap(); + writer.finish().unwrap(); + } + data +} diff --git a/crates/spark-connect-server/tests/support/mod.rs b/crates/spark-connect-server/tests/support/mod.rs new file mode 100644 index 00000000..6723e0aa --- /dev/null +++ b/crates/spark-connect-server/tests/support/mod.rs @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Explicit backend outputs for frontend protocol and schema tests. +//! +//! Range.end is only a fixture lookup key. This backend neither interprets SQL +//! nor implements Spark operators; each case supplies batches or a chosen error. +//! Test binaries use different subsets of the fixture constructors. + +#![allow(dead_code)] + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow::array::Int64Array; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::record_batch::RecordBatch; +use spark_connect_proto as proto; +use spark_connect_server::engine::{ + BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, + ResultBatch, SessionContext, +}; +use tonic::{Code, Status}; + +#[derive(Clone)] +pub enum Batch { + Arrow(RecordBatch), + Ipc(Vec, i64), +} + +#[derive(Clone)] +enum Output { + Query { + schema: SchemaRef, + batches: Vec, + error: Option, + }, + Rejected(Code), +} + +#[derive(Clone)] +pub struct FixtureEngine { + outputs: HashMap, +} + +impl Default for FixtureEngine { + fn default() -> Self { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let batch = |start, end| { + RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int64Array::from_iter_values(start..end))], + ) + .unwrap() + }; + Self { + outputs: HashMap::new(), + } + .with_batches(0, schema.clone(), Vec::new()) + .with_batches(1, schema.clone(), vec![batch(0, 1)]) + .with_batches( + 25_000, + schema.clone(), + vec![batch(0, 8192), batch(8192, 16_384), batch(16_384, 25_000)], + ) + } +} + +impl FixtureEngine { + pub fn with_batches(self, selector: i64, schema: SchemaRef, batches: Vec) -> Self { + self.with_results( + selector, + schema, + batches.into_iter().map(Batch::Arrow).collect(), + ) + } + + pub fn with_results(mut self, selector: i64, schema: SchemaRef, batches: Vec) -> Self { + self.outputs.insert( + selector, + Output::Query { + schema, + batches, + error: None, + }, + ); + self + } + + pub fn reject(mut self, selector: i64, code: Code) -> Self { + self.outputs.insert(selector, Output::Rejected(code)); + self + } + + pub fn stream_error(mut self, selector: i64, code: Code) -> Self { + self.outputs.insert( + selector, + Output::Query { + schema: Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), + batches: Vec::new(), + error: Some(code), + }, + ); + self + } +} + +impl Engine for FixtureEngine { + fn create_session(&self, _: SessionContext) -> Result, Status> { + Ok(Arc::new(self.clone())) + } +} + +#[tonic::async_trait] +impl EngineSession for FixtureEngine { + async fn prepare(&self, operation: proto::plan::OpType) -> Result { + let proto::plan::OpType::Root(relation) = operation else { + return Err(Status::unimplemented("fixture accepts root selectors only")); + }; + let Some(proto::relation::RelType::Range(selector)) = relation.rel_type else { + return Err(Status::unimplemented("fixture requires a range selector")); + }; + match self.outputs.get(&selector.end) { + Some(Output::Query { + schema, + batches, + error, + }) => Ok(PreparedOperation::Query(Box::new(FixtureQuery { + schema: schema.clone(), + batches: batches.clone(), + error: *error, + }))), + Some(Output::Rejected(code)) => Err(Status::new(*code, "prepared fixture error")), + None => Err(Status::unimplemented("unknown fixture selector")), + } + } + + async fn analyze( + &self, + _: proto::analyze_plan_request::Analyze, + ) -> Result { + Err(Status::unimplemented( + "fixture does not analyze query semantics", + )) + } + + async fn config( + &self, + _: proto::config_request::operation::OpType, + ) -> Result { + Err(Status::unimplemented( + "fixture does not configure query semantics", + )) + } +} + +struct FixtureQuery { + schema: SchemaRef, + batches: Vec, + error: Option, +} + +#[tonic::async_trait] +impl PreparedQuery for FixtureQuery { + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + async fn execute(self: Box) -> Result { + let batches = self.batches.into_iter().map(|batch| { + Ok(match batch { + Batch::Arrow(batch) => ResultBatch::Arrow(batch), + Batch::Ipc(data, row_count) => ResultBatch::Ipc { data, row_count }, + }) + }); + let failure = self + .error + .into_iter() + .map(|code| Err(Status::new(code, "streamed fixture error"))); + Ok(Box::pin(futures::stream::iter(batches.chain(failure)))) + } +} diff --git a/crates/spark-connect-server/tests/support/requests.rs b/crates/spark-connect-server/tests/support/requests.rs new file mode 100644 index 00000000..185fce4a --- /dev/null +++ b/crates/spark-connect-server/tests/support/requests.rs @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared request envelopes and terminal-stream assertions. Individual test +//! binaries use different subsets of these helpers. + +#![allow(dead_code)] + +use std::time::Duration; + +use proto::execute_plan_response::ResponseType; +use spark_connect_proto as proto; +use tonic::Code; +use uuid::Uuid; + +pub fn user(id: &str) -> Option { + Some(proto::UserContext { + user_id: id.into(), + ..Default::default() + }) +} + +pub fn relation(end: i64) -> proto::Relation { + proto::Relation { + rel_type: Some(proto::relation::RelType::Range(proto::Range { + end, + step: 1, + ..Default::default() + })), + ..Default::default() + } +} + +pub fn request(session: &str, end: i64) -> proto::ExecutePlanRequest { + proto::ExecutePlanRequest { + session_id: session.into(), + user_context: user("alice"), + operation_id: Some(Uuid::new_v4().to_string()), + plan: Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(relation(end))), + }), + ..Default::default() + } +} + +pub fn config_request( + session: &str, + input: proto::config_request::operation::OpType, +) -> proto::ConfigRequest { + proto::ConfigRequest { + session_id: session.into(), + user_context: user("alice"), + operation: Some(proto::config_request::Operation { + op_type: Some(input), + }), + ..Default::default() + } +} + +pub fn release(session: &str, operation: &str) -> proto::ReleaseExecuteRequest { + proto::ReleaseExecuteRequest { + session_id: session.into(), + user_context: user("alice"), + operation_id: operation.into(), + release: Some(proto::release_execute_request::Release::ReleaseAll( + proto::release_execute_request::ReleaseAll {}, + )), + ..Default::default() + } +} + +pub fn release_session(session: &str) -> proto::ReleaseSessionRequest { + proto::ReleaseSessionRequest { + session_id: session.into(), + user_context: user("alice"), + ..Default::default() + } +} + +pub async fn assert_cancelled(stream: &mut tonic::Streaming) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + match stream.message().await { + Err(status) => { + assert_eq!(status.code(), Code::Cancelled); + break; + } + Ok(Some(response)) => assert!(!matches!( + response.response_type, + Some(ResponseType::ResultComplete(_)) + )), + Ok(None) => panic!("cancelled execution must report cancellation"), + } + } + }) + .await + .expect("cancelled execution must terminate"); +} diff --git a/crates/spark-connect-server/tests/support/server.rs b/crates/spark-connect-server/tests/support/server.rs new file mode 100644 index 00000000..74602859 --- /dev/null +++ b/crates/spark-connect-server/tests/support/server.rs @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use spark_connect_proto::spark_connect_service_client::SparkConnectServiceClient; +use spark_connect_server::engine::Engine; +use tokio::sync::oneshot; +use tokio::task::JoinHandle; +use tonic::transport::Channel; + +pub struct Server { + // Some test binaries need a second connection to the same listener. + #[allow(dead_code)] + pub address: SocketAddr, + pub client: SparkConnectServiceClient, + pub shutdown: Option>, + task: Option>, +} + +impl Server { + pub async fn start_with_engine(engine: Arc) -> Self { + Self::start_with_options(engine, spark_connect_server::ServerOptions::default()).await + } + + pub async fn start_with_options( + engine: Arc, + options: spark_connect_server::ServerOptions, + ) -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (shutdown, stopped) = oneshot::channel(); + let task = tokio::spawn(async move { + spark_connect_server::serve_with_engine_and_options(listener, engine, options, async { + let _ = stopped.await; + }) + .await + .unwrap(); + }); + let client = SparkConnectServiceClient::connect(format!("http://{address}")) + .await + .unwrap(); + Self { + address, + client, + shutdown: Some(shutdown), + task: Some(task), + } + } + + pub async fn stop(mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + tokio::time::timeout(Duration::from_secs(5), self.task.take().unwrap()) + .await + .unwrap() + .unwrap(); + } +} + +impl Drop for Server { + fn drop(&mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + if let Some(task) = self.task.take() { + task.abort(); + } + } +} diff --git a/dev/design/ci.md b/dev/design/ci.md index 47ecc8a3..531267cb 100644 --- a/dev/design/ci.md +++ b/dev/design/ci.md @@ -16,13 +16,15 @@ The CI consists of these main workflows: and macOS universal2 wheels using the release recipe 4. **Coverage** (`.github/workflows/coverage.yml`) - Rust and Python coverage, with regression floors +5. **Experimental server frontend** (`.github/workflows/server-frontend.yml`) - Test the + optional Rust protocol frontend with stable Rust and MSRV 1.88 The other workflows are `lint.yml` (rustfmt, clippy, rustdoc, ruff), `generated-drift.yml`, `vendor-drift.yml`, `docs.yml`, `pandas-parity.yml`, `pipelines-parity.yml` and `release.yml`. Most workflows run on pushes and pull requests targeting `master` and `branch-*`. Several also -accept manual dispatch. `docs.yml` and `wheel-build-checks.yml` run only when files matching their -path filters change. The release wheel build checks cover changes to packaged Python sources, +accept manual dispatch. `docs.yml`, `wheel-build-checks.yml` and `server-frontend.yml` filter push +and pull-request runs to matching paths. The release wheel build checks cover packaged Python sources, protocol definitions, dependencies and build configuration. `release.yml` runs on a schedule and on manual dispatch. @@ -69,6 +71,18 @@ The same workflow has four more jobs: This job should always pass; failures indicate incomplete implementation or test regressions. +## Experimental server frontend (`server-frontend.yml`) + +This focused workflow runs on frontend, protocol, workspace manifest, +lockfile, Cargo configuration, toolchain and workflow changes. Each stable Rust and Rust +1.88.0 job has a 45-minute limit, runs `cargo test --locked -p spark-connect-server`, +and checks the protocol crate with its `server` feature both off and on. +The existing workspace lint job covers the frontend with its pinned Rust toolchain; +test jobs do not promote warnings from a floating or older compiler to errors. +Ubuntu 24.04 supplies a protobuf compiler with +proto3 optional support. Tests use injected engine fixtures and require no execution backend; +they cover frontend protocol behavior rather than Spark execution compatibility. + ## Python Connect Parity (`build_python_connect.yml`) **Status**: Gating. The parity and drop-in jobs pass only when every test that runs passes, except