From a09480f33c843e59fb98928724a8d6aa1f6c95c5 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 13:45:00 -0800 Subject: [PATCH 01/18] Add an experimental Rust Spark Connect protocol frontend --- .github/workflows/server-frontend.yml | 92 + Cargo.lock | 18 + Cargo.toml | 3 + README.md | 11 + SECURITY.md | 19 + crates/spark-connect-server/Cargo.toml | 46 + crates/spark-connect-server/README.md | 98 ++ crates/spark-connect-server/build.rs | 145 ++ crates/spark-connect-server/src/engine/mod.rs | 126 ++ crates/spark-connect-server/src/lib.rs | 63 + crates/spark-connect-server/src/schema.rs | 224 +++ crates/spark-connect-server/src/service.rs | 1232 +++++++++++++ .../tests/engine_injection.rs | 1528 +++++++++++++++++ crates/spark-connect-server/tests/protocol.rs | 903 ++++++++++ .../spark-connect-server/tests/support/mod.rs | 265 +++ dev/design/ci.md | 15 +- 16 files changed, 4786 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/server-frontend.yml create mode 100644 crates/spark-connect-server/Cargo.toml create mode 100644 crates/spark-connect-server/README.md create mode 100644 crates/spark-connect-server/build.rs create mode 100644 crates/spark-connect-server/src/engine/mod.rs create mode 100644 crates/spark-connect-server/src/lib.rs create mode 100644 crates/spark-connect-server/src/schema.rs create mode 100644 crates/spark-connect-server/src/service.rs create mode 100644 crates/spark-connect-server/tests/engine_injection.rs create mode 100644 crates/spark-connect-server/tests/protocol.rs create mode 100644 crates/spark-connect-server/tests/support/mod.rs diff --git a/.github/workflows/server-frontend.yml b/.github/workflows/server-frontend.yml new file mode 100644 index 00000000..7540c9ee --- /dev/null +++ b/.github/workflows/server-frontend.yml @@ -0,0 +1,92 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Server frontend CI + +on: + pull_request: + branches: [master, "branch-*"] + paths: &frontend-paths + - ".github/workflows/server-frontend.yml" + - "Cargo.toml" + - "Cargo.lock" + - ".cargo/**" + - "rust-toolchain*" + - "crates/spark-connect-server/**" + - "crates/spark-connect-proto/**" + - "crates/spark-connect-core/**" + - "crates/spark-connect/**" + push: + branches: [master, "branch-*"] + paths: *frontend-paths + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CARGO_BUILD_JOBS: "2" + CARGO_PROFILE_TEST_DEBUG: "0" + RUSTFLAGS: "-D warnings" + +jobs: + frontend: + name: Frontend (Rust ${{ matrix.rust-version }}) + runs-on: ubuntu-24.04 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + rust-version: [stable, "1.88.0"] + steps: + - uses: actions/checkout@v4 + + - name: Install Rust toolchain + # Use runner-provided rustup to keep actions limited to actions/*. + run: | + rustup toolchain install ${{ matrix.rust-version }} --profile minimal --component clippy + rustup default ${{ matrix.rust-version }} + + - name: Cache cargo and target + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-${{ runner.arch }}-cargo-server-frontend-${{ matrix.rust-version }}-${{ hashFiles('Cargo.lock') }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-cargo-server-frontend-${{ matrix.rust-version }}- + + - name: Install protobuf compiler + # Ubuntu 24.04 provides protoc with proto3 optional support (3.15+). + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + protoc --version + + - name: Test frontend + # Protocol and injected-engine tests use local fixtures without an execution backend. + run: cargo test --locked -p spark-connect-server + + - name: Strict Clippy for frontend + run: cargo clippy --locked -p spark-connect-server --all-targets --no-deps -- -D warnings diff --git a/Cargo.lock b/Cargo.lock index 40cf5dab..edfae584 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4048,6 +4048,24 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spark-connect-server" +version = "0.1.0" +dependencies = [ + "apache-spark-connect", + "apache-spark-connect-proto", + "arrow", + "futures", + "h2", + "prost", + "tokio", + "tokio-stream", + "tonic", + "tonic-build", + "tonic-prost", + "uuid", +] + [[package]] name = "sqlparser" version = "0.60.0" diff --git a/Cargo.toml b/Cargo.toml index fb6261a1..473aebeb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ members = [ "crates/spark-connect-build", "crates/pyspark-rs", "examples", + "crates/spark-connect-server", ] exclude = [ "examples/wasm-udfs", @@ -44,6 +45,8 @@ exclude = [ # (so `cargo test --workspace` stays green without it). Build them explicitly with # `cargo build --manifest-path examples/wasm-udfs/Cargo.toml` (etc) in CI or when # the target is installed locally. +# The experimental protocol frontend is opt-in; build or test it with +# `-p spark-connect-server`. It is included in workspace lint checks. default-members = [ "crates/spark-connect-proto", "crates/spark-connect-core", diff --git a/README.md b/README.md index d75af986..fcc755db 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,17 @@ fn main() -> Result<(), Box> { See the [WASM UDF guide](https://apache.github.io/spark-connect-rust/udfs/) for the full setup, SQL registration, supported types, and more. +## Experimental server frontend + +The unpublished [`spark-connect-server`](crates/spark-connect-server/README.md) +library provides a Rust Spark Connect protocol frontend. It manages protocol sessions, +operations and Arrow IPC responses, delegating planning and execution to an +application-provided `Engine`. The library includes no execution backend or standalone +server executable; its protocol tests use injected engine fixtures. + +The frontend is intended for local, trusted experiments. Read its +[security model](SECURITY.md#experimental-server-frontend) before embedding it. + ## Contributing Issues are tracked in ASF JIRA under diff --git a/SECURITY.md b/SECURITY.md index 80e53253..d80df8b4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -25,3 +25,22 @@ material, TLS validation) inherit from that model. Automated security scanning agents should consult that document for the project's in-scope / out-of-scope declarations before reporting issues. + +## Experimental server frontend + +The optional `spark-connect-server` library is an experimental protocol +frontend for local, trusted workloads. It does not provide authentication, +authorization or TLS. Session IDs, operation IDs and client-provided user +context identify protocol state; they do not establish a caller's identity +or access rights. + +Bind its listener only to loopback. The embedding application supplies the +listener and the `Engine` implementation; the library does not enforce the +listener's network scope. Any process that can reach it can submit requests +to that engine. Protocol limits and session isolation are resource and +correctness guards, not a security boundary between mutually untrusted users. + +This frontend does not include an execution backend or a standalone server. +Its injected-engine tests establish frontend protocol behavior, not safe +exposure of an execution engine to untrusted clients. Apache Spark's operator +security guidance remains relevant to any application that connects an engine. diff --git a/crates/spark-connect-server/Cargo.toml b/crates/spark-connect-server/Cargo.toml new file mode 100644 index 00000000..68def7fa --- /dev/null +++ b/crates/spark-connect-server/Cargo.toml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "spark-connect-server" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false +description = "Experimental Spark Connect server frontend with an injectable execution engine" + +[lints] +workspace = true + +[dependencies] +spark-connect-proto = { package = "apache-spark-connect-proto", path = "../spark-connect-proto" } +arrow.workspace = true +futures.workspace = true +prost.workspace = true +tokio = { workspace = true, features = ["net"] } +tokio-stream = { version = "0.1", features = ["net"] } +tonic = { workspace = true, features = ["server", "router"] } +tonic-prost.workspace = true +uuid.workspace = true + +[build-dependencies] +tonic-build = "0.14" + +[dev-dependencies] +h2 = "0.4" +spark-connect = { package = "apache-spark-connect", path = "../spark-connect" } diff --git a/crates/spark-connect-server/README.md b/crates/spark-connect-server/README.md new file mode 100644 index 00000000..f236fcc8 --- /dev/null +++ b/crates/spark-connect-server/README.md @@ -0,0 +1,98 @@ + + +# Experimental Spark Connect server frontend + +This unpublished library serves a subset of the Spark Connect protocol using the +workspace's `apache-spark-connect-proto` messages. The caller supplies an +`Engine` implementation and a bound Tokio TCP listener to `serve_with_engine`. +No execution engine or standalone server executable is included. + +Rust owns protocol sessions, operation IDs, response framing and result-stream +lifetimes. Each protocol session receives an `EngineSession` that owns its +planning, catalog and configuration state. Its `prepare` method returns a +`PreparedQuery` or command response payloads; the frontend adds protocol IDs and +`ResultComplete`. The engine decides which plans, commands and metadata requests +it can execute and reports unsupported inputs explicitly. + +## Protocol subset + +The frontend routes `ExecutePlan`, `AnalyzePlan` and `Config` to the engine. It +implements `ReleaseExecute` and `ReleaseSession`, including operation cancellation +through `ReleaseAll` and session recreation when `allow_reconnect` permits it. +Completed responses are not retained. Reattachment, artifact upload/status, +`Interrupt`, enriched error details, session cloning and status queries return +`UNIMPLEMENTED`. + +The frontend validates UUID session and operation IDs, rejects duplicate active +operation IDs within a session, and checks client-observed server session IDs. +Client user IDs distinguish session keys but are not authenticated identities. + +## Arrow results + +A prepared query declares an Arrow schema before execution. The frontend converts +supported types to the Spark schema representation and checks each batch against +that declaration. Equivalent Arrow representations and differences in batch +nullability or metadata are accepted. Declared field metadata and unsupported +types fail explicitly. + +Engines may yield Arrow record batches or complete Arrow IPC streams containing +one batch and its declared row count. The frontend encodes record batches and +forwards IPC that fits the response limit; oversized batches are split by rows. +Result envelopes, including schema and batch responses, are limited to 3 MiB. +A single row or schema that cannot fit fails with `RESOURCE_EXHAUSTED`. Each batch +carries a cumulative row offset, and empty results still include an empty batch. + +## Lifetime and trust requirements + +Engine methods and streams must cancel their owned work when dropped. The +frontend cancels them when a client disconnects, an operation is released, a +session is released or server shutdown begins. Results pass through a bounded +channel, so unread responses eventually stop polling the engine. Blocking Arrow +encoding retains operation ownership until its worker finishes. + +`ReleaseAll` waits up to two seconds for teardown and reports `DEADLINE_EXCEEDED` +if it is still draining. Shutdown cancels sessions and waits up to two seconds +for transport drain; returning from `serve_with_engine` does not prove that an +engine's independently scheduled or blocking work has finished. The caller must +honor the engine's cleanup contract before releasing its resources. + +This is an experiment for trusted local clients and trusted engine outputs. +Keep the supplied listener on loopback: this frontend supplies no authentication, +authorization or TLS. Requests can be up to 64 MiB. Active sessions and operations +have no admission limit or idle expiry; only released-session tombstones are +bounded, to the most recent 4096 distinct IDs. An evicted ID can create a new +session. These limits do not provide isolation or complete memory and CPU quotas. +Execution permissions and resource access are the engine's responsibility. + +## Build and test + +Use the workspace Rust toolchain and a `protoc` compiler with proto3 optional +support. From the repository root: + +```bash +cargo test --locked -p spark-connect-server +cargo clippy --locked -p spark-connect-server --all-targets --no-deps -- -D warnings +``` + +The tests run a real gRPC frontend against deterministic injected engines. They +cover wire framing, schema and IPC handling, backend error propagation, +backpressure, operation ownership, cancellation and session lifetime. Fixture +relations select predefined outputs; they do not implement Spark operators or +establish query semantics or Spark server compatibility. diff --git a/crates/spark-connect-server/build.rs b/crates/spark-connect-server/build.rs new file mode 100644 index 00000000..3ffc7807 --- /dev/null +++ b/crates/spark-connect-server/build.rs @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use tonic_build::manual::{Builder, Method, Service}; + +fn main() { + // The official protocol crate already generates messages and client bindings. + // Generate only the standard service routes, using those exact message types. + // Keep this RPC list and its streaming signatures synchronized with + // spark-connect-proto/proto/spark/connect/base.proto on each protocol bump. + let methods = [ + ( + "execute_plan", + "ExecutePlan", + "ExecutePlanRequest", + "ExecutePlanResponse", + false, + true, + ), + ( + "analyze_plan", + "AnalyzePlan", + "AnalyzePlanRequest", + "AnalyzePlanResponse", + false, + false, + ), + ( + "config", + "Config", + "ConfigRequest", + "ConfigResponse", + false, + false, + ), + ( + "add_artifacts", + "AddArtifacts", + "AddArtifactsRequest", + "AddArtifactsResponse", + true, + false, + ), + ( + "artifact_status", + "ArtifactStatus", + "ArtifactStatusesRequest", + "ArtifactStatusesResponse", + false, + false, + ), + ( + "interrupt", + "Interrupt", + "InterruptRequest", + "InterruptResponse", + false, + false, + ), + ( + "reattach_execute", + "ReattachExecute", + "ReattachExecuteRequest", + "ExecutePlanResponse", + false, + true, + ), + ( + "release_execute", + "ReleaseExecute", + "ReleaseExecuteRequest", + "ReleaseExecuteResponse", + false, + false, + ), + ( + "release_session", + "ReleaseSession", + "ReleaseSessionRequest", + "ReleaseSessionResponse", + false, + false, + ), + ( + "fetch_error_details", + "FetchErrorDetails", + "FetchErrorDetailsRequest", + "FetchErrorDetailsResponse", + false, + false, + ), + ( + "clone_session", + "CloneSession", + "CloneSessionRequest", + "CloneSessionResponse", + false, + false, + ), + ( + "get_status", + "GetStatus", + "GetStatusRequest", + "GetStatusResponse", + false, + false, + ), + ]; + let mut service = Service::builder() + .name("SparkConnectService") + .package("spark.connect"); + for (name, route, input, output, client_stream, server_stream) in methods { + let mut method = Method::builder() + .name(name) + .route_name(route) + .input_type(format!("spark_connect_proto::{input}")) + .output_type(format!("spark_connect_proto::{output}")) + .codec_path("tonic_prost::ProstCodec"); + if client_stream { + method = method.client_streaming(); + } + if server_stream { + method = method.server_streaming(); + } + service = service.method(method.build()); + } + Builder::new() + .build_client(false) + .compile(&[service.build()]); + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/crates/spark-connect-server/src/engine/mod.rs b/crates/spark-connect-server/src/engine/mod.rs new file mode 100644 index 00000000..c4a337da --- /dev/null +++ b/crates/spark-connect-server/src/engine/mod.rs @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Native engine boundary for Spark Connect planning and execution. +//! +//! The frontend owns protocol IDs, operation lifetime, framing and backpressure. +//! Engines own their catalog, configuration and native plans. This Rust API is +//! an initial in-process contract for an injected execution engine. + +use std::pin::Pin; +use std::sync::Arc; + +use arrow::datatypes::SchemaRef; +use arrow::record_batch::RecordBatch; +use futures::Stream; +use spark_connect_proto as proto; +use tonic::Status; + +/// Convert an engine Arrow schema into the Spark Connect schema representation. +pub use crate::schema::spark_schema; + +/// Protocol context for a new backend session. +#[derive(Clone, Debug)] +pub struct SessionContext { + pub session_id: String, + /// Client-supplied metadata, not an authenticated identity. + pub user_context: Option, +} + +/// Factory for sessions bound to one native engine. +pub trait Engine: Send + Sync { + /// Allocate local session state without blocking or performing I/O. + /// Defer asynchronous initialization to the session methods. The frontend + /// serializes creation to ensure one backend session per protocol session. + fn create_session(&self, context: SessionContext) -> Result, Status>; +} + +/// Backend-owned catalog, configuration and planning state. +/// +/// Dropping a method future must cancel its work. Dropping the final session +/// reference releases its resources after in-flight operations have finished. +// async_trait repeats Future's must_use annotation on generated trait methods. +#[allow(clippy::double_must_use)] +#[tonic::async_trait] +pub trait EngineSession: Send + Sync { + /// Prepare a query, or perform an eager protocol command. The frontend + /// reserves the operation ID before calling this method. + /// If cancelled, a command must leave its state change fully committed or + /// rolled back, with no partially applied changes. + async fn prepare(&self, operation: proto::plan::OpType) -> Result; + + async fn analyze( + &self, + input: proto::analyze_plan_request::Analyze, + ) -> Result; + + async fn config( + &self, + input: proto::config_request::operation::OpType, + ) -> Result; +} + +/// Prepared native query or command-specific response payloads. +pub enum PreparedOperation { + Query(Box), + /// The frontend adds envelope IDs and the final ResultComplete message. + Command(Vec), +} + +/// Query execution starts after the frontend validates and sends its schema. +// async_trait repeats Future's must_use annotation on generated trait methods. +#[allow(clippy::double_must_use)] +#[tonic::async_trait] +pub trait PreparedQuery: Send { + /// The declared Spark schema. Batches must have the same field names and + /// Spark-visible types; equivalent Arrow representations are accepted. + /// Batch nullability and metadata may differ and remain in each IPC batch, + /// while the protocol schema header uses this declaration. The declared + /// fields must not contain metadata, which Spark schema conversion rejects. + fn schema(&self) -> SchemaRef; + + /// Dropping this future or its returned stream must cancel backend work; + /// implementations must not leave detached execution tasks running. + async fn execute(self: Box) -> Result; +} + +/// Engine output, optionally already serialized for Spark Connect. +pub enum ResultBatch { + Arrow(RecordBatch), + /// A complete IPC stream containing its schema, exactly one record batch, + /// and EOS. The producer must supply that batch's actual nonnegative row count. + Ipc { + data: Vec, + row_count: i64, + }, +} + +impl From for ResultBatch { + fn from(batch: RecordBatch) -> Self { + Self::Arrow(batch) + } +} + +/// Pull-based results: the frontend polls only as client backpressure allows. +pub type BatchStream = Pin> + Send>>; + +/// Engine configuration output without frontend-owned protocol IDs. +#[derive(Default)] +pub struct ConfigResult { + pub pairs: Vec, + pub warnings: Vec, +} diff --git a/crates/spark-connect-server/src/lib.rs b/crates/spark-connect-server/src/lib.rs new file mode 100644 index 00000000..92f19de6 --- /dev/null +++ b/crates/spark-connect-server/src/lib.rs @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod engine; +mod schema; +mod service; + +// tonic's generated async-trait methods duplicate Future's must_use annotation. +#[allow(clippy::double_must_use)] +pub mod grpc { + include!(concat!( + env!("OUT_DIR"), + "/spark.connect.SparkConnectService.rs" + )); +} + +/// Serve with an injected execution engine and a fresh session registry. +/// +/// The frontend binds each protocol session to the engine session created for it. +/// The caller owns the engine and signals shutdown before releasing its resources. +pub async fn serve_with_engine( + listener: tokio::net::TcpListener, + engine: std::sync::Arc, + shutdown: impl std::future::Future + Send + 'static, +) -> Result<(), tonic::transport::Error> { + let service = std::sync::Arc::new(service::ConnectService::new(engine)); + let router = tonic::transport::Server::builder().add_service( + grpc::spark_connect_service_server::SparkConnectServiceServer::from_arc(service.clone()) + .max_decoding_message_size(64 * 1024 * 1024), + ); + let (draining, drain_started) = tokio::sync::oneshot::channel(); + let server = router.serve_with_incoming_shutdown( + tokio_stream::wrappers::TcpListenerStream::new(listener), + async move { + shutdown.await; + service.cancel_all(); + let _ = draining.send(()); + }, + ); + tokio::pin!(server); + tokio::select! { + result = &mut server => result, + _ = drain_started => { + // A peer that stops reading must not prevent local process shutdown. + tokio::time::timeout(std::time::Duration::from_secs(2), &mut server) + .await.unwrap_or(Ok(())) + } + } +} diff --git a/crates/spark-connect-server/src/schema.rs b/crates/spark-connect-server/src/schema.rs new file mode 100644 index 00000000..a6a37a49 --- /dev/null +++ b/crates/spark-connect-server/src/schema.rs @@ -0,0 +1,224 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! The supported Spark schema boundary. Unsupported Arrow types fail explicitly. + +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, FieldRef, Schema, TimeUnit}; +use prost::Message; +use spark_connect_proto as proto; +use tonic::Status; + +pub(crate) const MAX_RESULT_BYTES: usize = 3 * 1024 * 1024; + +pub(crate) fn result_schema(schema: &Schema) -> Result { + let schema = spark_schema(schema)?; + validate_schema_size(&schema)?; + Ok(schema) +} + +pub(crate) fn validate_schema_size(schema: &proto::DataType) -> Result<(), Status> { + if schema.encoded_len() > MAX_RESULT_BYTES { + return Err(Status::resource_exhausted( + "Spark schema exceeds the 3 MiB result limit", + )); + } + Ok(()) +} + +/// Compare Spark-visible fields and types, allowing equivalent Arrow physical +/// representations and differences in nullability or field/schema metadata. +pub(crate) fn comparable_schema(schema: &Schema) -> Result { + fn field(input: &FieldRef) -> FieldRef { + Arc::new(Field::new(input.name(), data_type(input.data_type()), true)) + } + + fn data_type(input: &DataType) -> DataType { + match input { + DataType::List(child) => DataType::List(field(child)), + DataType::LargeList(child) => DataType::LargeList(field(child)), + DataType::Struct(fields) => DataType::Struct(fields.iter().map(field).collect()), + DataType::Map(entries, sorted) => DataType::Map(field(entries), *sorted), + other => other.clone(), + } + } + + let fields: Vec<_> = schema.fields().iter().map(field).collect(); + spark_schema(&Schema::new(fields)) +} + +pub fn spark_schema(schema: &Schema) -> Result { + spark_type(&DataType::Struct(schema.fields().clone())) +} + +fn validate_field_metadata(field: &Field) -> Result<(), Status> { + if !field.metadata().is_empty() { + return Err(Status::unimplemented( + "Arrow field metadata in Spark schemas", + )); + } + Ok(()) +} + +fn spark_field(field: &Field) -> Result { + validate_field_metadata(field)?; + Ok(proto::data_type::StructField { + name: field.name().clone(), + data_type: Some(spark_type(field.data_type())?), + nullable: field.is_nullable(), + metadata: None, + }) +} + +fn spark_type(data_type: &DataType) -> Result { + use proto::data_type::{self as pb, Kind}; + let kind = match data_type { + DataType::Null => Kind::Null(pb::Null::default()), + DataType::Boolean => Kind::Boolean(pb::Boolean::default()), + DataType::Int8 => Kind::Byte(pb::Byte::default()), + DataType::Int16 => Kind::Short(pb::Short::default()), + DataType::Int32 => Kind::Integer(pb::Integer::default()), + DataType::Int64 => Kind::Long(pb::Long::default()), + DataType::Float32 => Kind::Float(pb::Float::default()), + DataType::Float64 => Kind::Double(pb::Double::default()), + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Kind::String(pb::String { + collation: "UTF8_BINARY".into(), + ..Default::default() + }), + DataType::Binary | DataType::LargeBinary | DataType::BinaryView => { + Kind::Binary(pb::Binary::default()) + } + DataType::Date32 => Kind::Date(pb::Date::default()), + DataType::Timestamp(TimeUnit::Microsecond, None) => { + Kind::TimestampNtz(pb::TimestampNtz::default()) + } + DataType::Timestamp(TimeUnit::Microsecond, Some(_)) => { + Kind::Timestamp(pb::Timestamp::default()) + } + DataType::Decimal128(precision, scale) => Kind::Decimal(pb::Decimal { + precision: Some(i32::from(*precision)), + scale: Some(i32::from(*scale)), + ..Default::default() + }), + DataType::List(field) | DataType::LargeList(field) => { + validate_field_metadata(field)?; + Kind::Array(Box::new(pb::Array { + element_type: Some(Box::new(spark_type(field.data_type())?)), + contains_null: field.is_nullable(), + ..Default::default() + })) + } + DataType::Struct(fields) => Kind::Struct(pb::Struct { + fields: fields + .iter() + .map(|field| spark_field(field)) + .collect::>()?, + ..Default::default() + }), + DataType::Map(entries, _) => { + validate_field_metadata(entries)?; + let DataType::Struct(fields) = entries.data_type() else { + return Err(Status::invalid_argument( + "Arrow map entries must be a struct", + )); + }; + if fields.len() != 2 { + return Err(Status::invalid_argument( + "Arrow map entries need key and value fields", + )); + } + validate_field_metadata(&fields[0])?; + validate_field_metadata(&fields[1])?; + Kind::Map(Box::new(pb::Map { + key_type: Some(Box::new(spark_type(fields[0].data_type())?)), + value_type: Some(Box::new(spark_type(fields[1].data_type())?)), + value_contains_null: fields[1].is_nullable(), + ..Default::default() + })) + } + other => { + return Err(Status::unimplemented(format!( + "Arrow result type {other:?}" + ))) + } + }; + Ok(proto::DataType { kind: Some(kind) }) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + fn annotated(name: &str, data_type: DataType, nullable: bool) -> Field { + Field::new(name, data_type, nullable).with_metadata(HashMap::from([( + "producer.note".into(), + "cannot be represented in Spark schemas".into(), + )])) + } + + #[test] + fn declared_metadata_is_rejected_on_list_and_every_map_child() { + let entries = |key: Field, value: Field| { + Field::new("entries", DataType::Struct(vec![key, value].into()), false) + }; + let types = [ + DataType::List(Arc::new(annotated("item", DataType::Int64, true))), + DataType::LargeList(Arc::new(annotated("item", DataType::Int64, true))), + DataType::Map( + Arc::new(annotated( + "entries", + DataType::Struct( + vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Int64, true), + ] + .into(), + ), + false, + )), + false, + ), + DataType::Map( + Arc::new(entries( + annotated("key", DataType::Utf8, false), + Field::new("value", DataType::Int64, true), + )), + false, + ), + DataType::Map( + Arc::new(entries( + Field::new("key", DataType::Utf8, false), + annotated("value", DataType::Int64, true), + )), + false, + ), + ]; + for data_type in types { + let schema = Schema::new(vec![Field::new("nested", data_type, true)]); + assert_eq!( + result_schema(&schema).unwrap_err().code(), + tonic::Code::Unimplemented + ); + // Batch annotations remain acceptable when comparing the declared + // Spark-visible shape, including all nested child fields. + comparable_schema(&schema).unwrap(); + } + } +} diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs new file mode 100644 index 00000000..b4f951da --- /dev/null +++ b/crates/spark-connect-server/src/service.rs @@ -0,0 +1,1232 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::{HashMap, VecDeque}; +use std::panic::AssertUnwindSafe; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; + +use arrow::ipc::{reader::StreamReader, writer::StreamWriter}; +use arrow::record_batch::RecordBatch; +use futures::{FutureExt, Stream, StreamExt}; +use prost::Message; +use spark_connect_proto as proto; +use tokio::sync::{mpsc, watch}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{Request, Response, Status}; +use uuid::Uuid; + +use crate::engine::{Engine, EngineSession, PreparedOperation, ResultBatch, SessionContext}; +use crate::grpc::spark_connect_service_server::SparkConnectService; +use crate::schema::{self, result_schema}; + +type Output = Pin> + Send>>; +type SessionKey = (String, String); +const MAX_ARROW_IPC_BYTES: usize = crate::schema::MAX_RESULT_BYTES; +const MAX_RELEASED_SESSIONS: usize = 4096; +const RELEASE_WAIT: std::time::Duration = std::time::Duration::from_secs(2); + +struct Session { + id: String, + backend: Arc, + cancelled: watch::Sender, + operations: Mutex>>, +} + +struct Operation { + cancelled: watch::Sender, + done: watch::Sender, +} + +impl Session { + fn new(backend: Arc) -> Self { + Self { + id: Uuid::new_v4().to_string(), + backend, + cancelled: watch::channel(false).0, + operations: Mutex::new(HashMap::new()), + } + } + + async fn while_open( + &self, + action: impl std::future::Future> + Send, + ) -> Result { + let mut cancelled = self.cancelled.subscribe(); + if *cancelled.borrow() { + return Err(Status::cancelled("session released")); + } + tokio::select! { + biased; + _ = cancelled.changed() => Err(Status::cancelled("session released")), + result = action => result, + } + } + + fn operation( + self: &Arc, + id: &str, + ) -> Result<(Arc, watch::Receiver), Status> { + // Subscribe before publishing: ReleaseAll can cancel as soon as the + // operation appears in the registry, before preparation is first polled. + let (cancelled, cancellation) = watch::channel(false); + let running = Arc::new(Operation { + cancelled, + done: watch::channel(false).0, + }); + let mut operations = self + .operations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if operations.contains_key(id) { + return Err(Status::already_exists("operation is already running")); + } + operations.insert(id.to_owned(), running.clone()); + Ok(( + Arc::new(OperationGuard { + session: self.clone(), + id: id.to_owned(), + running, + }), + cancellation, + )) + } +} + +// This guard spans preparation and execution, including eager commands. Drop the +// backend future/stream before the guard so ReleaseAll observes completed teardown. +struct OperationGuard { + session: Arc, + id: String, + running: Arc, +} + +impl Drop for OperationGuard { + fn drop(&mut self) { + self.session + .operations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&self.id); + self.running.done.send_replace(true); + } +} + +#[derive(Default)] +struct SessionRegistry { + // None is a recently released-session tombstone, retained until eviction. + entries: HashMap>>, + // Remember the most recent distinct closed IDs in release order. Repeated + // releases do not refresh retention. After eviction, an old ID may create a + // fresh session; active sessions never count toward this finite budget. + released: VecDeque, +} + +impl SessionRegistry { + fn release(&mut self, key: SessionKey, allow_reconnect: bool) -> Option> { + // Releasing a missing or already closed session is a no-op. In + // particular, a later reconnecting release cannot erase a tombstone. + let session = self.entries.get_mut(&key)?.take()?; + if allow_reconnect { + self.entries.remove(&key); + } else { + self.released.push_back(key); + if self.released.len() > MAX_RELEASED_SESSIONS { + if let Some(oldest) = self.released.pop_front() { + self.entries.remove(&oldest); + } + } + } + Some(session) + } +} + +pub(crate) struct ConnectService { + engine: Arc, + sessions: Mutex, + closing: std::sync::atomic::AtomicBool, +} + +impl ConnectService { + pub(crate) fn new(engine: Arc) -> Self { + Self { + engine, + sessions: Mutex::new(SessionRegistry::default()), + closing: std::sync::atomic::AtomicBool::new(false), + } + } + + fn key(id: &str, user: Option<&proto::UserContext>) -> Result { + Uuid::parse_str(id).map_err(|_| Status::invalid_argument("session_id must be a UUID"))?; + Ok(( + user.map(|u| u.user_id.clone()).unwrap_or_default(), + id.into(), + )) + } + + fn session( + &self, + id: &str, + user: Option<&proto::UserContext>, + observed: Option<&str>, + ) -> Result, Status> { + self.find_session(id, user, observed, true) + } + + fn find_session( + &self, + id: &str, + user: Option<&proto::UserContext>, + observed: Option<&str>, + create: bool, + ) -> Result, Status> { + let key = Self::key(id, user)?; + let mut sessions = self + .sessions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.closing.load(std::sync::atomic::Ordering::SeqCst) { + return Err(Status::unavailable("server is shutting down")); + } + let session = match sessions.entries.get(&key) { + Some(Some(session)) => session.clone(), + Some(None) => return Err(Status::failed_precondition("session was released")), + None if !create => return Err(Status::not_found("session does not exist")), + // An observed ID refers to a previous incarnation; a new session + // cannot satisfy it. Reject before allocating backend resources. + None if observed.is_some() => { + return Err(Status::failed_precondition( + "server-side session ID changed", + )); + } + None => { + let backend = self.engine.create_session(SessionContext { + session_id: id.to_owned(), + user_context: user.cloned(), + })?; + let session = Arc::new(Session::new(backend)); + sessions.entries.insert(key, Some(session.clone())); + session + } + }; + if observed.is_some_and(|id| id != session.id) { + return Err(Status::failed_precondition( + "server-side session ID changed", + )); + } + Ok(session) + } + + pub(crate) fn cancel_all(&self) { + self.closing + .store(true, std::sync::atomic::Ordering::SeqCst); + // Stalled HTTP/2 connections can retain the service after the bounded + // transport drain. Remove its registry ownership now so backend sessions + // can close once their cancelled operations finish. + let sessions = std::mem::take( + &mut *self + .sessions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + ); + for session in sessions.entries.into_values().flatten() { + session.cancelled.send_replace(true); + } + } +} + +fn response( + session_id: &str, + session: &Session, + operation_id: &str, + response_type: Option, +) -> proto::ExecutePlanResponse { + proto::ExecutePlanResponse { + session_id: session_id.into(), + server_side_session_id: session.id.clone(), + operation_id: operation_id.into(), + response_id: Uuid::new_v4().to_string(), + response_type, + ..Default::default() + } +} + +fn checked_response( + message: proto::ExecutePlanResponse, +) -> Result { + if message.encoded_len() > MAX_ARROW_IPC_BYTES { + return Err(Status::resource_exhausted( + "ExecutePlan response exceeds the 3 MiB result limit", + )); + } + Ok(message) +} + +fn complete(id: &str, session: &Session, operation: &str) -> proto::ExecutePlanResponse { + response( + id, + session, + operation, + Some(proto::execute_plan_response::ResponseType::ResultComplete( + proto::execute_plan_response::ResultComplete {}, + )), + ) +} + +fn encode_batch(batch: &RecordBatch) -> Result, Status> { + use arrow::datatypes::{DataType, FieldRef, Schema}; + use arrow::record_batch::RecordBatchOptions; + + fn materialized_field(field: &FieldRef) -> FieldRef { + let data_type = materialized_type(field.data_type()); + if data_type == *field.data_type() { + field.clone() + } else { + Arc::new(field.as_ref().clone().with_data_type(data_type)) + } + } + + fn materialized_type(data_type: &DataType) -> DataType { + match data_type { + DataType::Utf8View => DataType::LargeUtf8, + DataType::BinaryView => DataType::LargeBinary, + DataType::List(field) => DataType::List(materialized_field(field)), + DataType::LargeList(field) => DataType::LargeList(materialized_field(field)), + DataType::Struct(fields) => { + DataType::Struct(fields.iter().map(materialized_field).collect()) + } + DataType::Map(field, sorted) => DataType::Map(materialized_field(field), *sorted), + other => other.clone(), + } + } + + // IPC writes all backing buffers of view arrays even after a row slice. Materialize + // their bytes, including nested children, so splitting measures only retained rows. + // Use 64-bit offsets so materializing a batch cannot overflow 32-bit byte offsets + // before the result-size check has a chance to split it. + let source_schema = batch.schema(); + let fields: Vec<_> = source_schema + .fields() + .iter() + .map(materialized_field) + .collect(); + let changed = fields + .iter() + .zip(source_schema.fields()) + .any(|(target, source)| target.data_type() != source.data_type()); + let materialized = if changed { + let columns = batch + .columns() + .iter() + .zip(&fields) + .map(|(column, field)| { + if column.data_type() == field.data_type() { + Ok(column.clone()) + } else { + arrow::compute::cast(column.as_ref(), field.data_type()) + } + }) + .collect::, _>>() + .map_err(|error| Status::internal(error.to_string()))?; + let target_schema = Arc::new(Schema::new_with_metadata( + fields, + source_schema.metadata().clone(), + )); + Some( + RecordBatch::try_new_with_options( + target_schema, + columns, + &RecordBatchOptions::new().with_row_count(Some(batch.num_rows())), + ) + .map_err(|error| Status::internal(error.to_string()))?, + ) + } else { + None + }; + let batch = materialized.as_ref().unwrap_or(batch); + let mut bytes = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut bytes, batch.schema().as_ref()) + .map_err(|e| Status::internal(e.to_string()))?; + writer + .write(batch) + .map_err(|e| Status::internal(e.to_string()))?; + writer + .finish() + .map_err(|e| Status::internal(e.to_string()))?; + } + Ok(bytes) +} + +fn estimated_rows_per_batch(batch: &RecordBatch) -> usize { + let bytes = batch.get_array_memory_size(); + // Start below the exact IPC limit to allow schema/offset overhead. This is + // only an estimate: skewed rows and annotations are checked after encoding. + // Include retained backing allocations so even sliced/view inputs begin + // conservatively without materializing an entire oversized engine batch. + rows_for_bytes(batch.num_rows(), bytes) +} + +fn rows_for_bytes(rows: usize, bytes: usize) -> usize { + (rows as u128 * (MAX_ARROW_IPC_BYTES / 2) as u128 / bytes.max(1) as u128) + .clamp(1, rows.max(1) as u128) as usize +} + +// Fields drop in declaration order, including when encoding panics or a +// cancelled caller drops a completed JoinHandle without observing its output. +struct EncodingBatch { + batch: RecordBatch, + _guard: Arc, +} + +impl EncodingBatch { + fn encode(self) -> Result<(Self, Vec), Status> { + let data = encode_batch(&self.batch)?; + Ok((self, data)) + } +} + +async fn send_batch( + sender: &mpsc::Sender>, + id: &str, + operation: &str, + batch: ResultBatch, + offset: &mut i64, + guard: &Arc, +) -> Result { + let (batch, rows_per_batch) = match batch { + ResultBatch::Arrow(batch) => { + let rows = estimated_rows_per_batch(&batch); + (batch, rows) + } + ResultBatch::Ipc { data, row_count } => { + if row_count < 0 { + return Err(Status::internal("negative engine IPC row count")); + } + let batch = proto::execute_plan_response::ArrowBatch { + row_count, + data: data.into(), + start_offset: Some(*offset), + ..Default::default() + }; + let result = response( + id, + &guard.session, + operation, + Some(proto::execute_plan_response::ResponseType::ArrowBatch( + batch.clone(), + )), + ); + if result.encoded_len() <= MAX_ARROW_IPC_BYTES { + *offset = offset + .checked_add(row_count) + .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; + return Ok(sender.send(Ok(result)).await.is_ok()); + } + drop(result); + // Ordinary IPC is forwarded unchanged. Only oversized payloads need + // vectors so the existing row splitter can enforce the envelope limit. + let guard = guard.clone(); + tokio::task::spawn_blocking(move || { + let _guard = guard; + let ipc_bytes = batch.data.len(); + let mut reader = StreamReader::try_new(&batch.data[..], None) + .map_err(|e| Status::internal(format!("invalid engine IPC: {e}")))?; + let batch = reader + .next() + .transpose() + .map_err(|e| Status::internal(format!("invalid engine IPC: {e}")))? + .ok_or_else(|| Status::internal("engine IPC contains no record batch"))?; + if i64::try_from(batch.num_rows()).ok() != Some(row_count) + || reader.next().is_some() + { + return Err(Status::internal( + "engine IPC batch count differs from its declaration", + )); + } + // Decoded columns can share the full IPC allocation. Use its + // serialized size instead of counting that allocation per column. + let rows = rows_for_bytes(batch.num_rows(), ipc_bytes); + Ok((batch, rows)) + }) + .await + .map_err(|_| Status::internal("Arrow IPC decoding task failed"))?? + } + }; + send_arrow_batch(sender, id, operation, batch, rows_per_batch, offset, guard).await +} + +async fn send_arrow_batch( + sender: &mpsc::Sender>, + id: &str, + operation: &str, + batch: RecordBatch, + rows_per_batch: usize, + offset: &mut i64, + guard: &Arc, +) -> Result { + let mut pending = vec![(batch, rows_per_batch)]; + while let Some((batch, rows_per_batch)) = pending.pop() { + // Keep the remainder as one slice instead of allocating a pending entry + // per row in a large engine batch. + let batch = if batch.num_rows() > rows_per_batch { + pending.push(( + batch.slice(rows_per_batch, batch.num_rows() - rows_per_batch), + rows_per_batch, + )); + batch.slice(0, rows_per_batch) + } else { + batch + }; + // Arrow materialization/IPC encoding is CPU work. Retain operation + // ownership if cancellation drops the JoinHandle, so ReleaseAll cannot + // claim drain while a result batch is still being encoded. + let encoding = EncodingBatch { + batch, + _guard: guard.clone(), + }; + let (encoded, data) = tokio::task::spawn_blocking(move || encoding.encode()) + .await + .map_err(|_| Status::internal("Arrow result encoding task failed"))??; + let batch = &encoded.batch; + let result = response( + id, + &guard.session, + operation, + Some(proto::execute_plan_response::ResponseType::ArrowBatch( + proto::execute_plan_response::ArrowBatch { + row_count: batch.num_rows() as i64, + data: data.into(), + start_offset: Some(*offset), + ..Default::default() + }, + )), + ); + let result_bytes = result.encoded_len(); + if result_bytes > MAX_ARROW_IPC_BYTES { + if batch.num_rows() <= 1 { + return Err(Status::resource_exhausted( + "one Arrow row/schema exceeds the 3 MiB result limit", + )); + } + // Recheck every slice including the envelope: skew and schema + // overhead can exceed the initial memory estimate. + let rows_per_batch = ((batch.num_rows() as u128 * MAX_ARROW_IPC_BYTES as u128 * 9 + / (result_bytes as u128 * 10)) as usize) + .clamp(1, batch.num_rows() - 1); + drop(result); + pending.push((batch.clone(), rows_per_batch)); + continue; + } + *offset = offset + .checked_add(batch.num_rows() as i64) + .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; + if sender.send(Ok(result)).await.is_err() { + return Ok(false); + } + } + Ok(true) +} + +#[tonic::async_trait] +impl SparkConnectService for ConnectService { + type ExecutePlanStream = Output; + type ReattachExecuteStream = Output; + + async fn execute_plan( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let operation = req + .operation_id + .unwrap_or_else(|| Uuid::new_v4().to_string()); + Uuid::parse_str(&operation) + .map_err(|_| Status::invalid_argument("operation_id must be a UUID"))?; + let op = req + .plan + .and_then(|p| p.op_type) + .ok_or_else(|| Status::invalid_argument("missing plan"))?; + let session = self.session( + &req.session_id, + req.user_context.as_ref(), + req.client_observed_server_side_session_id.as_deref(), + )?; + let mut cancelled = session.cancelled.subscribe(); + if *cancelled.borrow() { + return Err(Status::cancelled("session released")); + } + // Reserve before calling the backend: preparing a command can mutate state. + let (guard, mut operation_cancelled) = session.operation(&operation)?; + let prepared = tokio::select! { + biased; + _ = cancelled.changed() => return Err(Status::cancelled("session released")), + _ = operation_cancelled.changed() => return Err(Status::cancelled("operation released")), + prepared = session.backend.prepare(op) => prepared?, + }; + let query = match prepared { + PreparedOperation::Query(query) => query, + PreparedOperation::Command(payloads) => { + let mut messages = payloads + .into_iter() + .map(|payload| { + let message = + response(&req.session_id, &session, &operation, Some(payload)); + checked_response(message) + }) + .collect::, Status>>()?; + messages.push(checked_response(complete( + &req.session_id, + &session, + &operation, + ))?); + return Ok(Response::new(Box::pin(tokio_stream::iter( + messages.into_iter().map(Ok), + )))); + } + }; + let arrow_schema = query.schema(); + let spark_schema = result_schema(&arrow_schema)?; + let expected_batch_schema = schema::comparable_schema(&arrow_schema)?; + let (sender, receiver) = mpsc::channel(2); + tokio::spawn(async move { + let run = async { + let mut header = response(&req.session_id, &session, &operation, None); + header.schema = Some(spark_schema); + let header = checked_response(header)?; + if sender.send(Ok(header)).await.is_err() { + return Ok(()); + } + let mut batches = query.execute().await?; + let mut offset = 0; + let mut sent = false; + while let Some(batch) = batches.next().await { + let batch = batch?; + let batch_schema = match &batch { + ResultBatch::Arrow(batch) => batch.schema(), + ResultBatch::Ipc { data, .. } => { + StreamReader::try_new(data.as_slice(), None) + .map_err(|e| Status::internal(format!("invalid engine IPC: {e}")))? + .schema() + } + }; + let actual_schema = + schema::comparable_schema(batch_schema.as_ref()).map_err(|error| { + Status::internal(format!( + "invalid engine batch schema: {}", + error.message() + )) + })?; + if actual_schema != expected_batch_schema { + return Err(Status::internal( + "engine batch schema differs from its declared schema", + )); + } + if !send_batch( + &sender, + &req.session_id, + &operation, + batch, + &mut offset, + &guard, + ) + .await? + { + return Ok(()); + } + sent = true; + } + // Teardown can itself panic. Complete only after the backend + // stream has been dropped within the panic boundary. + drop(batches); + if !sent { + let batch = RecordBatch::new_empty(arrow_schema); + if !send_batch( + &sender, + &req.session_id, + &operation, + batch.into(), + &mut offset, + &guard, + ) + .await? + { + return Ok(()); + } + } + let _ = sender + .send(Ok(checked_response(complete( + &req.session_id, + &session, + &operation, + ))?)) + .await; + Ok::<_, Status>(()) + }; + let result = AssertUnwindSafe(async { + tokio::select! { + biased; + _ = cancelled.changed() => Err(Status::cancelled("session released")), + _ = operation_cancelled.changed() => Err(Status::cancelled("operation released")), + _ = sender.closed() => Ok(()), + result = run => result, + } + }) + .catch_unwind() + .await + .unwrap_or_else(|_| Err(Status::internal("engine execution panicked"))); + drop(guard); + // A slow peer can keep this terminal send pending after cancellation. + // Backend lifetime must end before waiting for that response slot. + drop(session); + if let Err(error) = result { + let _ = sender.send(Err(error)).await; + } + }); + Ok(Response::new(Box::pin(ReceiverStream::new(receiver)))) + } + + async fn analyze_plan( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let input = req + .analyze + .ok_or_else(|| Status::invalid_argument("missing analysis"))?; + let session = self.session( + &req.session_id, + req.user_context.as_ref(), + req.client_observed_server_side_session_id.as_deref(), + )?; + let result = session.while_open(session.backend.analyze(input)).await?; + if let proto::analyze_plan_response::Result::Schema(result) = &result { + if let Some(schema) = &result.schema { + schema::validate_schema_size(schema)?; + } + } + Ok(Response::new(proto::AnalyzePlanResponse { + session_id: req.session_id, + server_side_session_id: session.id.clone(), + result: Some(result), + })) + } + + async fn config( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let input = req + .operation + .and_then(|operation| operation.op_type) + .ok_or_else(|| Status::invalid_argument("missing config operation"))?; + let session = self.session( + &req.session_id, + req.user_context.as_ref(), + req.client_observed_server_side_session_id.as_deref(), + )?; + let result = session.while_open(session.backend.config(input)).await?; + Ok(Response::new(proto::ConfigResponse { + session_id: req.session_id, + server_side_session_id: session.id.clone(), + pairs: result.pairs, + warnings: result.warnings, + })) + } + + async fn release_execute( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + Uuid::parse_str(&req.operation_id) + .map_err(|_| Status::invalid_argument("operation_id must be a UUID"))?; + let session = self.find_session( + &req.session_id, + req.user_context.as_ref(), + req.client_observed_server_side_session_id.as_deref(), + false, + )?; + // Completed responses are not retained; release is idempotent for absent operations. + let running = session + .operations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&req.operation_id) + .cloned(); + let operation_id = running.as_ref().map(|_| req.operation_id.clone()); + if matches!( + req.release, + Some(proto::release_execute_request::Release::ReleaseAll(_)) + ) { + if let Some(running) = running { + let mut done = running.done.subscribe(); + running.cancelled.send_replace(true); + tokio::time::timeout(RELEASE_WAIT, async { + while !*done.borrow_and_update() { + if done.changed().await.is_err() { + break; + } + } + }) + .await + .map_err(|_| { + Status::deadline_exceeded("operation cancellation is still draining") + })?; + } + } + Ok(Response::new(proto::ReleaseExecuteResponse { + session_id: req.session_id, + server_side_session_id: session.id.clone(), + operation_id, + })) + } + + async fn release_session( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let key = Self::key(&req.session_id, req.user_context.as_ref())?; + let session = { + let mut sessions = self + .sessions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + sessions.release(key, req.allow_reconnect) + }; + let server_id = session.as_ref().map(|s| s.id.clone()).unwrap_or_default(); + if let Some(session) = session { + session.cancelled.send_replace(true); + } + Ok(Response::new(proto::ReleaseSessionResponse { + session_id: req.session_id, + server_side_session_id: server_id, + })) + } + + async fn reattach_execute( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "execution reattachment is not supported", + )) + } + async fn add_artifacts( + &self, + _: Request>, + ) -> Result, Status> { + Err(Status::unimplemented("artifact uploads are not supported")) + } + async fn artifact_status( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("artifact status is not supported")) + } + async fn interrupt( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "operation interruption is not supported", + )) + } + async fn fetch_error_details( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "enriched error details are not supported", + )) + } + async fn clone_session( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "server-side session cloning is not supported", + )) + } + async fn get_status( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("status queries are not supported")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct RegistryEngine; + struct RegistrySession; + + impl Engine for RegistryEngine { + fn create_session( + &self, + _context: SessionContext, + ) -> Result, Status> { + Ok(Arc::new(RegistrySession)) + } + } + + #[tonic::async_trait] + impl EngineSession for RegistrySession { + async fn prepare( + &self, + _operation: proto::plan::OpType, + ) -> Result { + Err(Status::unimplemented( + "registry fixture does not execute queries", + )) + } + + async fn analyze( + &self, + _input: proto::analyze_plan_request::Analyze, + ) -> Result { + Err(Status::unimplemented( + "registry fixture does not analyze queries", + )) + } + + async fn config( + &self, + _input: proto::config_request::operation::OpType, + ) -> Result { + Err(Status::unimplemented( + "registry fixture has no configuration", + )) + } + } + + async fn release(service: &ConnectService, id: &str, allow_reconnect: bool) { + service + .release_session(Request::new(proto::ReleaseSessionRequest { + session_id: id.into(), + allow_reconnect, + ..Default::default() + })) + .await + .unwrap(); + } + + #[test] + fn schema_response_limit_includes_protocol_envelope() { + let session = Session::new(Arc::new(RegistrySession)); + let schema = proto::DataType { + kind: Some(proto::data_type::Kind::Struct(proto::data_type::Struct { + fields: vec![proto::data_type::StructField { + name: "x".repeat(MAX_ARROW_IPC_BYTES - 64), + ..Default::default() + }], + ..Default::default() + })), + }; + assert!(schema.encoded_len() < MAX_ARROW_IPC_BYTES); + let mut header = response( + &Uuid::new_v4().to_string(), + &session, + &Uuid::new_v4().to_string(), + None, + ); + header.schema = Some(schema); + assert_eq!( + checked_response(header).unwrap_err().code(), + tonic::Code::ResourceExhausted + ); + } + + #[test] + fn initial_row_budget_splits_large_batches_before_encoding() { + use arrow::array::Int64Array; + use arrow::datatypes::{DataType, Field, Schema}; + + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Int64, + false, + )])), + vec![Arc::new(Int64Array::from_iter_values(0..1_000_000))], + ) + .unwrap(); + let rows = estimated_rows_per_batch(&batch); + assert!(rows > 0 && rows < batch.num_rows()); + assert!(rows * std::mem::size_of::() <= MAX_ARROW_IPC_BYTES / 2); + assert_eq!( + estimated_rows_per_batch(&RecordBatch::new_empty(batch.schema())), + 1 + ); + } + + #[tokio::test] + async fn existing_ipc_is_forwarded_unchanged() { + use arrow::array::Int64Array; + use arrow::datatypes::{DataType, Field, Schema}; + use arrow::ipc::writer::IpcWriteOptions; + + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), + vec![Arc::new(Int64Array::from_iter_values(0..10))], + ) + .unwrap(); + // Legacy IPC framing differs from the frontend encoder's default, so + // byte equality also proves that the payload was not re-encoded. + let options = IpcWriteOptions::try_new(8, true, arrow::ipc::MetadataVersion::V4).unwrap(); + let mut data = Vec::new(); + let mut writer = + StreamWriter::try_new_with_options(&mut data, batch.schema().as_ref(), options) + .unwrap(); + writer.write(&batch).unwrap(); + writer.finish().unwrap(); + drop(writer); + assert_ne!(data, encode_batch(&batch).unwrap()); + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("ipc").unwrap(); + let (sender, mut receiver) = mpsc::channel(1); + let mut offset = 7; + assert!(send_batch( + &sender, + "session", + "ipc", + ResultBatch::Ipc { + data: data.clone(), + row_count: 10 + }, + &mut offset, + &guard, + ) + .await + .unwrap()); + let result = receiver.recv().await.unwrap().unwrap(); + let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + result.response_type + else { + panic!("expected Arrow response"); + }; + assert_eq!(&batch.data[..], data.as_slice()); + assert_eq!(batch.row_count, 10); + assert_eq!(batch.start_offset, Some(7)); + assert_eq!(offset, 17); + } + + #[tokio::test] + async fn oversized_ipc_is_split_by_rows_and_checks_its_declared_count() { + use arrow::array::Int64Array; + use arrow::datatypes::{DataType, Field, Schema}; + + let batch = RecordBatch::try_new( + Arc::new(Schema::new( + (0..10) + .map(|column| Field::new(format!("id{column}"), DataType::Int64, false)) + .collect::>(), + )), + (0..10) + .map(|column| { + Arc::new(Int64Array::from_iter_values( + (0..50_000).map(|row| row + column * 50_000), + )) as arrow::array::ArrayRef + }) + .collect(), + ) + .unwrap(); + let data = encode_batch(&batch).unwrap(); + assert!(data.len() > MAX_ARROW_IPC_BYTES); + let expected_batches = data.len().div_ceil(MAX_ARROW_IPC_BYTES / 2); + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("ipc").unwrap(); + let (sender, mut receiver) = mpsc::channel(64); + let mut offset = 0; + assert!(send_batch( + &sender, + "session", + "ipc", + ResultBatch::Ipc { + data: data.clone(), + row_count: 50_000 + }, + &mut offset, + &guard, + ) + .await + .unwrap()); + let (mut rows, mut batches) = (0, 0); + while let Ok(result) = receiver.try_recv() { + let result = result.unwrap(); + assert!(result.encoded_len() <= MAX_ARROW_IPC_BYTES); + let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + result.response_type + else { + panic!("expected Arrow response"); + }; + assert_eq!(batch.start_offset, Some(rows)); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + let decoded = reader.next().unwrap().unwrap(); + assert_eq!(decoded.num_columns(), 10); + assert_eq!(decoded.num_rows() as i64, batch.row_count); + for (column, values) in decoded.columns().iter().enumerate() { + let ids = values.as_any().downcast_ref::().unwrap(); + let start = rows + column as i64 * 50_000; + assert!(ids + .values() + .iter() + .copied() + .eq(start..start + batch.row_count)); + } + rows += batch.row_count; + batches += 1; + assert!(reader.next().is_none()); + } + assert_eq!(rows, 50_000); + assert_eq!(offset, rows); + assert!((expected_batches..=expected_batches + 1).contains(&batches)); + for row_count in [-1, 49_999] { + let error = send_batch( + &sender, + "session", + "ipc", + ResultBatch::Ipc { + data: data.clone(), + row_count, + }, + &mut offset, + &guard, + ) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::Internal); + assert_eq!(offset, rows); + } + assert!(receiver.try_recv().is_err()); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn cancelled_encoding_wait_retains_operation_until_blocking_task_finishes() { + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let operation = Uuid::new_v4().to_string(); + let (guard, _) = session.operation(&operation).unwrap(); + let mut done = guard.running.done.subscribe(); + let encoding = EncodingBatch { + batch: RecordBatch::new_empty(Arc::new(arrow::datatypes::Schema::empty())), + _guard: guard.clone(), + }; + let (started, ready) = tokio::sync::oneshot::channel(); + let (release, resume) = std::sync::mpsc::channel(); + let waiting = tokio::spawn(async move { + tokio::task::spawn_blocking(move || { + let _ = started.send(()); + resume + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + encoding.encode() + }) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(5), ready) + .await + .unwrap() + .unwrap(); + waiting.abort(); + let _ = waiting.await; + drop(guard); + assert!(!*done.borrow()); + assert!(session.operations.lock().unwrap().contains_key(&operation)); + release.send(()).unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while !*done.borrow_and_update() { + done.changed().await.unwrap(); + } + }) + .await + .unwrap(); + assert!(!session.operations.lock().unwrap().contains_key(&operation)); + } + + #[tokio::test] + async fn released_session_retention_is_bounded_without_evicting_active_sessions() { + let service = ConnectService::new(Arc::new(RegistryEngine)); + let active_id = Uuid::new_v4().to_string(); + let active = service.session(&active_id, None, None).unwrap(); + + let oldest = Uuid::new_v4().to_string(); + let previous = service.session(&oldest, None, None).unwrap(); + release(&service, &oldest, false).await; + assert!(*previous.cancelled.borrow()); + release(&service, &oldest, false).await; + release(&service, &oldest, true).await; + assert!(matches!(service.session(&oldest, None, None), + Err(status) if status.code() == tonic::Code::FailedPrecondition)); + { + let registry = service.sessions.lock().unwrap(); + assert_eq!(registry.entries.len(), 2); + assert_eq!(registry.released.len(), 1); + } + let mut newest = oldest.clone(); + for _ in 1..MAX_RELEASED_SESSIONS { + newest = Uuid::new_v4().to_string(); + service.session(&newest, None, None).unwrap(); + release(&service, &newest, false).await; + } + release(&service, &oldest, false).await; + assert!(matches!( + service.session(&oldest, None, None), + Err(status) if status.code() == tonic::Code::FailedPrecondition + )); + { + let registry = service.sessions.lock().unwrap(); + assert_eq!(registry.released.len(), MAX_RELEASED_SESSIONS); + assert_eq!(registry.entries.len(), MAX_RELEASED_SESSIONS + 1); + } + + // Unknown releases neither allocate tombstones nor evict closed IDs. + let unknown = Uuid::new_v4().to_string(); + release(&service, &unknown, false).await; + release(&service, &unknown, true).await; + assert!(matches!(service.session(&oldest, None, None), + Err(status) if status.code() == tonic::Code::FailedPrecondition)); + service.session(&unknown, None, None).unwrap(); + release(&service, &unknown, false).await; + assert!(service.session(&oldest, None, None).is_ok()); + assert!(matches!( + service.session(&newest, None, None), + Err(status) if status.code() == tonic::Code::FailedPrecondition + )); + assert!(Arc::ptr_eq( + &active, + &service.session(&active_id, None, None).unwrap() + )); + let registry = service.sessions.lock().unwrap(); + assert_eq!(registry.released.len(), MAX_RELEASED_SESSIONS); + assert_eq!(registry.entries.len(), MAX_RELEASED_SESSIONS + 2); + } + + #[tokio::test] + async fn reconnecting_release_of_live_session_allows_a_new_backend() { + let service = ConnectService::new(Arc::new(RegistryEngine)); + let id = Uuid::new_v4().to_string(); + let previous = service.session(&id, None, None).unwrap(); + release(&service, &id, true).await; + assert!(*previous.cancelled.borrow()); + let current = service.session(&id, None, None).unwrap(); + assert!(!Arc::ptr_eq(¤t, &previous)); + } +} diff --git a/crates/spark-connect-server/tests/engine_injection.rs b/crates/spark-connect-server/tests/engine_injection.rs new file mode 100644 index 00000000..a3a81bd3 --- /dev/null +++ b/crates/spark-connect-server/tests/engine_injection.rs @@ -0,0 +1,1528 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Real gRPC tests of the injected backend boundary. The fake is independent of Spark execution. + +use std::collections::HashMap; +use std::io::Cursor; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::task::{Context, Poll}; +use std::time::Duration; + +use arrow::array::{Int64Array, StringArray}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::ipc::reader::StreamReader; +use arrow::ipc::writer::StreamWriter; +use arrow::record_batch::RecordBatch; +use futures::{Stream, StreamExt}; +use prost::Message; +use proto::execute_plan_response::ResponseType; +use proto::spark_connect_service_client::SparkConnectServiceClient; +use spark_connect_proto as proto; +use spark_connect_server::engine::{ + BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, + ResultBatch, SessionContext, +}; +use tokio::sync::oneshot; +use tokio::task::JoinHandle; +use tonic::transport::Channel; +use tonic::{Code, Status}; +use uuid::Uuid; + +const LIVE: i64 = 1_000_000; +const BLOCKED_PREPARE: i64 = 2_000_000; +const MISMATCHED_BATCH: i64 = 3_000_000; +const ANNOTATED_BATCH: i64 = 4_000_000; +const FLOODED_RESULT: i64 = 5_000_000; +const PANICKING_EXECUTION: i64 = 6_000_000; +const PANICKING_STREAM: i64 = 7_000_000; +const PANICKING_STREAM_DROP: i64 = 8_000_000; +const MISMATCHED_IPC_BATCH: i64 = 9_000_000; +const INVALID_IPC_BATCH: i64 = 10_000_000; + +#[derive(Default)] +struct Spy { + factory_calls: Mutex>, + fail_next: AtomicBool, + sessions_dropped: AtomicUsize, + prepares: AtomicUsize, + commands: AtomicUsize, + analyses: AtomicUsize, + configs: AtomicUsize, + executions: AtomicUsize, + streams_dropped: AtomicUsize, + batches_polled: AtomicUsize, + blocked_prepares_dropped: AtomicUsize, + blocked_analyses_dropped: AtomicUsize, + blocked_configs_dropped: AtomicUsize, + metadata_drop_observations: Mutex>, + slow_drop: Mutex)>>, + stream_drop_gate: Mutex>>, +} + +#[derive(Default)] +struct DropGate { + released: Mutex, + condition: Condvar, + entered: AtomicUsize, + exited: AtomicBool, + timed_out: AtomicBool, +} + +impl DropGate { + fn wait(&self) { + let released = self.released.lock().unwrap(); + self.entered.fetch_add(1, Ordering::SeqCst); + let (released, timeout) = self + .condition + .wait_timeout_while(released, Duration::from_secs(10), |released| !*released) + .unwrap(); + self.timed_out + .store(timeout.timed_out() && !*released, Ordering::SeqCst); + self.exited.store(true, Ordering::SeqCst); + } + + fn release(&self) { + *self.released.lock().unwrap() = true; + self.condition.notify_all(); + } +} + +// Even a failing assertion must never leave a worker blocked in the fake Drop. +struct UnblockDrop(Arc); + +impl Drop for UnblockDrop { + fn drop(&mut self) { + self.0.release(); + } +} + +struct FakeEngine(Arc); + +impl Engine for FakeEngine { + fn create_session(&self, context: SessionContext) -> Result, Status> { + let drop_gate = self + .0 + .slow_drop + .lock() + .unwrap() + .as_ref() + .filter(|(id, _)| id == &context.session_id) + .map(|(_, gate)| gate.clone()); + let user = context + .user_context + .map(|user| user.user_id) + .unwrap_or_default(); + self.0 + .factory_calls + .lock() + .unwrap() + .push((user, context.session_id)); + if self.0.fail_next.swap(false, Ordering::SeqCst) { + return Err(Status::unavailable( + "fake factory is temporarily unavailable", + )); + } + Ok(Arc::new(FakeSession { + spy: self.0.clone(), + configs: Mutex::new(HashMap::new()), + views: Mutex::new(HashMap::new()), + drop_gate, + })) + } +} + +struct FakeSession { + spy: Arc, + configs: Mutex>, + views: Mutex>, + drop_gate: Option>, +} + +impl Drop for FakeSession { + fn drop(&mut self) { + if let Some(gate) = &self.drop_gate { + // Keep Drop synchronous, while allowing Tokio to replace this worker + // so the test measures registry contention rather than timer starvation. + tokio::task::block_in_place(|| gate.wait()); + } + self.spy.metadata_drop_observations.lock().unwrap().push(( + self.spy.blocked_analyses_dropped.load(Ordering::SeqCst), + self.spy.blocked_configs_dropped.load(Ordering::SeqCst), + )); + self.spy.sessions_dropped.fetch_add(1, Ordering::SeqCst); + } +} + +struct BlockedPrepare(Arc); + +impl Drop for BlockedPrepare { + fn drop(&mut self) { + self.0 + .blocked_prepares_dropped + .fetch_add(1, Ordering::SeqCst); + } +} + +enum MetadataCall { + Analysis, + Config, +} + +struct BlockedMetadata { + spy: Arc, + call: MetadataCall, +} + +impl Drop for BlockedMetadata { + fn drop(&mut self) { + let counter = match self.call { + MetadataCall::Analysis => &self.spy.blocked_analyses_dropped, + MetadataCall::Config => &self.spy.blocked_configs_dropped, + }; + counter.fetch_add(1, Ordering::SeqCst); + } +} + +#[tonic::async_trait] +impl EngineSession for FakeSession { + async fn prepare(&self, op: proto::plan::OpType) -> Result { + self.spy.prepares.fetch_add(1, Ordering::SeqCst); + let (value, live, batch_mode) = match op { + proto::plan::OpType::Root(relation) => match relation.rel_type { + Some(proto::relation::RelType::Range(range)) => { + if range.end == BLOCKED_PREPARE { + let _dropped = BlockedPrepare(self.spy.clone()); + std::future::pending::<()>().await; + } + if range.end == -1 { + return Err(Status::invalid_argument("fake rejected the plan")); + } + // Deliberately unlike range semantics: this proves the injected + // backend, rather than the default planner, executed the request. + let batch_mode = match range.end { + MISMATCHED_BATCH => BatchMode::WrongType, + MISMATCHED_IPC_BATCH => BatchMode::WrongTypeIpc, + INVALID_IPC_BATCH => BatchMode::InvalidIpc, + ANNOTATED_BATCH => BatchMode::DifferentAnnotations, + FLOODED_RESULT => BatchMode::Flooded, + PANICKING_EXECUTION => BatchMode::PanicExecute, + PANICKING_STREAM => BatchMode::PanicStream, + PANICKING_STREAM_DROP => BatchMode::PanicDrop, + _ => BatchMode::Declared, + }; + (999, range.end == LIVE, batch_mode) + } + Some(proto::relation::RelType::Read(read)) => { + let Some(proto::read::ReadType::NamedTable(table)) = read.read_type else { + return Err(Status::unimplemented("fake read")); + }; + let value = self + .views + .lock() + .unwrap() + .get(&table.unparsed_identifier) + .copied() + .ok_or_else(|| Status::not_found("fake view not found"))?; + (value, false, BatchMode::Declared) + } + _ => return Err(Status::unimplemented("fake relation")), + }, + proto::plan::OpType::Command(command) => { + self.spy.commands.fetch_add(1, Ordering::SeqCst); + match command.command_type { + Some(proto::command::CommandType::CreateDataframeView(view)) => { + let Some(proto::relation::RelType::Range(input)) = + view.input.and_then(|input| input.rel_type) + else { + return Err(Status::invalid_argument("fake view requires range input")); + }; + let mut views = self.views.lock().unwrap(); + if !view.replace && views.contains_key(&view.name) { + return Err(Status::already_exists("fake view already exists")); + } + views.insert(view.name, input.end); + return Ok(PreparedOperation::Command(Vec::new())); + } + Some(proto::command::CommandType::SqlCommand(command)) => { + return Ok(PreparedOperation::Command(vec![ + ResponseType::SqlCommandResult( + proto::execute_plan_response::SqlCommandResult { + relation: command.input, + }, + ), + ])); + } + _ => return Err(Status::unimplemented("fake command")), + } + } + _ => return Err(Status::unimplemented("fake operation")), + }; + Ok(PreparedOperation::Query(Box::new(FakeQuery { + spy: self.spy.clone(), + value, + live, + batch_mode, + }))) + } + + async fn analyze( + &self, + input: proto::analyze_plan_request::Analyze, + ) -> Result { + self.spy.analyses.fetch_add(1, Ordering::SeqCst); + match input { + proto::analyze_plan_request::Analyze::Schema(_) => { + Ok(proto::analyze_plan_response::Result::Schema( + proto::analyze_plan_response::Schema { + schema: Some(proto::DataType { + kind: Some(proto::data_type::Kind::Struct(proto::data_type::Struct { + fields: vec![proto::data_type::StructField { + name: "x".repeat(3 * 1024 * 1024 + 1), + data_type: Some(proto::DataType { + kind: Some(proto::data_type::Kind::Long( + proto::data_type::Long::default(), + )), + }), + nullable: false, + metadata: None, + }], + ..Default::default() + })), + }), + }, + )) + } + proto::analyze_plan_request::Analyze::IsStreaming(_) => { + let _dropped = BlockedMetadata { + spy: self.spy.clone(), + call: MetadataCall::Analysis, + }; + std::future::pending().await + } + proto::analyze_plan_request::Analyze::IsLocal(_) => { + Ok(proto::analyze_plan_response::Result::IsLocal( + proto::analyze_plan_response::IsLocal { is_local: true }, + )) + } + _ => Err(Status::unimplemented("fake analysis")), + } + } + + async fn config( + &self, + input: proto::config_request::operation::OpType, + ) -> Result { + use proto::config_request::operation::OpType; + self.spy.configs.fetch_add(1, Ordering::SeqCst); + if matches!(&input, OpType::Get(get) if get.keys.iter().any(|key| key == "fake.block")) { + let _dropped = BlockedMetadata { + spy: self.spy.clone(), + call: MetadataCall::Config, + }; + return std::future::pending().await; + } + let mut configs = self.configs.lock().unwrap(); + match input { + OpType::Set(set) => { + for pair in set.pairs { + configs.insert( + pair.key, + pair.value + .ok_or_else(|| Status::invalid_argument("fake config value"))?, + ); + } + Ok(ConfigResult { + pairs: Vec::new(), + warnings: vec!["handled by fake backend".into()], + }) + } + OpType::GetOption(get) => Ok(ConfigResult { + pairs: get + .keys + .into_iter() + .map(|key| { + let value = configs.get(&key).cloned(); + proto::KeyValue { key, value } + }) + .collect(), + warnings: Vec::new(), + }), + _ => Err(Status::unimplemented("fake config operation")), + } + } +} + +struct FakeQuery { + spy: Arc, + value: i64, + live: bool, + batch_mode: BatchMode, +} + +enum BatchMode { + Declared, + WrongType, + WrongTypeIpc, + InvalidIpc, + DifferentAnnotations, + Flooded, + PanicExecute, + PanicStream, + PanicDrop, +} + +#[tonic::async_trait] +impl PreparedQuery for FakeQuery { + fn schema(&self) -> SchemaRef { + Arc::new(Schema::new(vec![Field::new( + "backend_value", + DataType::Int64, + false, + )])) + } + + async fn execute(self: Box) -> Result { + self.spy.executions.fetch_add(1, Ordering::SeqCst); + if matches!(self.batch_mode, BatchMode::PanicExecute) { + panic!("fake query execution panic"); + } + let batch = match self.batch_mode { + BatchMode::Declared + | BatchMode::InvalidIpc + | BatchMode::PanicExecute + | BatchMode::PanicStream + | BatchMode::PanicDrop => RecordBatch::try_new( + self.schema(), + vec![Arc::new(Int64Array::from(vec![self.value]))], + ), + BatchMode::WrongType | BatchMode::WrongTypeIpc => RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "backend_value", + DataType::Utf8, + false, + )])), + vec![Arc::new(StringArray::from(vec!["wrong type"]))], + ), + BatchMode::DifferentAnnotations => RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "backend_value", + DataType::Int64, + true, + ) + .with_metadata(HashMap::from([( + "backend.note".into(), + "ignored annotation".into(), + )]))])), + vec![Arc::new(Int64Array::from(vec![self.value]))], + ), + BatchMode::Flooded => RecordBatch::try_new( + self.schema(), + vec![Arc::new(Int64Array::from(vec![self.value; 64 * 1024]))], + ), + } + .unwrap(); + if matches!(self.batch_mode, BatchMode::Flooded) { + let spy = self.spy.clone(); + return Ok(Box::pin(SpyStream { + inner: Box::pin(futures::stream::repeat_with(move || { + spy.batches_polled.fetch_add(1, Ordering::SeqCst); + Ok(batch.clone().into()) + })), + spy: self.spy.clone(), + panic_on_drop: false, + })); + } + let result = match self.batch_mode { + BatchMode::WrongTypeIpc => { + let mut data = Vec::new(); + { + let mut writer = + StreamWriter::try_new(&mut data, batch.schema().as_ref()).unwrap(); + writer.write(&batch).unwrap(); + writer.finish().unwrap(); + } + ResultBatch::Ipc { data, row_count: 1 } + } + BatchMode::InvalidIpc => ResultBatch::Ipc { + data: b"invalid IPC".to_vec(), + row_count: 1, + }, + _ => batch.into(), + }; + let first = futures::stream::iter(vec![Ok(result)]); + let inner: BatchStream = if matches!(self.batch_mode, BatchMode::PanicStream) { + Box::pin(first.chain(futures::stream::poll_fn(|_| { + panic!("fake stream panic after a partial result"); + }))) + } else if self.live { + Box::pin(first.chain(futures::stream::pending())) + } else { + Box::pin(first) + }; + Ok(Box::pin(SpyStream { + inner, + spy: self.spy.clone(), + panic_on_drop: matches!(self.batch_mode, BatchMode::PanicDrop), + })) + } +} + +struct SpyStream { + inner: BatchStream, + spy: Arc, + panic_on_drop: bool, +} + +impl Stream for SpyStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + self.get_mut().inner.as_mut().poll_next(context) + } +} + +impl Drop for SpyStream { + fn drop(&mut self) { + let gate = self.spy.stream_drop_gate.lock().unwrap().clone(); + if let Some(gate) = gate { + tokio::task::block_in_place(|| gate.wait()); + } + self.spy.streams_dropped.fetch_add(1, Ordering::SeqCst); + assert!(!self.panic_on_drop, "fake stream teardown panic"); + } +} + +struct Server { + address: std::net::SocketAddr, + client: SparkConnectServiceClient, + shutdown: Option>, + task: Option>, +} + +impl Server { + async fn start(spy: Arc) -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (shutdown, stopped) = oneshot::channel(); + let task = tokio::spawn(async move { + spark_connect_server::serve_with_engine(listener, Arc::new(FakeEngine(spy)), async { + let _ = stopped.await; + }) + .await + .unwrap(); + }); + let client = SparkConnectServiceClient::connect(format!("http://{address}")) + .await + .unwrap(); + Self { + address, + client, + shutdown: Some(shutdown), + task: Some(task), + } + } + + async fn stop(mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + tokio::time::timeout(Duration::from_secs(5), self.task.take().unwrap()) + .await + .unwrap() + .unwrap(); + } +} + +impl Drop for Server { + fn drop(&mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + if let Some(task) = self.task.take() { + task.abort(); + } + } +} + +fn user(id: &str) -> Option { + Some(proto::UserContext { + user_id: id.into(), + ..Default::default() + }) +} + +fn relation(end: i64) -> proto::Relation { + proto::Relation { + rel_type: Some(proto::relation::RelType::Range(proto::Range { + end, + step: 1, + ..Default::default() + })), + ..Default::default() + } +} + +fn request(session: &str, end: i64) -> proto::ExecutePlanRequest { + proto::ExecutePlanRequest { + session_id: session.into(), + user_context: user("alice"), + operation_id: Some(Uuid::new_v4().to_string()), + plan: Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(relation(end))), + }), + ..Default::default() + } +} + +fn command_request( + session: &str, + command_type: proto::command::CommandType, +) -> proto::ExecutePlanRequest { + let mut request = request(session, 0); + request.plan = Some(proto::Plan { + op_type: Some(proto::plan::OpType::Command(proto::Command { + command_type: Some(command_type), + })), + }); + request +} + +fn config_request( + session: &str, + input: proto::config_request::operation::OpType, +) -> proto::ConfigRequest { + proto::ConfigRequest { + session_id: session.into(), + user_context: user("alice"), + operation: Some(proto::config_request::Operation { + op_type: Some(input), + }), + ..Default::default() + } +} + +fn get_config(session: &str) -> proto::ConfigRequest { + config_request( + session, + proto::config_request::operation::OpType::GetOption(proto::config_request::GetOption { + keys: vec!["spark.fake.setting".into()], + }), + ) +} + +fn release(session: &str, operation: &str) -> proto::ReleaseExecuteRequest { + proto::ReleaseExecuteRequest { + session_id: session.into(), + user_context: user("alice"), + operation_id: operation.into(), + release: Some(proto::release_execute_request::Release::ReleaseAll( + proto::release_execute_request::ReleaseAll {}, + )), + ..Default::default() + } +} + +fn release_session(session: &str) -> proto::ReleaseSessionRequest { + proto::ReleaseSessionRequest { + session_id: session.into(), + user_context: user("alice"), + ..Default::default() + } +} + +async fn wait_for_count(counter: &AtomicUsize, expected: usize) { + tokio::time::timeout(Duration::from_secs(5), async { + while counter.load(Ordering::SeqCst) < expected { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("backend lifecycle event must arrive"); + assert_eq!(counter.load(Ordering::SeqCst), expected); +} + +async fn collect( + stream: &mut tonic::Streaming, +) -> Vec { + let mut responses = Vec::new(); + while let Some(response) = stream.message().await.unwrap() { + responses.push(response); + } + responses +} + +fn values(responses: &[proto::ExecutePlanResponse]) -> Vec { + let mut values = Vec::new(); + for response in responses { + if let Some(ResponseType::ArrowBatch(batch)) = &response.response_type { + for batch in StreamReader::try_new(Cursor::new(batch.data.clone()), None).unwrap() { + let batch = batch.unwrap(); + let array = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + values.extend((0..array.len()).map(|index| array.value(index))); + } + } + } + values +} + +async fn assert_cancelled(stream: &mut tonic::Streaming) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + match stream.message().await { + Err(status) => { + assert_eq!(status.code(), Code::Cancelled); + break; + } + Ok(Some(response)) => assert!(!matches!( + response.response_type, + Some(ResponseType::ResultComplete(_)) + )), + Ok(None) => panic!("cancelled fake execution must report cancellation"), + } + } + }) + .await + .expect("cancelled fake execution must terminate"); +} + +async fn start_live( + server: &mut Server, + request: proto::ExecutePlanRequest, +) -> tonic::Streaming { + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + assert!(matches!( + stream.message().await.unwrap().unwrap().response_type, + Some(ResponseType::ArrowBatch(_)) + )); + stream +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn injected_backend_owns_query_analysis_config_and_catalog_state() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let mut first = server.client.clone(); + let mut second = server.client.clone(); + let (left, right) = tokio::join!( + first.config(get_config(&session)), + second.config(get_config(&session)) + ); + left.unwrap(); + right.unwrap(); + assert_eq!( + spy.factory_calls.lock().unwrap().len(), + 1, + "concurrent requests must share one backend session" + ); + + let set = config_request( + &session, + proto::config_request::operation::OpType::Set(proto::config_request::Set { + pairs: vec![proto::KeyValue { + key: "spark.fake.setting".into(), + value: Some("backend-owned".into()), + }], + silent: Some(false), + }), + ); + let configured = server.client.config(set).await.unwrap().into_inner(); + assert_eq!(configured.warnings, vec!["handled by fake backend"]); + assert_eq!( + server + .client + .config(get_config(&session)) + .await + .unwrap() + .into_inner() + .pairs[0] + .value + .as_deref(), + Some("backend-owned") + ); + let analysis = server + .client + .analyze_plan(proto::AnalyzePlanRequest { + session_id: session.clone(), + user_context: user("alice"), + analyze: Some(proto::analyze_plan_request::Analyze::IsLocal( + proto::analyze_plan_request::IsLocal { + plan: request(&session, 99).plan, + }, + )), + ..Default::default() + }) + .await + .unwrap() + .into_inner(); + assert!(matches!( + analysis.result, + Some(proto::analyze_plan_response::Result::IsLocal( + proto::analyze_plan_response::IsLocal { is_local: true } + )) + )); + + let mut stream = server + .client + .execute_plan(request(&session, 99)) + .await + .unwrap() + .into_inner(); + let responses = collect(&mut stream).await; + assert_eq!(values(&responses), vec![999]); + assert!(matches!( + responses.last().unwrap().response_type, + Some(ResponseType::ResultComplete(_)) + )); + let create_view = command_request( + &session, + proto::command::CommandType::CreateDataframeView(proto::CreateDataFrameViewCommand { + input: Some(relation(77)), + name: "fake_view".into(), + is_global: false, + replace: false, + }), + ); + let mut created = server + .client + .execute_plan(create_view) + .await + .unwrap() + .into_inner(); + collect(&mut created).await; + let mut read_view = request(&session, 0); + read_view.plan = Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(proto::Relation { + rel_type: Some(proto::relation::RelType::Read(proto::Read { + read_type: Some(proto::read::ReadType::NamedTable(proto::read::NamedTable { + unparsed_identifier: "fake_view".into(), + ..Default::default() + })), + ..Default::default() + })), + ..Default::default() + })), + }); + let mut stream = server + .client + .execute_plan(read_view.clone()) + .await + .unwrap() + .into_inner(); + assert_eq!(values(&collect(&mut stream).await), vec![77]); + let sql = command_request( + &session, + proto::command::CommandType::SqlCommand(proto::SqlCommand { + input: Some(relation(13)), + ..Default::default() + }), + ); + let mut stream = server.client.execute_plan(sql).await.unwrap().into_inner(); + let responses = collect(&mut stream).await; + assert!(matches!( + responses[0].response_type, + Some(ResponseType::SqlCommandResult(_)) + )); + + let mut other_user = get_config(&session); + other_user.user_context = user("bob"); + assert_eq!( + server + .client + .config(other_user) + .await + .unwrap() + .into_inner() + .pairs[0] + .value, + None + ); + read_view.user_context = user("bob"); + assert_eq!( + server + .client + .execute_plan(read_view) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + let other_session = Uuid::new_v4().to_string(); + assert_eq!( + server + .client + .config(get_config(&other_session)) + .await + .unwrap() + .into_inner() + .pairs[0] + .value, + None + ); + assert_eq!( + *spy.factory_calls.lock().unwrap(), + vec![ + ("alice".into(), session.clone()), + ("bob".into(), session), + ("alice".into(), other_session), + ] + ); + assert_eq!(spy.analyses.load(Ordering::SeqCst), 1); + assert_eq!(spy.commands.load(Ordering::SeqCst), 2); + assert_eq!(spy.executions.load(Ordering::SeqCst), 2); + server.stop().await; + wait_for_count(&spy.sessions_dropped, 3).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn invalid_requests_do_not_open_backends_and_factory_failures_can_retry() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + assert_eq!( + server + .client + .execute_plan(request("invalid", 99)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + let mut invalid = request(&session, 99); + invalid.operation_id = Some("invalid".into()); + assert_eq!( + server + .client + .execute_plan(invalid) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_execute(release(&session, &Uuid::new_v4().to_string())) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + assert!(spy.factory_calls.lock().unwrap().is_empty()); + spy.fail_next.store(true, Ordering::SeqCst); + assert_eq!( + server + .client + .config(get_config(&session)) + .await + .unwrap_err() + .code(), + Code::Unavailable + ); + server.client.config(get_config(&session)).await.unwrap(); + assert_eq!(spy.factory_calls.lock().unwrap().len(), 2); + server + .client + .release_session(release_session(&session)) + .await + .unwrap(); + wait_for_count(&spy.sessions_dropped, 1).await; + server + .client + .release_session(release_session(&session)) + .await + .unwrap(); + assert_eq!(spy.factory_calls.lock().unwrap().len(), 2); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn native_streams_drop_on_release_disconnect_session_release_and_shutdown() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let live = request(&session, LIVE); + let operation = live.operation_id.clone().unwrap(); + let mut stream = start_live(&mut server, live.clone()).await; + let mut duplicate = command_request( + &session, + proto::command::CommandType::SqlCommand(proto::SqlCommand { + input: Some(relation(1)), + ..Default::default() + }), + ); + duplicate.operation_id = Some(operation.clone()); + assert_eq!( + server + .client + .execute_plan(duplicate) + .await + .unwrap_err() + .code(), + Code::AlreadyExists + ); + assert_eq!(spy.commands.load(Ordering::SeqCst), 0); + assert_eq!(spy.prepares.load(Ordering::SeqCst), 1); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_eq!( + spy.streams_dropped.load(Ordering::SeqCst), + 1, + "release must wait for backend stream teardown" + ); + assert_cancelled(&mut stream).await; + + let abandoned = start_live(&mut server, live.clone()).await; + drop(abandoned); + wait_for_count(&spy.streams_dropped, 2).await; + let mut restarted = tokio::time::timeout(Duration::from_secs(5), async { + loop { + match server.client.execute_plan(live.clone()).await { + Ok(response) => break response.into_inner(), + Err(status) if status.code() == Code::AlreadyExists => { + tokio::time::sleep(Duration::from_millis(10)).await + } + Err(status) => panic!("unexpected restart error: {status}"), + } + } + }) + .await + .expect("disconnected operation must be removed"); + assert!(restarted.message().await.unwrap().unwrap().schema.is_some()); + assert!(matches!( + restarted.message().await.unwrap().unwrap().response_type, + Some(ResponseType::ArrowBatch(_)) + )); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_cancelled(&mut restarted).await; + wait_for_count(&spy.streams_dropped, 3).await; + + let mut live = start_live(&mut server, request(&session, LIVE)).await; + server + .client + .release_session(release_session(&session)) + .await + .unwrap(); + assert_cancelled(&mut live).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert_eq!( + server + .client + .execute_plan(request(&session, 99)) + .await + .unwrap_err() + .code(), + Code::FailedPrecondition + ); + let other = Uuid::new_v4().to_string(); + let mut live = start_live(&mut server, request(&other, LIVE)).await; + server.shutdown.take().unwrap().send(()).unwrap(); + assert_cancelled(&mut live).await; + server.stop().await; + wait_for_count(&spy.streams_dropped, 5).await; + wait_for_count(&spy.sessions_dropped, 2).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shutdown_releases_backend_session_while_a_result_remains_unread() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy.clone()).await; + let (mut client, connection) = h2::client::Builder::new() + .initial_window_size(1024) + .initial_connection_window_size(1024) + .handshake::<_, tonic::codegen::Bytes>( + tokio::net::TcpStream::connect(server.address) + .await + .unwrap(), + ) + .await + .unwrap(); + let connection = tokio::spawn(async move { + let _ = connection.await; + }); + let request = request(&Uuid::new_v4().to_string(), FLOODED_RESULT).encode_to_vec(); + let mut payload = vec![0]; + payload.extend_from_slice(&(request.len() as u32).to_be_bytes()); + payload.extend_from_slice(&request); + let request = tonic::codegen::http::Request::builder() + .method("POST") + .uri(format!( + "http://{}/spark.connect.SparkConnectService/ExecutePlan", + server.address + )) + .header("content-type", "application/grpc") + .body(()) + .unwrap(); + let (response, mut upload) = client.send_request(request, false).unwrap(); + upload.send_data(payload.into(), true).unwrap(); + let response = tokio::time::timeout(Duration::from_secs(5), response) + .await + .expect("response headers must arrive") + .unwrap(); + // Keep the body and HTTP/2 connection alive without reading or releasing + // capacity. Each 512 KiB batch exceeds the stream's 1 KiB receive window. + tokio::time::timeout(Duration::from_secs(5), async { + while spy.batches_polled.load(Ordering::SeqCst) < 4 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the frontend result queue must fill behind the unread peer"); + assert_eq!(spy.sessions_dropped.load(Ordering::SeqCst), 0); + server.shutdown.take().unwrap().send(()).unwrap(); + // Shutdown must release the session registry and the producer's session + // reference before waiting to send a terminal error through the full queue. + wait_for_count(&spy.streams_dropped, 1).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert!(!response.body().is_end_stream()); + drop(response); + drop(upload); + drop(client); + connection.abort(); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_reservation_precedes_prepare_and_cleans_up_after_cancellation_and_errors() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let blocked = request(&session, BLOCKED_PREPARE); + let operation = blocked.operation_id.clone().unwrap(); + let mut pending_client = server.client.clone(); + let pending = tokio::spawn(async move { pending_client.execute_plan(blocked).await }); + wait_for_count(&spy.prepares, 1).await; + let mut duplicate = command_request( + &session, + proto::command::CommandType::SqlCommand(proto::SqlCommand { + input: Some(relation(1)), + ..Default::default() + }), + ); + duplicate.operation_id = Some(operation.clone()); + assert_eq!( + server + .client + .execute_plan(duplicate) + .await + .unwrap_err() + .code(), + Code::AlreadyExists + ); + assert_eq!(spy.commands.load(Ordering::SeqCst), 0); + assert_eq!(spy.prepares.load(Ordering::SeqCst), 1); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_eq!(spy.blocked_prepares_dropped.load(Ordering::SeqCst), 1); + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), pending) + .await + .unwrap() + .unwrap() + .unwrap_err() + .code(), + Code::Cancelled + ); + + let mut retry = request(&session, 99); + retry.operation_id = Some(operation); + let mut stream = server + .client + .execute_plan(retry) + .await + .unwrap() + .into_inner(); + assert_eq!(values(&collect(&mut stream).await), vec![999]); + let rejected = request(&session, -1); + let mut retry = request(&session, 99); + retry.operation_id.clone_from(&rejected.operation_id); + assert_eq!( + server + .client + .execute_plan(rejected) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + let mut stream = server + .client + .execute_plan(retry) + .await + .unwrap() + .into_inner(); + assert_eq!(values(&collect(&mut stream).await), vec![999]); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn release_and_shutdown_cancel_backend_metadata_futures_before_dropping_sessions() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let mut analysis_client = server.client.clone(); + let analysis_session = session.clone(); + let analysis = tokio::spawn(async move { + analysis_client + .analyze_plan(proto::AnalyzePlanRequest { + session_id: analysis_session.clone(), + user_context: user("alice"), + analyze: Some(proto::analyze_plan_request::Analyze::IsStreaming( + proto::analyze_plan_request::IsStreaming { + plan: request(&analysis_session, 99).plan, + }, + )), + ..Default::default() + }) + .await + }); + wait_for_count(&spy.analyses, 1).await; + server + .client + .release_session(release_session(&session)) + .await + .unwrap(); + let status = tokio::time::timeout(Duration::from_secs(5), analysis) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert_eq!(status.code(), Code::Cancelled); + wait_for_count(&spy.blocked_analyses_dropped, 1).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert_eq!( + *spy.metadata_drop_observations.lock().unwrap(), + vec![(1, 0)] + ); + + let session = Uuid::new_v4().to_string(); + let mut config_client = server.client.clone(); + let config = tokio::spawn(async move { + config_client + .config(config_request( + &session, + proto::config_request::operation::OpType::Get(proto::config_request::Get { + keys: vec!["fake.block".into()], + }), + )) + .await + }); + wait_for_count(&spy.configs, 1).await; + server.shutdown.take().unwrap().send(()).unwrap(); + let status = tokio::time::timeout(Duration::from_secs(5), config) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert_eq!(status.code(), Code::Cancelled); + wait_for_count(&spy.blocked_configs_dropped, 1).await; + server.stop().await; + wait_for_count(&spy.sessions_dropped, 2).await; + assert_eq!( + *spy.metadata_drop_observations.lock().unwrap(), + vec![(1, 0), (1, 1)] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn slow_backend_drop_does_not_hold_the_global_session_registry_lock() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy.clone()).await; + // Channel clones share an HTTP/2 connection. Establish a new TCP connection + // before blocking Drop so the probe isolates the global registry lock. + let mut independent = SparkConnectServiceClient::connect(format!("http://{}", server.address)) + .await + .unwrap(); + let slow_session = Uuid::new_v4().to_string(); + let gate = Arc::new(DropGate::default()); + let unblock = UnblockDrop(gate.clone()); + *spy.slow_drop.lock().unwrap() = Some((slow_session.clone(), gate.clone())); + server + .client + .config(get_config(&slow_session)) + .await + .unwrap(); + let mut release_client = server.client.clone(); + let released = tokio::spawn(async move { + release_client + .release_session(release_session(&slow_session)) + .await + }); + // Store outcomes without asserting while Drop is blocked. Its own ten-second + // deadline and the RAII unblock guard also cover failures in this test. + let entered = tokio::time::timeout(Duration::from_secs(2), async { + while gate.entered.load(Ordering::SeqCst) == 0 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await; + let other = tokio::time::timeout( + Duration::from_secs(1), + independent.config(get_config(&Uuid::new_v4().to_string())), + ) + .await; + let still_blocked = !gate.exited.load(Ordering::SeqCst); + drop(unblock); + let released = tokio::time::timeout(Duration::from_secs(5), released).await; + server.stop().await; + + assert!(entered.is_ok(), "slow Drop must start"); + other + .expect("another session must remain responsive while Drop is blocked") + .unwrap(); + assert!( + still_blocked, + "the independent request must complete before Drop exits" + ); + assert!( + !gate.timed_out.load(Ordering::SeqCst), + "Drop must be explicitly unblocked" + ); + released.unwrap().unwrap().unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn frontend_rejects_oversized_analysis_and_batch_type_mismatches() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let status = server + .client + .analyze_plan(proto::AnalyzePlanRequest { + session_id: session.clone(), + user_context: user("alice"), + analyze: Some(proto::analyze_plan_request::Analyze::Schema( + proto::analyze_plan_request::Schema { + plan: request(&session, 99).plan, + }, + )), + ..Default::default() + }) + .await + .unwrap_err(); + assert_eq!(status.code(), Code::ResourceExhausted); + + let mut mismatch = server + .client + .execute_plan(request(&session, MISMATCHED_BATCH)) + .await + .unwrap() + .into_inner(); + let header = mismatch.message().await.unwrap().unwrap(); + assert!(header.schema.is_some()); + assert!(header.response_type.is_none()); + // The very next event must be the error: no inconsistent Arrow payload or + // successful completion may escape after the declared Int64 schema. + assert_eq!(mismatch.message().await.unwrap_err().code(), Code::Internal); + + let mut compatible = server + .client + .execute_plan(request(&session, ANNOTATED_BATCH)) + .await + .unwrap() + .into_inner(); + let responses = collect(&mut compatible).await; + assert_eq!(values(&responses), vec![999]); + assert!(matches!( + responses.last().unwrap().response_type, + Some(ResponseType::ResultComplete(_)) + )); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn frontend_rejects_mismatched_and_invalid_ipc_before_forwarding() { + let mut server = Server::start(Arc::new(Spy::default())).await; + let session = Uuid::new_v4().to_string(); + for (selector, message) in [ + ( + MISMATCHED_IPC_BATCH, + "engine batch schema differs from its declared schema", + ), + (INVALID_IPC_BATCH, "invalid engine IPC:"), + ] { + let mut stream = server + .client + .execute_plan(request(&session, selector)) + .await + .unwrap() + .into_inner(); + let header = stream.message().await.unwrap().unwrap(); + assert!(header.schema.is_some()); + assert!(header.response_type.is_none()); + // The next event is the schema-validation error, before any IPC payload + // or successful completion can escape the declared schema header. + let status = stream.message().await.unwrap_err(); + assert_eq!(status.code(), Code::Internal); + assert!(status.message().starts_with(message), "{status}"); + } + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn panicking_execution_reports_internal_after_header_or_partial_result() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + for selector in [PANICKING_EXECUTION, PANICKING_STREAM, PANICKING_STREAM_DROP] { + let mut stream = server + .client + .execute_plan(request(&session, selector)) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + if selector != PANICKING_EXECUTION { + assert!(matches!( + stream.message().await.unwrap().unwrap().response_type, + Some(ResponseType::ArrowBatch(_)) + )); + } + let error = tokio::time::timeout(Duration::from_secs(5), stream.message()) + .await + .unwrap() + .unwrap_err(); + assert_eq!(error.code(), Code::Internal); + assert!(error.message().contains("panicked")); + } + let mut recovered = server + .client + .execute_plan(request(&session, 1)) + .await + .unwrap() + .into_inner(); + assert_eq!(values(&collect(&mut recovered).await), vec![999]); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn oversized_command_payload_fails_before_sending_any_success_message() { + let spy = Arc::new(Spy::default()); + let mut server = Server::start(spy).await; + let session = Uuid::new_v4().to_string(); + let command = command_request( + &session, + proto::command::CommandType::SqlCommand(proto::SqlCommand { + input: Some(proto::Relation { + rel_type: Some(proto::relation::RelType::Read(proto::Read { + read_type: Some(proto::read::ReadType::NamedTable(proto::read::NamedTable { + unparsed_identifier: "x".repeat(3 * 1024 * 1024 + 1), + ..Default::default() + })), + ..Default::default() + })), + ..Default::default() + }), + ..Default::default() + }), + ); + assert_eq!( + server + .client + .execute_plan(command) + .await + .unwrap_err() + .code(), + Code::ResourceExhausted + ); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn release_all_timeout_keeps_cancellation_and_operation_ownership_until_drain() { + let spy = Arc::new(Spy::default()); + let gate = Arc::new(DropGate::default()); + *spy.stream_drop_gate.lock().unwrap() = Some(gate.clone()); + let unblock = UnblockDrop(gate.clone()); + let mut server = Server::start(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let original = request(&session, LIVE); + let operation = original.operation_id.clone().unwrap(); + let mut stream = server + .client + .execute_plan(original.clone()) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + assert!(matches!( + stream.message().await.unwrap().unwrap().response_type, + Some(ResponseType::ArrowBatch(_)) + )); + let error = server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap_err(); + assert_eq!(error.code(), Code::DeadlineExceeded); + assert!(!gate.exited.load(Ordering::SeqCst)); + assert_eq!( + server + .client + .execute_plan(original.clone()) + .await + .unwrap_err() + .code(), + Code::AlreadyExists + ); + drop(unblock); + assert_cancelled(&mut stream).await; + assert!(!gate.timed_out.load(Ordering::SeqCst)); + *spy.stream_drop_gate.lock().unwrap() = None; + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + let mut recovered = server + .client + .execute_plan(original) + .await + .unwrap() + .into_inner(); + assert!(recovered.message().await.unwrap().unwrap().schema.is_some()); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_cancelled(&mut recovered).await; + server.stop().await; +} diff --git a/crates/spark-connect-server/tests/protocol.rs b/crates/spark-connect-server/tests/protocol.rs new file mode 100644 index 00000000..6a4ef03c --- /dev/null +++ b/crates/spark-connect-server/tests/protocol.rs @@ -0,0 +1,903 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Verify observable Spark Connect wire behavior independently of DataFrame conveniences. + +use std::collections::HashSet; +use std::io::Cursor; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::Int64Array; +use arrow::ipc::reader::StreamReader; +use prost::Message; +use proto::execute_plan_response::ResponseType; +use proto::spark_connect_service_client::SparkConnectServiceClient; +use spark_connect_proto as proto; +use spark_connect_server::engine::{Engine, EngineSession, SessionContext}; + +mod support; +use support::{FixtureEngine, LIVE_FIXTURE}; +use tokio::sync::oneshot; +use tokio::task::JoinHandle; +use tonic::transport::Channel; +use tonic::Code; +use uuid::Uuid; + +struct Server { + client: SparkConnectServiceClient, + shutdown: Option>, + task: Option>, +} + +impl Server { + async fn start() -> Self { + Self::start_with_engine(Arc::new(FixtureEngine::default())).await + } + + async fn start_with_engine(engine: Arc) -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (shutdown, stopped) = oneshot::channel(); + let task = tokio::spawn(async move { + spark_connect_server::serve_with_engine(listener, engine, async { + let _ = stopped.await; + }) + .await + .unwrap(); + }); + let client = SparkConnectServiceClient::connect(format!("http://{address}")) + .await + .unwrap(); + Self { + client, + shutdown: Some(shutdown), + task: Some(task), + } + } + + async fn stop(mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + tokio::time::timeout(Duration::from_secs(5), self.task.take().unwrap()) + .await + .unwrap() + .unwrap(); + } +} + +#[derive(Default)] +struct CountingEngine { + sessions_created: AtomicUsize, +} + +impl Engine for CountingEngine { + fn create_session( + &self, + context: SessionContext, + ) -> Result, tonic::Status> { + self.sessions_created.fetch_add(1, Ordering::SeqCst); + FixtureEngine::default().create_session(context) + } +} + +impl Drop for Server { + fn drop(&mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + if let Some(task) = self.task.take() { + task.abort(); + } + } +} + +fn range(session: &str, end: i64) -> proto::ExecutePlanRequest { + proto::ExecutePlanRequest { + session_id: session.into(), + user_context: Some(proto::UserContext::default()), + operation_id: Some(Uuid::new_v4().to_string()), + plan: Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(proto::Relation { + rel_type: Some(proto::relation::RelType::Range(proto::Range { + start: Some(0), + end, + step: 1, + num_partitions: None, + })), + ..Default::default() + })), + }), + ..Default::default() + } +} + +fn release(session: &str, operation: &str) -> proto::ReleaseExecuteRequest { + proto::ReleaseExecuteRequest { + session_id: session.into(), + user_context: Some(proto::UserContext::default()), + operation_id: operation.into(), + release: Some(proto::release_execute_request::Release::ReleaseAll( + proto::release_execute_request::ReleaseAll {}, + )), + ..Default::default() + } +} + +fn config_request( + session: &str, + operation: proto::config_request::operation::OpType, +) -> proto::ConfigRequest { + proto::ConfigRequest { + session_id: session.into(), + user_context: Some(proto::UserContext::default()), + operation: Some(proto::config_request::Operation { + op_type: Some(operation), + }), + ..Default::default() + } +} + +async fn assert_cancelled(stream: &mut tonic::Streaming) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + match stream.message().await { + Ok(Some(response)) => assert!(!matches!( + response.response_type, + Some(ResponseType::ResultComplete(_)) + )), + Ok(None) => panic!("cancelled execution must report cancellation"), + Err(status) => { + assert_eq!(status.code(), Code::Cancelled); + break; + } + } + } + }) + .await + .expect("cancelled stream must terminate"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn arrow_stream_preserves_schema_ids_offsets_and_empty_results() { + let mut server = Server::start().await; + let session = Uuid::new_v4().to_string(); + let mut server_session = None; + for end in [0, 25_000] { + let request = range(&session, end); + let operation = request.operation_id.clone().unwrap(); + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + let mut response_ids = HashSet::new(); + let (mut rows, mut batches, mut schemas, mut complete) = (0, 0, 0, false); + while let Some(response) = stream.message().await.unwrap() { + assert!(!complete, "ResultComplete must be the last response"); + assert_eq!(response.session_id, session); + assert_eq!(response.operation_id, operation); + Uuid::parse_str(&response.response_id).unwrap(); + assert!(response_ids.insert(response.response_id)); + Uuid::parse_str(&response.server_side_session_id).unwrap(); + let stable = server_session.get_or_insert(response.server_side_session_id.clone()); + assert_eq!(*stable, response.server_side_session_id); + if let Some(schema) = response.schema { + let Some(proto::data_type::Kind::Struct(schema)) = schema.kind else { + panic!("struct schema"); + }; + assert_eq!(schema.fields.len(), 1); + assert_eq!(schema.fields[0].name, "id"); + assert!(!schema.fields[0].nullable); + assert!(matches!( + schema.fields[0].data_type.as_ref().unwrap().kind, + Some(proto::data_type::Kind::Long(_)) + )); + schemas += 1; + } + match response.response_type { + Some(ResponseType::ArrowBatch(batch)) => { + assert_eq!(batch.start_offset, Some(rows)); + let reader = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); + let mut batch_rows = 0; + for record in reader { + let record = record.unwrap(); + let values = record + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for index in 0..values.len() { + assert_eq!(values.value(index), rows + batch_rows + index as i64); + } + batch_rows += record.num_rows() as i64; + } + assert_eq!(batch_rows, batch.row_count); + rows += batch_rows; + batches += 1; + } + Some(ResponseType::ResultComplete(_)) => complete = true, + None => {} + other => panic!("unexpected result: {other:?}"), + } + } + assert_eq!(rows, end); + assert_eq!(schemas, 1); + assert!(complete); + assert!(batches >= 1, "empty execution must still return Arrow data"); + if end > 0 { + // The fixture supplies multiple physical batches; all must reach the wire. + assert!(batches > 1, "large result should stream multiple batches"); + } + } + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn observed_session_mismatch_and_released_sessions_are_rejected() { + let mut server = Server::start().await; + let session = Uuid::new_v4().to_string(); + let mut first = server + .client + .execute_plan(range(&session, 0)) + .await + .unwrap() + .into_inner(); + let original = first + .message() + .await + .unwrap() + .unwrap() + .server_side_session_id; + while first.message().await.unwrap().is_some() {} + let mut mismatch = range(&session, 1); + mismatch.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); + let error = server.client.execute_plan(mismatch).await.unwrap_err(); + assert_eq!(error.code(), Code::FailedPrecondition); + let release = proto::ReleaseSessionRequest { + session_id: session.clone(), + user_context: Some(proto::UserContext::default()), + ..Default::default() + }; + let released = server + .client + .release_session(release.clone()) + .await + .unwrap() + .into_inner(); + assert_eq!(released.server_side_session_id, original); + server.client.release_session(release).await.unwrap(); + let error = server + .client + .execute_plan(range(&session, 1)) + .await + .unwrap_err(); + assert_eq!(error.code(), Code::FailedPrecondition); + // A separate session on the same transport remains usable. + let mut other = server + .client + .execute_plan(range(&Uuid::new_v4().to_string(), 0)) + .await + .unwrap() + .into_inner(); + while other.message().await.unwrap().is_some() {} + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn release_all_cancels_a_pending_backend_stream_and_waits_for_teardown() { + let engine = Arc::new(FixtureEngine::default()); + let mut server = Server::start_with_engine(engine.clone()).await; + let session = Uuid::new_v4().to_string(); + let request = range(&session, LIVE_FIXTURE); + let operation = request.operation_id.clone().unwrap(); + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + let header = stream.message().await.unwrap().unwrap(); + assert!(header.schema.is_some()); + let batch = stream.message().await.unwrap().unwrap(); + assert!(matches!( + batch.response_type, + Some(ResponseType::ArrowBatch(_)) + )); + let release = proto::ReleaseExecuteRequest { + session_id: session.clone(), + user_context: Some(proto::UserContext::default()), + operation_id: operation.clone(), + client_observed_server_side_session_id: Some(header.server_side_session_id), + release: Some(proto::release_execute_request::Release::ReleaseAll( + proto::release_execute_request::ReleaseAll {}, + )), + ..Default::default() + }; + let released = tokio::time::timeout( + Duration::from_secs(3), + server.client.release_execute(release.clone()), + ) + .await + .expect("ReleaseAll must finish without consuming the result") + .unwrap() + .into_inner(); + assert_eq!(released.operation_id.as_deref(), Some(operation.as_str())); + assert_eq!( + engine.streams_dropped.load(Ordering::SeqCst), + 1, + "ReleaseAll must wait until the backend stream is dropped" + ); + let absent = server + .client + .release_execute(release) + .await + .unwrap() + .into_inner(); + assert_eq!( + absent.operation_id, None, + "completed teardown must remove the operation" + ); + assert_cancelled(&mut stream).await; + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn arrow_payloads_are_byte_limited_and_oversized_rows_fail() { + use arrow::array::{LargeStringArray, StringArray, StringViewArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use arrow::record_batch::RecordBatch; + + let session = Uuid::new_v4().to_string(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("s", DataType::Utf8, false), + ])); + // Skewed string sizes make initial byte-ratio slices too large, forcing + // refinement while preserving every value and row offset. + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from_iter_values(0..8192)), + Arc::new(StringArray::from_iter_values( + (0..8192).map(|id| "x".repeat(if id % 64 == 0 { 200000 } else { 1000 })), + )), + ], + ) + .unwrap(); + let oversized_schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)])); + let oversized = RecordBatch::try_new( + oversized_schema.clone(), + vec![Arc::new(StringArray::from(vec!["x".repeat(4000000)]))], + ) + .unwrap(); + let engine = FixtureEngine::default() + .with_batches(2, schema, vec![batch]) + .with_batches(3, oversized_schema, vec![oversized]); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let mut stream = server + .client + .execute_plan(range(&session, 2)) + .await + .unwrap() + .into_inner(); + let (mut rows, mut batches, mut complete) = (0, 0, false); + while let Some(response) = stream.message().await.unwrap() { + assert!(response.encoded_len() <= 3 * 1024 * 1024); + match response.response_type { + Some(ResponseType::ArrowBatch(batch)) => { + assert_eq!(batch.start_offset, Some(rows)); + let records = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); + let mut decoded = 0; + for record in records { + let record = record.unwrap(); + let ids = record + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let strings = record.column(1); + for index in 0..record.num_rows() { + let id = rows + decoded + index as i64; + assert_eq!(ids.value(index), id); + let value = match strings.data_type() { + arrow::datatypes::DataType::Utf8 => strings + .as_any() + .downcast_ref::() + .unwrap() + .value(index), + arrow::datatypes::DataType::LargeUtf8 => strings + .as_any() + .downcast_ref::() + .unwrap() + .value(index), + arrow::datatypes::DataType::Utf8View => strings + .as_any() + .downcast_ref::() + .unwrap() + .value(index), + other => panic!("unexpected string type: {other:?}"), + }; + assert_eq!(value.len(), if id % 64 == 0 { 200000 } else { 1000 }); + assert!(value.bytes().all(|byte| byte == b'x')); + } + decoded += record.num_rows() as i64; + } + assert_eq!(decoded, batch.row_count); + rows += decoded; + batches += 1; + } + Some(ResponseType::ResultComplete(_)) => complete = true, + None => {} + other => panic!("unexpected result: {other:?}"), + } + } + assert_eq!(rows, 8192); + assert!(batches > 1); + assert!(complete); + + let mut oversized = server + .client + .execute_plan(range(&session, 3)) + .await + .unwrap() + .into_inner(); + assert!(oversized.message().await.unwrap().unwrap().schema.is_some()); + let status = oversized.message().await.unwrap_err(); + assert_eq!(status.code(), Code::ResourceExhausted); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn release_session_cancels_live_stream_and_live_operation_ids_are_unique() { + let mut server = Server::start().await; + let session = Uuid::new_v4().to_string(); + let request = range(&session, LIVE_FIXTURE); + let mut stream = server + .client + .execute_plan(request.clone()) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + let duplicate = server.client.execute_plan(request).await.unwrap_err(); + assert_eq!(duplicate.code(), Code::AlreadyExists); + server + .client + .release_session(proto::ReleaseSessionRequest { + session_id: session.clone(), + user_context: Some(proto::UserContext::default()), + ..Default::default() + }) + .await + .unwrap(); + assert_cancelled(&mut stream).await; + assert_eq!( + server + .client + .execute_plan(range(&session, 0)) + .await + .unwrap_err() + .code(), + Code::FailedPrecondition + ); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn invalid_requests_do_not_allocate_sessions_and_unknown_release_is_rejected() { + let engine = Arc::new(CountingEngine::default()); + let mut server = Server::start_with_engine(engine.clone()).await; + let valid = Uuid::new_v4().to_string(); + assert_eq!( + server + .client + .execute_plan(range("not-a-uuid", 0)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + let mut invalid_operation = range(&valid, 0); + invalid_operation.operation_id = Some("not-a-uuid".into()); + assert_eq!( + server + .client + .execute_plan(invalid_operation) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + for plan in [None, Some(proto::Plan { op_type: None })] { + let mut request = range(&valid, 0); + request.plan = plan; + assert_eq!( + server + .client + .execute_plan(request) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + } + assert_eq!( + server + .client + .analyze_plan(proto::AnalyzePlanRequest { + session_id: valid.clone(), + ..Default::default() + }) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + for operation in [ + None, + Some(proto::config_request::Operation { op_type: None }), + ] { + assert_eq!( + server + .client + .config(proto::ConfigRequest { + session_id: valid.clone(), + operation, + ..Default::default() + }) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + } + // A stale observed ID cannot match a session that does not exist. Repeated + // requests must not consume backend session capacity while being rejected. + for _ in 0..3 { + let mut stale = range(&valid, 0); + stale.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); + assert_eq!( + server.client.execute_plan(stale).await.unwrap_err().code(), + Code::FailedPrecondition + ); + } + let mut stale = config_request( + &valid, + proto::config_request::operation::OpType::GetOption(proto::config_request::GetOption { + keys: vec!["missing".into()], + }), + ); + stale.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); + assert_eq!( + server.client.config(stale).await.unwrap_err().code(), + Code::FailedPrecondition + ); + // Identifier and envelope validation must precede backend session allocation. + assert_eq!(engine.sessions_created.load(Ordering::SeqCst), 0); + assert_eq!( + server + .client + .release_execute(release(&valid, &Uuid::new_v4().to_string())) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + assert_eq!( + server + .client + .release_execute(release("not-a-uuid", &valid)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_execute(release(&valid, "not-a-uuid")) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + let unknown = Uuid::new_v4().to_string(); + assert_eq!( + server + .client + .release_execute(release(&unknown, &valid)) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + assert_eq!(engine.sessions_created.load(Ordering::SeqCst), 0); + server + .client + .config(config_request( + &valid, + proto::config_request::operation::OpType::GetOption(proto::config_request::GetOption { + keys: vec!["spark.sql.session.timeZone".into()], + }), + )) + .await + .unwrap(); + assert_eq!(engine.sessions_created.load(Ordering::SeqCst), 1); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn disconnect_removes_operation_and_shutdown_cancels_live_streams() { + let mut server = Server::start().await; + let session = Uuid::new_v4().to_string(); + let request = range(&session, LIVE_FIXTURE); + let operation = request.operation_id.clone().unwrap(); + let mut abandoned = server + .client + .execute_plan(request.clone()) + .await + .unwrap() + .into_inner(); + assert!(abandoned.message().await.unwrap().unwrap().schema.is_some()); + drop(abandoned); + let mut resumed = tokio::time::timeout(Duration::from_secs(5), async { + loop { + match server.client.execute_plan(request.clone()).await { + Ok(response) => break response.into_inner(), + Err(status) if status.code() == Code::AlreadyExists => { + tokio::time::sleep(Duration::from_millis(20)).await; + } + Err(status) => panic!("unexpected retry failure: {status}"), + } + } + }) + .await + .expect("disconnect must remove the operation registry entry"); + assert!(resumed.message().await.unwrap().unwrap().schema.is_some()); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_cancelled(&mut resumed).await; + + let mut live = server + .client + .execute_plan(range(&session, LIVE_FIXTURE)) + .await + .unwrap() + .into_inner(); + assert!(live.message().await.unwrap().unwrap().schema.is_some()); + server.shutdown.take().unwrap().send(()).unwrap(); + assert_cancelled(&mut live).await; + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn backend_status_codes_survive_prepare_and_stream_failures() { + let engine = FixtureEngine::default() + .reject(10, Code::Unimplemented) + .reject(11, Code::InvalidArgument) + .stream_error(12, Code::ResourceExhausted); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let session = Uuid::new_v4().to_string(); + for (selector, expected) in [(10, Code::Unimplemented), (11, Code::InvalidArgument)] { + let status = server + .client + .execute_plan(range(&session, selector)) + .await + .unwrap_err(); + assert_eq!(status.code(), expected); + } + let mut stream = server + .client + .execute_plan(range(&session, 12)) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + assert_eq!( + stream.message().await.unwrap_err().code(), + Code::ResourceExhausted + ); + // Backend failures must not poison the protocol session or its transport. + let mut recovered = server + .client + .execute_plan(range(&session, 1)) + .await + .unwrap() + .into_inner(); + let (mut rows, mut complete) = (0, false); + while let Some(response) = recovered.message().await.unwrap() { + match response.response_type { + Some(ResponseType::ArrowBatch(batch)) => rows += batch.row_count, + Some(ResponseType::ResultComplete(_)) => complete = true, + None => {} + other => panic!("unexpected result: {other:?}"), + } + } + assert_eq!(rows, 1); + assert!(complete); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn deeply_nested_requests_hit_the_standard_protobuf_decoder_limit() { + let mut server = Server::start().await; + let session = Uuid::new_v4().to_string(); + let mut request = range(&session, 1); + let Some(proto::plan::OpType::Root(mut relation)) = request.plan.take().unwrap().op_type else { + unreachable!() + }; + for _ in 0..60 { + relation = proto::Relation { + rel_type: Some(proto::relation::RelType::Filter(Box::new(proto::Filter { + input: Some(Box::new(relation)), + condition: Some(proto::Expression { + expr_type: Some(proto::expression::ExprType::Literal( + proto::expression::Literal { + literal_type: Some(proto::expression::literal::LiteralType::Boolean( + true, + )), + ..Default::default() + }, + )), + ..Default::default() + }), + }))), + ..Default::default() + }; + } + request.plan = Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(relation)), + }); + let status = server.client.execute_plan(request).await.unwrap_err(); + // Each filter introduces nested protobuf messages. The normal decoder rejects + // this request before the backend can inspect it. + assert_eq!(status.code(), Code::Internal); + assert!(status.message().contains("recursion"), "{status}"); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_backing_buffers_are_compacted_before_splitting_including_struct_children() { + use std::sync::Arc; + + use arrow::array::{BinaryArray, BinaryViewArray, StringArray, StringViewArray, StructArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use arrow::record_batch::RecordBatch; + + let session = Uuid::new_v4().to_string(); + let text = |id: i64| format!("{id:04}{}", "x".repeat(1020)); + let nested_text = |id: i64| format!("n{id:04}{}", "y".repeat(1019)); + let binary = |id: i64| { + let mut bytes = id.to_le_bytes().to_vec(); + bytes.extend(std::iter::repeat_n((id % 251) as u8, 1016)); + bytes + }; + let strings = StringViewArray::from_iter_values((0..4096).map(text)); + let binaries = BinaryViewArray::from_iter_values((0..4096).map(binary)); + let nested_strings = StringViewArray::from_iter_values((0..4096).map(nested_text)); + // A one-row slice of these view arrays still references backing buffers + // larger than the response limit, although each individual value is 1 KiB. + for size in [ + strings + .data_buffers() + .iter() + .map(|buffer| buffer.len()) + .sum::(), + binaries + .data_buffers() + .iter() + .map(|buffer| buffer.len()) + .sum::(), + nested_strings + .data_buffers() + .iter() + .map(|buffer| buffer.len()) + .sum::(), + ] { + assert!(size > 3 * 1024 * 1024); + } + let nested_fields: arrow::datatypes::Fields = + vec![Field::new("text", DataType::Utf8View, false)].into(); + let nested = StructArray::new(nested_fields.clone(), vec![Arc::new(nested_strings)], None); + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8View, false), + Field::new("binary", DataType::BinaryView, false), + Field::new("nested", DataType::Struct(nested_fields), false), + ])), + vec![ + Arc::new(Int64Array::from_iter_values(0..4096)), + Arc::new(strings), + Arc::new(binaries), + Arc::new(nested), + ], + ) + .unwrap(); + let engine = FixtureEngine::default().with_batches(42, batch.schema(), vec![batch]); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let request = range(&session, 42); + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + let (mut rows, mut batches, mut complete) = (0, 0, false); + while let Some(response) = stream.message().await.unwrap() { + assert!(response.encoded_len() <= 3 * 1024 * 1024); + match response.response_type { + Some(ResponseType::ArrowBatch(batch)) => { + assert_eq!(batch.start_offset, Some(rows)); + let records = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); + let mut decoded = 0; + for record in records { + let record = record.unwrap(); + let ids = record + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(record.column(1).data_type(), &DataType::LargeUtf8); + assert_eq!(record.column(2).data_type(), &DataType::LargeBinary); + let strings = arrow::compute::cast(record.column(1), &DataType::Utf8).unwrap(); + let strings = strings.as_any().downcast_ref::().unwrap(); + let binaries = + arrow::compute::cast(record.column(2), &DataType::Binary).unwrap(); + let binaries = binaries.as_any().downcast_ref::().unwrap(); + let nested = record + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(nested.column(0).data_type(), &DataType::LargeUtf8); + let nested_strings = + arrow::compute::cast(nested.column(0), &DataType::Utf8).unwrap(); + let nested_strings = nested_strings + .as_any() + .downcast_ref::() + .unwrap(); + for index in 0..record.num_rows() { + let id = rows + decoded + index as i64; + assert_eq!(ids.value(index), id); + assert_eq!(strings.value(index), text(id)); + assert_eq!(binaries.value(index), binary(id)); + assert_eq!(nested_strings.value(index), nested_text(id)); + } + decoded += record.num_rows() as i64; + } + assert_eq!(decoded, batch.row_count); + rows += decoded; + batches += 1; + } + Some(ResponseType::ResultComplete(_)) => complete = true, + None => {} + other => panic!("unexpected result: {other:?}"), + } + } + assert_eq!(rows, 4096); + assert!(batches > 1); + assert!(complete); + server.stop().await; +} diff --git a/crates/spark-connect-server/tests/support/mod.rs b/crates/spark-connect-server/tests/support/mod.rs new file mode 100644 index 00000000..0df3a439 --- /dev/null +++ b/crates/spark-connect-server/tests/support/mod.rs @@ -0,0 +1,265 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Explicit backend outputs for frontend protocol tests. +//! +//! Range.end is only a fixture lookup key. This backend neither interprets SQL +//! nor implements Spark operators; each case supplies Arrow batches, a pending +//! stream, or a chosen failure. These tests do not verify query execution semantics. + +use std::collections::HashMap; +use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use arrow::array::Int64Array; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::record_batch::RecordBatch; +use futures::{Stream, StreamExt}; +use spark_connect_proto as proto; +use spark_connect_server::engine::{ + BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, + ResultBatch, SessionContext, +}; +use tonic::{Code, Status}; + +pub const LIVE_FIXTURE: i64 = 1_000_000_000_000; + +#[derive(Clone)] +enum Ending { + Complete, + Pending, + Error(Code), +} + +#[derive(Clone)] +enum Output { + Query { + schema: SchemaRef, + batches: Vec, + ending: Ending, + }, + Rejected(Code), +} + +pub struct FixtureEngine { + outputs: HashMap, + pub streams_dropped: Arc, +} + +impl Default for FixtureEngine { + fn default() -> Self { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let batch = |start, end| { + RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int64Array::from_iter_values(start..end))], + ) + .unwrap() + }; + let mut engine = Self { + outputs: HashMap::new(), + streams_dropped: Arc::new(AtomicUsize::new(0)), + }; + engine.outputs.insert( + 0, + Output::Query { + schema: schema.clone(), + batches: Vec::new(), + ending: Ending::Complete, + }, + ); + engine.outputs.insert( + 1, + Output::Query { + schema: schema.clone(), + batches: vec![batch(0, 1)], + ending: Ending::Complete, + }, + ); + engine.outputs.insert( + 25_000, + Output::Query { + schema: schema.clone(), + batches: vec![batch(0, 8192), batch(8192, 16_384), batch(16_384, 25_000)], + ending: Ending::Complete, + }, + ); + engine.outputs.insert( + LIVE_FIXTURE, + Output::Query { + schema: schema.clone(), + batches: vec![batch(0, 8192)], + ending: Ending::Pending, + }, + ); + engine + } +} + +impl FixtureEngine { + pub fn with_batches( + mut self, + selector: i64, + schema: SchemaRef, + batches: Vec, + ) -> Self { + self.outputs.insert( + selector, + Output::Query { + schema, + batches, + ending: Ending::Complete, + }, + ); + self + } + + pub fn reject(mut self, selector: i64, code: Code) -> Self { + self.outputs.insert(selector, Output::Rejected(code)); + self + } + + pub fn stream_error(mut self, selector: i64, code: Code) -> Self { + self.outputs.insert( + selector, + Output::Query { + schema: Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), + batches: Vec::new(), + ending: Ending::Error(code), + }, + ); + self + } +} + +impl Engine for FixtureEngine { + fn create_session(&self, _context: SessionContext) -> Result, Status> { + Ok(Arc::new(FixtureSession { + outputs: self.outputs.clone(), + streams_dropped: self.streams_dropped.clone(), + })) + } +} + +struct FixtureSession { + outputs: HashMap, + streams_dropped: Arc, +} + +#[tonic::async_trait] +impl EngineSession for FixtureSession { + async fn prepare(&self, operation: proto::plan::OpType) -> Result { + let proto::plan::OpType::Root(relation) = operation else { + return Err(Status::unimplemented("fixture accepts root selectors only")); + }; + let Some(proto::relation::RelType::Range(selector)) = relation.rel_type else { + return Err(Status::unimplemented("fixture requires a range selector")); + }; + match self.outputs.get(&selector.end) { + Some(Output::Query { + schema, + batches, + ending, + }) => Ok(PreparedOperation::Query(Box::new(FixtureQuery { + schema: schema.clone(), + batches: batches.clone(), + ending: ending.clone(), + streams_dropped: self.streams_dropped.clone(), + }))), + Some(Output::Rejected(code)) => Err(Status::new(*code, "prepared fixture error")), + None => Err(Status::unimplemented("unknown fixture selector")), + } + } + + async fn analyze( + &self, + _input: proto::analyze_plan_request::Analyze, + ) -> Result { + Err(Status::unimplemented( + "fixture does not analyze query semantics", + )) + } + + async fn config( + &self, + input: proto::config_request::operation::OpType, + ) -> Result { + let proto::config_request::operation::OpType::GetOption(input) = input else { + return Err(Status::unimplemented( + "fixture supports optional config lookup only", + )); + }; + Ok(ConfigResult { + pairs: input + .keys + .into_iter() + .map(|key| proto::KeyValue { key, value: None }) + .collect(), + warnings: Vec::new(), + }) + } +} + +struct FixtureQuery { + schema: SchemaRef, + batches: Vec, + ending: Ending, + streams_dropped: Arc, +} + +#[tonic::async_trait] +impl PreparedQuery for FixtureQuery { + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + async fn execute(self: Box) -> Result { + let first = futures::stream::iter(self.batches.into_iter().map(|batch| Ok(batch.into()))); + let inner: BatchStream = match self.ending { + Ending::Complete => Box::pin(first), + Ending::Pending => Box::pin(first.chain(futures::stream::pending())), + Ending::Error(code) => Box::pin(first.chain(futures::stream::once(async move { + Err(Status::new(code, "streamed fixture error")) + }))), + }; + Ok(Box::pin(TrackedStream { + inner, + streams_dropped: self.streams_dropped, + })) + } +} + +struct TrackedStream { + inner: BatchStream, + streams_dropped: Arc, +} + +impl Stream for TrackedStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + self.inner.as_mut().poll_next(context) + } +} + +impl Drop for TrackedStream { + fn drop(&mut self) { + self.streams_dropped.fetch_add(1, Ordering::SeqCst); + } +} diff --git a/dev/design/ci.md b/dev/design/ci.md index 47ecc8a3..bd789c10 100644 --- a/dev/design/ci.md +++ b/dev/design/ci.md @@ -16,13 +16,15 @@ The CI consists of these main workflows: and macOS universal2 wheels using the release recipe 4. **Coverage** (`.github/workflows/coverage.yml`) - Rust and Python coverage, with regression floors +5. **Experimental server frontend** (`.github/workflows/server-frontend.yml`) - Test and lint + the optional Rust protocol frontend with stable Rust and MSRV 1.88 The other workflows are `lint.yml` (rustfmt, clippy, rustdoc, ruff), `generated-drift.yml`, `vendor-drift.yml`, `docs.yml`, `pandas-parity.yml`, `pipelines-parity.yml` and `release.yml`. Most workflows run on pushes and pull requests targeting `master` and `branch-*`. Several also -accept manual dispatch. `docs.yml` and `wheel-build-checks.yml` run only when files matching their -path filters change. The release wheel build checks cover changes to packaged Python sources, +accept manual dispatch. `docs.yml`, `wheel-build-checks.yml` and `server-frontend.yml` filter push +and pull-request runs to matching paths. The release wheel build checks cover packaged Python sources, protocol definitions, dependencies and build configuration. `release.yml` runs on a schedule and on manual dispatch. @@ -69,6 +71,15 @@ The same workflow has four more jobs: This job should always pass; failures indicate incomplete implementation or test regressions. +## Experimental server frontend (`server-frontend.yml`) + +This focused workflow runs on frontend, protocol/client dependency, workspace manifest, +lockfile, Cargo configuration, toolchain and workflow changes. Each stable Rust and Rust +1.88.0 job has a 45-minute limit and runs `cargo test --locked -p spark-connect-server` +and strict Clippy for all frontend targets. Ubuntu 24.04 supplies a protobuf compiler with +proto3 optional support. Tests use injected engine fixtures and require no execution backend; +they cover frontend protocol behavior rather than Spark execution compatibility. + ## Python Connect Parity (`build_python_connect.yml`) **Status**: Gating. The parity and drop-in jobs pass only when every test that runs passes, except From abe667a62b26cb2027435cc3d79d7adee3cc319a Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 14:05:12 -0800 Subject: [PATCH 02/18] Keep frontend checks independent of client feature selection --- .github/workflows/server-frontend.yml | 2 -- Cargo.lock | 1 - crates/spark-connect-server/Cargo.toml | 1 - crates/spark-connect-server/tests/protocol.rs | 21 ++++++++++++++----- dev/design/ci.md | 2 +- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/server-frontend.yml b/.github/workflows/server-frontend.yml index 7540c9ee..a043bb4d 100644 --- a/.github/workflows/server-frontend.yml +++ b/.github/workflows/server-frontend.yml @@ -30,8 +30,6 @@ on: - "rust-toolchain*" - "crates/spark-connect-server/**" - "crates/spark-connect-proto/**" - - "crates/spark-connect-core/**" - - "crates/spark-connect/**" push: branches: [master, "branch-*"] paths: *frontend-paths diff --git a/Cargo.lock b/Cargo.lock index edfae584..61cee6fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4052,7 +4052,6 @@ dependencies = [ name = "spark-connect-server" version = "0.1.0" dependencies = [ - "apache-spark-connect", "apache-spark-connect-proto", "arrow", "futures", diff --git a/crates/spark-connect-server/Cargo.toml b/crates/spark-connect-server/Cargo.toml index 68def7fa..a27d0413 100644 --- a/crates/spark-connect-server/Cargo.toml +++ b/crates/spark-connect-server/Cargo.toml @@ -43,4 +43,3 @@ tonic-build = "0.14" [dev-dependencies] h2 = "0.4" -spark-connect = { package = "apache-spark-connect", path = "../spark-connect" } diff --git a/crates/spark-connect-server/tests/protocol.rs b/crates/spark-connect-server/tests/protocol.rs index 6a4ef03c..f863426c 100644 --- a/crates/spark-connect-server/tests/protocol.rs +++ b/crates/spark-connect-server/tests/protocol.rs @@ -740,7 +740,7 @@ async fn backend_status_codes_survive_prepare_and_stream_failures() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn deeply_nested_requests_hit_the_standard_protobuf_decoder_limit() { +async fn deeply_nested_requests_follow_the_configured_protobuf_decoder_limit() { let mut server = Server::start().await; let session = Uuid::new_v4().to_string(); let mut request = range(&session, 1); @@ -769,11 +769,22 @@ async fn deeply_nested_requests_hit_the_standard_protobuf_decoder_limit() { request.plan = Some(proto::Plan { op_type: Some(proto::plan::OpType::Root(relation)), }); + let decoded = proto::ExecutePlanRequest::decode(request.encode_to_vec().as_slice()); let status = server.client.execute_plan(request).await.unwrap_err(); - // Each filter introduces nested protobuf messages. The normal decoder rejects - // this request before the backend can inspect it. - assert_eq!(status.code(), Code::Internal); - assert!(status.message().contains("recursion"), "{status}"); + match decoded { + Err(error) => { + // The default decoder rejects this before the backend can inspect it. + assert!(error.to_string().contains("recursion"), "{error}"); + assert_eq!(status.code(), Code::Internal); + assert!(status.message().contains("recursion"), "{status}"); + } + Ok(_) => { + // Other workspace crates enable prost/no-recursion-limit. In that + // configuration the unsupported Filter reaches the fixture backend. + assert_eq!(status.code(), Code::Unimplemented); + assert_eq!(status.message(), "fixture requires a range selector"); + } + } server.stop().await; } diff --git a/dev/design/ci.md b/dev/design/ci.md index bd789c10..e2f68161 100644 --- a/dev/design/ci.md +++ b/dev/design/ci.md @@ -73,7 +73,7 @@ This job should always pass; failures indicate incomplete implementation or test ## Experimental server frontend (`server-frontend.yml`) -This focused workflow runs on frontend, protocol/client dependency, workspace manifest, +This focused workflow runs on frontend, protocol, workspace manifest, lockfile, Cargo configuration, toolchain and workflow changes. Each stable Rust and Rust 1.88.0 job has a 45-minute limit and runs `cargo test --locked -p spark-connect-server` and strict Clippy for all frontend targets. Ubuntu 24.04 supplies a protobuf compiler with From d5cd54024dcafd7357958403457c77061c90f979 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 14:20:12 -0800 Subject: [PATCH 03/18] Enforce consistent Arrow output schemas in the frontend --- .github/workflows/server-frontend.yml | 8 +- crates/spark-connect-server/README.md | 17 +- crates/spark-connect-server/src/engine/mod.rs | 10 +- crates/spark-connect-server/src/lib.rs | 2 + crates/spark-connect-server/src/service.rs | 101 +++- .../tests/engine_injection.rs | 83 +-- crates/spark-connect-server/tests/protocol.rs | 82 +-- .../tests/schema_consistency.rs | 492 ++++++++++++++++++ .../tests/support/server.rs | 80 +++ dev/design/ci.md | 10 +- 10 files changed, 719 insertions(+), 166 deletions(-) create mode 100644 crates/spark-connect-server/tests/schema_consistency.rs create mode 100644 crates/spark-connect-server/tests/support/server.rs diff --git a/.github/workflows/server-frontend.yml b/.github/workflows/server-frontend.yml index a043bb4d..e088800f 100644 --- a/.github/workflows/server-frontend.yml +++ b/.github/workflows/server-frontend.yml @@ -45,7 +45,8 @@ concurrency: env: CARGO_BUILD_JOBS: "2" CARGO_PROFILE_TEST_DEBUG: "0" - RUSTFLAGS: "-D warnings" + # Strict linting uses the pinned toolchain in lint.yml. + RUSTFLAGS: "" jobs: frontend: @@ -62,7 +63,7 @@ jobs: - name: Install Rust toolchain # Use runner-provided rustup to keep actions limited to actions/*. run: | - rustup toolchain install ${{ matrix.rust-version }} --profile minimal --component clippy + rustup toolchain install ${{ matrix.rust-version }} --profile minimal rustup default ${{ matrix.rust-version }} - name: Cache cargo and target @@ -85,6 +86,3 @@ jobs: - name: Test frontend # Protocol and injected-engine tests use local fixtures without an execution backend. run: cargo test --locked -p spark-connect-server - - - name: Strict Clippy for frontend - run: cargo clippy --locked -p spark-connect-server --all-targets --no-deps -- -D warnings diff --git a/crates/spark-connect-server/README.md b/crates/spark-connect-server/README.md index f236fcc8..b00cdd0c 100644 --- a/crates/spark-connect-server/README.md +++ b/crates/spark-connect-server/README.md @@ -43,18 +43,29 @@ Completed responses are not retained. Reattachment, artifact upload/status, The frontend validates UUID session and operation IDs, rejects duplicate active operation IDs within a session, and checks client-observed server session IDs. Client user IDs distinguish session keys but are not authenticated identities. +Protobuf nesting follows the selected Prost features; dependency feature +unification can disable its recursion limit, and the frontend adds no independent +nesting guard. ## Arrow results A prepared query declares an Arrow schema before execution. The frontend converts supported types to the Spark schema representation and checks each batch against -that declaration. Equivalent Arrow representations and differences in batch -nullability or metadata are accepted. Declared field metadata and unsupported -types fail explicitly. +that declaration. Batches may use Spark-equivalent Arrow representations and +nullability different from the declaration. Within one result, emitted IPC +batches must share physical types, field names and nullability, so clients can +assemble them into one Arrow table. Metadata may differ between batches. +Declared field metadata and unsupported types fail explicitly. Engines may yield Arrow record batches or complete Arrow IPC streams containing one batch and its declared row count. The frontend encodes record batches and forwards IPC that fits the response limit; oversized batches are split by rows. +Encoding materializes string/binary view arrays, including nested children, to +`LargeUtf8`/`LargeBinary`. Small view-typed IPC remains unchanged, so mixing it with +oversized view-typed IPC can produce different physical schemas. The frontend +reports `INTERNAL` before sending a batch whose output schema differs from earlier +batches; it does not complete a result containing mixed schemas. Engines should +use consistent non-view IPC types when mixing batch sizes. Result envelopes, including schema and batch responses, are limited to 3 MiB. A single row or schema that cannot fit fails with `RESOURCE_EXHAUSTED`. Each batch carries a cumulative row offset, and empty results still include an empty batch. diff --git a/crates/spark-connect-server/src/engine/mod.rs b/crates/spark-connect-server/src/engine/mod.rs index c4a337da..26798364 100644 --- a/crates/spark-connect-server/src/engine/mod.rs +++ b/crates/spark-connect-server/src/engine/mod.rs @@ -88,9 +88,13 @@ pub enum PreparedOperation { pub trait PreparedQuery: Send { /// The declared Spark schema. Batches must have the same field names and /// Spark-visible types; equivalent Arrow representations are accepted. - /// Batch nullability and metadata may differ and remain in each IPC batch, - /// while the protocol schema header uses this declaration. The declared - /// fields must not contain metadata, which Spark schema conversion rejects. + /// Batch nullability and metadata may differ from this declaration, but + /// emitted IPC batches in one result must share physical types, field names + /// and nullability. Metadata may differ between batches. Encoding materializes + /// view arrays to large-offset types, including when splitting oversized IPC; + /// mixing those with unchanged view IPC fails explicitly before the changed + /// output schema is sent. Declared fields must not contain metadata, which + /// Spark schema conversion rejects. fn schema(&self) -> SchemaRef; /// Dropping this future or its returned stream must cancel backend work; diff --git a/crates/spark-connect-server/src/lib.rs b/crates/spark-connect-server/src/lib.rs index 92f19de6..026e4f6f 100644 --- a/crates/spark-connect-server/src/lib.rs +++ b/crates/spark-connect-server/src/lib.rs @@ -19,6 +19,8 @@ pub mod engine; mod schema; mod service; +/// Generated server bindings for custom embedding. This experimental API follows +/// the RPC table in `build.rs` and may change with the protocol. // tonic's generated async-trait methods duplicate Future's must_use annotation. #[allow(clippy::double_must_use)] pub mod grpc { diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index b4f951da..87460eed 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -20,6 +20,7 @@ use std::panic::AssertUnwindSafe; use std::pin::Pin; use std::sync::{Arc, Mutex}; +use arrow::datatypes::Schema; use arrow::ipc::{reader::StreamReader, writer::StreamWriter}; use arrow::record_batch::RecordBatch; use futures::{FutureExt, Stream, StreamExt}; @@ -36,7 +37,7 @@ use crate::schema::{self, result_schema}; type Output = Pin> + Send>>; type SessionKey = (String, String); -const MAX_ARROW_IPC_BYTES: usize = crate::schema::MAX_RESULT_BYTES; +const MAX_RESULT_BYTES: usize = crate::schema::MAX_RESULT_BYTES; const MAX_RELEASED_SESSIONS: usize = 4096; const RELEASE_WAIT: std::time::Duration = std::time::Duration::from_secs(2); @@ -268,7 +269,7 @@ fn response( fn checked_response( message: proto::ExecutePlanResponse, ) -> Result { - if message.encoded_len() > MAX_ARROW_IPC_BYTES { + if message.encoded_len() > MAX_RESULT_BYTES { return Err(Status::resource_exhausted( "ExecutePlan response exceeds the 3 MiB result limit", )); @@ -372,6 +373,45 @@ fn encode_batch(batch: &RecordBatch) -> Result, Status> { Ok(bytes) } +// PySpark assembles one table from all IPC batches. Its physical types and +// nullability must agree across batches, while field/schema metadata may differ. +fn check_output_schema(data: &[u8], expected: &mut Option) -> Result<(), Status> { + use arrow::datatypes::{DataType, Field, FieldRef}; + + fn field(input: &FieldRef) -> FieldRef { + Arc::new(Field::new( + input.name(), + data_type(input.data_type()), + input.is_nullable(), + )) + } + + fn data_type(input: &DataType) -> DataType { + match input { + DataType::List(child) => DataType::List(field(child)), + DataType::LargeList(child) => DataType::LargeList(field(child)), + DataType::Struct(fields) => DataType::Struct(fields.iter().map(field).collect()), + DataType::Map(entries, sorted) => DataType::Map(field(entries), *sorted), + other => other.clone(), + } + } + + let schema = StreamReader::try_new(data, None) + .map_err(|error| Status::internal(format!("invalid output IPC: {error}")))? + .schema(); + let actual = Schema::new(schema.fields().iter().map(field).collect::>()); + match expected { + Some(expected) if *expected != actual => Err(Status::internal( + "Arrow output schema changed within one result", + )), + Some(_) => Ok(()), + None => { + *expected = Some(actual); + Ok(()) + } + } +} + fn estimated_rows_per_batch(batch: &RecordBatch) -> usize { let bytes = batch.get_array_memory_size(); // Start below the exact IPC limit to allow schema/offset overhead. This is @@ -382,7 +422,7 @@ fn estimated_rows_per_batch(batch: &RecordBatch) -> usize { } fn rows_for_bytes(rows: usize, bytes: usize) -> usize { - (rows as u128 * (MAX_ARROW_IPC_BYTES / 2) as u128 / bytes.max(1) as u128) + (rows as u128 * (MAX_RESULT_BYTES / 2) as u128 / bytes.max(1) as u128) .clamp(1, rows.max(1) as u128) as usize } @@ -406,6 +446,7 @@ async fn send_batch( operation: &str, batch: ResultBatch, offset: &mut i64, + output_schema: &mut Option, guard: &Arc, ) -> Result { let (batch, rows_per_batch) = match batch { @@ -431,7 +472,8 @@ async fn send_batch( batch.clone(), )), ); - if result.encoded_len() <= MAX_ARROW_IPC_BYTES { + if result.encoded_len() <= MAX_RESULT_BYTES { + check_output_schema(&batch.data, output_schema)?; *offset = offset .checked_add(row_count) .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; @@ -467,7 +509,17 @@ async fn send_batch( .map_err(|_| Status::internal("Arrow IPC decoding task failed"))?? } }; - send_arrow_batch(sender, id, operation, batch, rows_per_batch, offset, guard).await + send_arrow_batch( + sender, + id, + operation, + batch, + rows_per_batch, + offset, + output_schema, + guard, + ) + .await } async fn send_arrow_batch( @@ -477,6 +529,7 @@ async fn send_arrow_batch( batch: RecordBatch, rows_per_batch: usize, offset: &mut i64, + output_schema: &mut Option, guard: &Arc, ) -> Result { let mut pending = vec![(batch, rows_per_batch)]; @@ -503,21 +556,22 @@ async fn send_arrow_batch( .await .map_err(|_| Status::internal("Arrow result encoding task failed"))??; let batch = &encoded.batch; + let payload = proto::execute_plan_response::ArrowBatch { + row_count: batch.num_rows() as i64, + data: data.into(), + start_offset: Some(*offset), + ..Default::default() + }; let result = response( id, &guard.session, operation, Some(proto::execute_plan_response::ResponseType::ArrowBatch( - proto::execute_plan_response::ArrowBatch { - row_count: batch.num_rows() as i64, - data: data.into(), - start_offset: Some(*offset), - ..Default::default() - }, + payload.clone(), )), ); let result_bytes = result.encoded_len(); - if result_bytes > MAX_ARROW_IPC_BYTES { + if result_bytes > MAX_RESULT_BYTES { if batch.num_rows() <= 1 { return Err(Status::resource_exhausted( "one Arrow row/schema exceeds the 3 MiB result limit", @@ -525,13 +579,14 @@ async fn send_arrow_batch( } // Recheck every slice including the envelope: skew and schema // overhead can exceed the initial memory estimate. - let rows_per_batch = ((batch.num_rows() as u128 * MAX_ARROW_IPC_BYTES as u128 * 9 + let rows_per_batch = ((batch.num_rows() as u128 * MAX_RESULT_BYTES as u128 * 9 / (result_bytes as u128 * 10)) as usize) .clamp(1, batch.num_rows() - 1); drop(result); pending.push((batch.clone(), rows_per_batch)); continue; } + check_output_schema(&payload.data, output_schema)?; *offset = offset .checked_add(batch.num_rows() as i64) .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; @@ -613,6 +668,7 @@ impl SparkConnectService for ConnectService { } let mut batches = query.execute().await?; let mut offset = 0; + let mut output_schema = None; let mut sent = false; while let Some(batch) = batches.next().await { let batch = batch?; @@ -642,6 +698,7 @@ impl SparkConnectService for ConnectService { &operation, batch, &mut offset, + &mut output_schema, &guard, ) .await? @@ -661,6 +718,7 @@ impl SparkConnectService for ConnectService { &operation, batch.into(), &mut offset, + &mut output_schema, &guard, ) .await? @@ -935,13 +993,13 @@ mod tests { let schema = proto::DataType { kind: Some(proto::data_type::Kind::Struct(proto::data_type::Struct { fields: vec![proto::data_type::StructField { - name: "x".repeat(MAX_ARROW_IPC_BYTES - 64), + name: "x".repeat(MAX_RESULT_BYTES - 64), ..Default::default() }], ..Default::default() })), }; - assert!(schema.encoded_len() < MAX_ARROW_IPC_BYTES); + assert!(schema.encoded_len() < MAX_RESULT_BYTES); let mut header = response( &Uuid::new_v4().to_string(), &session, @@ -971,7 +1029,7 @@ mod tests { .unwrap(); let rows = estimated_rows_per_batch(&batch); assert!(rows > 0 && rows < batch.num_rows()); - assert!(rows * std::mem::size_of::() <= MAX_ARROW_IPC_BYTES / 2); + assert!(rows * std::mem::size_of::() <= MAX_RESULT_BYTES / 2); assert_eq!( estimated_rows_per_batch(&RecordBatch::new_empty(batch.schema())), 1 @@ -1004,6 +1062,7 @@ mod tests { let (guard, _) = session.operation("ipc").unwrap(); let (sender, mut receiver) = mpsc::channel(1); let mut offset = 7; + let mut output_schema = None; assert!(send_batch( &sender, "session", @@ -1013,6 +1072,7 @@ mod tests { row_count: 10 }, &mut offset, + &mut output_schema, &guard, ) .await @@ -1050,12 +1110,13 @@ mod tests { ) .unwrap(); let data = encode_batch(&batch).unwrap(); - assert!(data.len() > MAX_ARROW_IPC_BYTES); - let expected_batches = data.len().div_ceil(MAX_ARROW_IPC_BYTES / 2); + assert!(data.len() > MAX_RESULT_BYTES); + let expected_batches = data.len().div_ceil(MAX_RESULT_BYTES / 2); let session = Arc::new(Session::new(Arc::new(RegistrySession))); let (guard, _) = session.operation("ipc").unwrap(); let (sender, mut receiver) = mpsc::channel(64); let mut offset = 0; + let mut output_schema = None; assert!(send_batch( &sender, "session", @@ -1065,6 +1126,7 @@ mod tests { row_count: 50_000 }, &mut offset, + &mut output_schema, &guard, ) .await @@ -1072,7 +1134,7 @@ mod tests { let (mut rows, mut batches) = (0, 0); while let Ok(result) = receiver.try_recv() { let result = result.unwrap(); - assert!(result.encoded_len() <= MAX_ARROW_IPC_BYTES); + assert!(result.encoded_len() <= MAX_RESULT_BYTES); let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = result.response_type else { @@ -1109,6 +1171,7 @@ mod tests { row_count, }, &mut offset, + &mut output_schema, &guard, ) .await diff --git a/crates/spark-connect-server/tests/engine_injection.rs b/crates/spark-connect-server/tests/engine_injection.rs index a3a81bd3..739a8124 100644 --- a/crates/spark-connect-server/tests/engine_injection.rs +++ b/crates/spark-connect-server/tests/engine_injection.rs @@ -17,6 +17,9 @@ //! Real gRPC tests of the injected backend boundary. The fake is independent of Spark execution. +#[path = "support/server.rs"] +mod server; + use std::collections::HashMap; use std::io::Cursor; use std::pin::Pin; @@ -34,14 +37,12 @@ use futures::{Stream, StreamExt}; use prost::Message; use proto::execute_plan_response::ResponseType; use proto::spark_connect_service_client::SparkConnectServiceClient; +use server::Server; use spark_connect_proto as proto; use spark_connect_server::engine::{ BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, ResultBatch, SessionContext, }; -use tokio::sync::oneshot; -use tokio::task::JoinHandle; -use tonic::transport::Channel; use tonic::{Code, Status}; use uuid::Uuid; @@ -509,56 +510,8 @@ impl Drop for SpyStream { } } -struct Server { - address: std::net::SocketAddr, - client: SparkConnectServiceClient, - shutdown: Option>, - task: Option>, -} - -impl Server { - async fn start(spy: Arc) -> Self { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let (shutdown, stopped) = oneshot::channel(); - let task = tokio::spawn(async move { - spark_connect_server::serve_with_engine(listener, Arc::new(FakeEngine(spy)), async { - let _ = stopped.await; - }) - .await - .unwrap(); - }); - let client = SparkConnectServiceClient::connect(format!("http://{address}")) - .await - .unwrap(); - Self { - address, - client, - shutdown: Some(shutdown), - task: Some(task), - } - } - - async fn stop(mut self) { - if let Some(shutdown) = self.shutdown.take() { - let _ = shutdown.send(()); - } - tokio::time::timeout(Duration::from_secs(5), self.task.take().unwrap()) - .await - .unwrap() - .unwrap(); - } -} - -impl Drop for Server { - fn drop(&mut self) { - if let Some(shutdown) = self.shutdown.take() { - let _ = shutdown.send(()); - } - if let Some(task) = self.task.take() { - task.abort(); - } - } +async fn start_server(spy: Arc) -> Server { + Server::start_with_engine(Arc::new(FakeEngine(spy))).await } fn user(id: &str) -> Option { @@ -727,7 +680,7 @@ async fn start_live( #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn injected_backend_owns_query_analysis_config_and_catalog_state() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; let session = Uuid::new_v4().to_string(); let mut first = server.client.clone(); let mut second = server.client.clone(); @@ -904,7 +857,7 @@ async fn injected_backend_owns_query_analysis_config_and_catalog_state() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn invalid_requests_do_not_open_backends_and_factory_failures_can_retry() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; let session = Uuid::new_v4().to_string(); assert_eq!( server @@ -966,7 +919,7 @@ async fn invalid_requests_do_not_open_backends_and_factory_failures_can_retry() #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn native_streams_drop_on_release_disconnect_session_release_and_shutdown() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; let session = Uuid::new_v4().to_string(); let live = request(&session, LIVE); let operation = live.operation_id.clone().unwrap(); @@ -1060,7 +1013,7 @@ async fn native_streams_drop_on_release_disconnect_session_release_and_shutdown( #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn shutdown_releases_backend_session_while_a_result_remains_unread() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; let (mut client, connection) = h2::client::Builder::new() .initial_window_size(1024) .initial_connection_window_size(1024) @@ -1119,7 +1072,7 @@ async fn shutdown_releases_backend_session_while_a_result_remains_unread() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn operation_reservation_precedes_prepare_and_cleans_up_after_cancellation_and_errors() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; let session = Uuid::new_v4().to_string(); let blocked = request(&session, BLOCKED_PREPARE); let operation = blocked.operation_id.clone().unwrap(); @@ -1195,7 +1148,7 @@ async fn operation_reservation_precedes_prepare_and_cleans_up_after_cancellation #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn release_and_shutdown_cancel_backend_metadata_futures_before_dropping_sessions() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; let session = Uuid::new_v4().to_string(); let mut analysis_client = server.client.clone(); let analysis_session = session.clone(); @@ -1264,7 +1217,7 @@ async fn release_and_shutdown_cancel_backend_metadata_futures_before_dropping_se #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn slow_backend_drop_does_not_hold_the_global_session_registry_lock() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; // Channel clones share an HTTP/2 connection. Establish a new TCP connection // before blocking Drop so the probe isolates the global registry lock. let mut independent = SparkConnectServiceClient::connect(format!("http://{}", server.address)) @@ -1321,7 +1274,7 @@ async fn slow_backend_drop_does_not_hold_the_global_session_registry_lock() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn frontend_rejects_oversized_analysis_and_batch_type_mismatches() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; let session = Uuid::new_v4().to_string(); let status = server .client @@ -1369,7 +1322,7 @@ async fn frontend_rejects_oversized_analysis_and_batch_type_mismatches() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn frontend_rejects_mismatched_and_invalid_ipc_before_forwarding() { - let mut server = Server::start(Arc::new(Spy::default())).await; + let mut server = start_server(Arc::new(Spy::default())).await; let session = Uuid::new_v4().to_string(); for (selector, message) in [ ( @@ -1399,7 +1352,7 @@ async fn frontend_rejects_mismatched_and_invalid_ipc_before_forwarding() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn panicking_execution_reports_internal_after_header_or_partial_result() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; let session = Uuid::new_v4().to_string(); for selector in [PANICKING_EXECUTION, PANICKING_STREAM, PANICKING_STREAM_DROP] { let mut stream = server @@ -1435,7 +1388,7 @@ async fn panicking_execution_reports_internal_after_header_or_partial_result() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn oversized_command_payload_fails_before_sending_any_success_message() { let spy = Arc::new(Spy::default()); - let mut server = Server::start(spy).await; + let mut server = start_server(spy).await; let session = Uuid::new_v4().to_string(); let command = command_request( &session, @@ -1471,7 +1424,7 @@ async fn release_all_timeout_keeps_cancellation_and_operation_ownership_until_dr let gate = Arc::new(DropGate::default()); *spy.stream_drop_gate.lock().unwrap() = Some(gate.clone()); let unblock = UnblockDrop(gate.clone()); - let mut server = Server::start(spy.clone()).await; + let mut server = start_server(spy.clone()).await; let session = Uuid::new_v4().to_string(); let original = request(&session, LIVE); let operation = original.operation_id.clone().unwrap(); diff --git a/crates/spark-connect-server/tests/protocol.rs b/crates/spark-connect-server/tests/protocol.rs index f863426c..1377bcc2 100644 --- a/crates/spark-connect-server/tests/protocol.rs +++ b/crates/spark-connect-server/tests/protocol.rs @@ -17,6 +17,10 @@ //! Verify observable Spark Connect wire behavior independently of DataFrame conveniences. +#[path = "support/server.rs"] +mod server; +mod support; + use std::collections::HashSet; use std::io::Cursor; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -27,59 +31,16 @@ use arrow::array::Int64Array; use arrow::ipc::reader::StreamReader; use prost::Message; use proto::execute_plan_response::ResponseType; -use proto::spark_connect_service_client::SparkConnectServiceClient; use spark_connect_proto as proto; use spark_connect_server::engine::{Engine, EngineSession, SessionContext}; -mod support; +use server::Server; use support::{FixtureEngine, LIVE_FIXTURE}; -use tokio::sync::oneshot; -use tokio::task::JoinHandle; -use tonic::transport::Channel; use tonic::Code; use uuid::Uuid; -struct Server { - client: SparkConnectServiceClient, - shutdown: Option>, - task: Option>, -} - -impl Server { - async fn start() -> Self { - Self::start_with_engine(Arc::new(FixtureEngine::default())).await - } - - async fn start_with_engine(engine: Arc) -> Self { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let (shutdown, stopped) = oneshot::channel(); - let task = tokio::spawn(async move { - spark_connect_server::serve_with_engine(listener, engine, async { - let _ = stopped.await; - }) - .await - .unwrap(); - }); - let client = SparkConnectServiceClient::connect(format!("http://{address}")) - .await - .unwrap(); - Self { - client, - shutdown: Some(shutdown), - task: Some(task), - } - } - - async fn stop(mut self) { - if let Some(shutdown) = self.shutdown.take() { - let _ = shutdown.send(()); - } - tokio::time::timeout(Duration::from_secs(5), self.task.take().unwrap()) - .await - .unwrap() - .unwrap(); - } +async fn start_server() -> Server { + Server::start_with_engine(Arc::new(FixtureEngine::default())).await } #[derive(Default)] @@ -97,17 +58,6 @@ impl Engine for CountingEngine { } } -impl Drop for Server { - fn drop(&mut self) { - if let Some(shutdown) = self.shutdown.take() { - let _ = shutdown.send(()); - } - if let Some(task) = self.task.take() { - task.abort(); - } - } -} - fn range(session: &str, end: i64) -> proto::ExecutePlanRequest { proto::ExecutePlanRequest { session_id: session.into(), @@ -176,7 +126,7 @@ async fn assert_cancelled(stream: &mut tonic::Streaming strings + DataType::Utf8 => strings .as_any() .downcast_ref::() .unwrap() .value(index), - arrow::datatypes::DataType::LargeUtf8 => strings + DataType::LargeUtf8 => strings .as_any() .downcast_ref::() .unwrap() .value(index), - arrow::datatypes::DataType::Utf8View => strings + DataType::Utf8View => strings .as_any() .downcast_ref::() .unwrap() @@ -467,7 +417,7 @@ async fn arrow_payloads_are_byte_limited_and_oversized_rows_fail() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn release_session_cancels_live_stream_and_live_operation_ids_are_unique() { - let mut server = Server::start().await; + let mut server = start_server().await; let session = Uuid::new_v4().to_string(); let request = range(&session, LIVE_FIXTURE); let mut stream = server @@ -646,7 +596,7 @@ async fn invalid_requests_do_not_allocate_sessions_and_unknown_release_is_reject #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn disconnect_removes_operation_and_shutdown_cancels_live_streams() { - let mut server = Server::start().await; + let mut server = start_server().await; let session = Uuid::new_v4().to_string(); let request = range(&session, LIVE_FIXTURE); let operation = request.operation_id.clone().unwrap(); @@ -741,7 +691,7 @@ async fn backend_status_codes_survive_prepare_and_stream_failures() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn deeply_nested_requests_follow_the_configured_protobuf_decoder_limit() { - let mut server = Server::start().await; + let mut server = start_server().await; let session = Uuid::new_v4().to_string(); let mut request = range(&session, 1); let Some(proto::plan::OpType::Root(mut relation)) = request.plan.take().unwrap().op_type else { @@ -790,8 +740,6 @@ async fn deeply_nested_requests_follow_the_configured_protobuf_decoder_limit() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn view_backing_buffers_are_compacted_before_splitting_including_struct_children() { - use std::sync::Arc; - use arrow::array::{BinaryArray, BinaryViewArray, StringArray, StringViewArray, StructArray}; use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatch; diff --git a/crates/spark-connect-server/tests/schema_consistency.rs b/crates/spark-connect-server/tests/schema_consistency.rs new file mode 100644 index 00000000..efb9e8df --- /dev/null +++ b/crates/spark-connect-server/tests/schema_consistency.rs @@ -0,0 +1,492 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Result schemas remain consumable as one Arrow table across output batches. + +#[path = "support/server.rs"] +mod server; + +use std::collections::HashMap; +use std::io::Cursor; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::{ + Array, ArrayRef, BinaryViewArray, LargeBinaryArray, LargeListArray, LargeStringArray, + ListArray, MapArray, StringArray, StringViewArray, StructArray, +}; +use arrow::buffer::OffsetBuffer; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::ipc::{reader::StreamReader, writer::StreamWriter}; +use arrow::record_batch::RecordBatch; +use prost::Message; +use spark_connect_proto as proto; +use spark_connect_server::engine::{ + BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, + ResultBatch, SessionContext, +}; +use tonic::{Code, Status}; +use uuid::Uuid; + +use server::Server; + +const RESULT_LIMIT: usize = 3 * 1024 * 1024; +const LARGE_ROWS: usize = 4096; + +#[derive(Clone)] +enum Batch { + Arrow(RecordBatch), + Ipc(Vec, i64), +} + +#[derive(Clone)] +struct BatchEngine { + schema: SchemaRef, + batches: Vec, +} + +impl Engine for BatchEngine { + fn create_session(&self, _: SessionContext) -> Result, Status> { + Ok(Arc::new(self.clone())) + } +} + +#[tonic::async_trait] +impl EngineSession for BatchEngine { + async fn prepare(&self, _: proto::plan::OpType) -> Result { + Ok(PreparedOperation::Query(Box::new(self.clone()))) + } + + async fn analyze( + &self, + _: proto::analyze_plan_request::Analyze, + ) -> Result { + Err(Status::unimplemented("schema fixture has no analysis")) + } + + async fn config( + &self, + _: proto::config_request::operation::OpType, + ) -> Result { + Err(Status::unimplemented("schema fixture has no configuration")) + } +} + +#[tonic::async_trait] +impl PreparedQuery for BatchEngine { + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + async fn execute(self: Box) -> Result { + Ok(Box::pin(futures::stream::iter( + self.batches.into_iter().map(|batch| { + Ok(match batch { + Batch::Arrow(batch) => ResultBatch::Arrow(batch), + Batch::Ipc(data, row_count) => ResultBatch::Ipc { data, row_count }, + }) + }), + ))) + } +} + +fn ipc(batch: &RecordBatch) -> Vec { + let mut data = Vec::new(); + let mut writer = StreamWriter::try_new(&mut data, batch.schema().as_ref()).unwrap(); + writer.write(batch).unwrap(); + writer.finish().unwrap(); + drop(writer); + data +} + +async fn execute(engine: BatchEngine) -> (Vec, Option) { + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let request = proto::ExecutePlanRequest { + session_id: Uuid::new_v4().to_string(), + operation_id: Some(Uuid::new_v4().to_string()), + plan: Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(proto::Relation { + rel_type: Some(proto::relation::RelType::Range(proto::Range { + end: 1, + step: 1, + ..Default::default() + })), + ..Default::default() + })), + }), + ..Default::default() + }; + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + let result = tokio::time::timeout(Duration::from_secs(30), async { + let mut responses = Vec::new(); + loop { + match stream.message().await { + Ok(Some(response)) => { + assert!(response.encoded_len() <= RESULT_LIMIT); + responses.push(response); + } + Ok(None) => break (responses, None), + Err(error) => break (responses, Some(error)), + } + } + }) + .await + .expect("schema fixture execution must terminate"); + server.stop().await; + result +} + +fn output_batches(responses: &[proto::ExecutePlanResponse]) -> Vec { + let mut offset = 0; + let mut batches = Vec::new(); + for response in responses { + if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + &response.response_type + { + assert_eq!(batch.start_offset, Some(offset)); + let mut reader = StreamReader::try_new(Cursor::new(&batch.data), None).unwrap(); + let decoded = reader.next().unwrap().unwrap(); + assert_eq!(decoded.num_rows() as i64, batch.row_count); + assert!(reader.next().is_none()); + offset += batch.row_count; + batches.push(decoded); + } + } + batches +} + +fn assert_schema_error(responses: &[proto::ExecutePlanResponse], status: Option) { + let status = status.expect("a changed output schema must fail"); + assert_eq!(status.code(), Code::Internal); + assert_eq!( + status.message(), + "Arrow output schema changed within one result" + ); + assert!(!completed(responses)); +} + +fn completed(responses: &[proto::ExecutePlanResponse]) -> bool { + responses.iter().any(|response| { + matches!( + response.response_type, + Some(proto::execute_plan_response::ResponseType::ResultComplete( + _ + )) + ) + }) +} + +fn strings(data_type: DataType, nullable: bool) -> RecordBatch { + let values: ArrayRef = match data_type { + DataType::Utf8 => Arc::new(StringArray::from(vec!["value"])), + DataType::LargeUtf8 => Arc::new(LargeStringArray::from(vec!["value"])), + _ => unreachable!(), + }; + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("value", data_type, nullable)])), + vec![values], + ) + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn changed_physical_types_and_nullability_fail_before_the_second_batch() { + let first = strings(DataType::Utf8, false); + for second in [ + strings(DataType::LargeUtf8, false), + strings(DataType::Utf8, true), + ] { + let (responses, status) = execute(BatchEngine { + schema: first.schema(), + batches: vec![Batch::Arrow(first.clone()), Batch::Arrow(second)], + }) + .await; + assert_schema_error(&responses, status); + let batches = output_batches(&responses); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].schema(), first.schema()); + } +} + +fn metadata_batch(note: &str) -> RecordBatch { + let metadata = HashMap::from([("producer.note".to_string(), note.to_string())]); + let item = + Arc::new(Field::new("item", DataType::LargeUtf8, true).with_metadata(metadata.clone())); + let values = ListArray::new( + item.clone(), + OffsetBuffer::new(vec![0, 1].into()), + Arc::new(LargeStringArray::from(vec!["value"])), + None, + ); + RecordBatch::try_new( + Arc::new(Schema::new_with_metadata( + vec![Field::new("value", DataType::List(item), true).with_metadata(metadata.clone())], + metadata, + )), + vec![Arc::new(values)], + ) + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn metadata_can_change_and_batch_schema_can_differ_from_the_declaration() { + let declared = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::List(Arc::new(Field::new("item", DataType::Utf8, false))), + false, + )])); + let (responses, status) = execute(BatchEngine { + schema: declared, + batches: vec![ + Batch::Arrow(metadata_batch("first")), + Batch::Arrow(metadata_batch("second")), + ], + }) + .await; + assert!(status.is_none(), "{status:?}"); + assert!(completed(&responses)); + let batches = output_batches(&responses); + assert_eq!(batches.len(), 2); + for (batch, note) in batches.iter().zip(["first", "second"]) { + let schema = batch.schema(); + assert_eq!(schema.metadata()["producer.note"], note); + assert_eq!(schema.field(0).metadata()["producer.note"], note); + let DataType::List(item) = schema.field(0).data_type() else { + panic!("expected list output"); + }; + assert_eq!(item.metadata()["producer.note"], note); + assert_eq!(item.data_type(), &DataType::LargeUtf8); + assert!(schema.field(0).is_nullable()); + assert!(item.is_nullable()); + } +} + +fn text(row: usize) -> String { + format!("{row:04}{}", "x".repeat(1020)) +} + +fn views(rows: usize) -> RecordBatch { + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Utf8View, + false, + )])), + vec![Arc::new(StringViewArray::from_iter_values( + (0..rows).map(text), + ))], + ) + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_ipc_is_unchanged_when_small_and_materialized_when_oversized() { + for rows in [2, LARGE_ROWS] { + let batch = views(rows); + let data = ipc(&batch); + assert_eq!(data.len() > RESULT_LIMIT, rows == LARGE_ROWS); + let (responses, status) = execute(BatchEngine { + schema: batch.schema(), + batches: vec![Batch::Ipc(data.clone(), rows as i64)], + }) + .await; + assert!(status.is_none(), "{status:?}"); + assert!(completed(&responses)); + let batches = output_batches(&responses); + if rows == 2 { + let payload = responses + .iter() + .find_map(|response| match &response.response_type { + Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) => { + Some(&batch.data) + } + _ => None, + }) + .unwrap(); + assert_eq!(payload.as_ref(), data.as_slice()); + assert_eq!( + batches[0].schema().field(0).data_type(), + &DataType::Utf8View + ); + } else { + assert!(batches.len() > 1); + let mut offset = 0; + for batch in batches { + assert_eq!(batch.schema().field(0).data_type(), &DataType::LargeUtf8); + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for row in 0..batch.num_rows() { + assert_eq!(values.value(row), text(offset + row)); + } + offset += batch.num_rows(); + } + assert_eq!(offset, rows); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn mixing_small_and_oversized_view_ipc_rejects_the_changed_output_schema() { + let small = views(2); + let large = views(LARGE_ROWS); + let small = Batch::Ipc(ipc(&small), 2); + let large = Batch::Ipc(ipc(&large), LARGE_ROWS as i64); + for large_first in [false, true] { + let (responses, status) = execute(BatchEngine { + schema: views(0).schema(), + batches: if large_first { + vec![large.clone(), small.clone()] + } else { + vec![small.clone(), large.clone()] + }, + }) + .await; + assert_schema_error(&responses, status); + let batches = output_batches(&responses); + let expected_type = if large_first { + DataType::LargeUtf8 + } else { + DataType::Utf8View + }; + let expected_rows = if large_first { LARGE_ROWS } else { 2 }; + assert_eq!( + batches.iter().map(RecordBatch::num_rows).sum::(), + expected_rows + ); + for batch in batches { + assert_eq!(batch.schema().field(0).data_type(), &expected_type); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_children_in_lists_large_lists_and_maps_are_materialized_when_split() { + let values: ArrayRef = Arc::new(StringViewArray::from_iter_values((0..LARGE_ROWS).map(text))); + let offsets = OffsetBuffer::new((0..=LARGE_ROWS as i32).collect::>().into()); + let item = Arc::new(Field::new("item", DataType::Utf8View, false)); + let lists = ListArray::new(item.clone(), offsets.clone(), values.clone(), None); + let large_lists = LargeListArray::new( + item, + OffsetBuffer::new((0..=LARGE_ROWS as i64).collect::>().into()), + values.clone(), + None, + ); + let entries = StructArray::new( + vec![ + Field::new("key", DataType::Utf8View, false), + Field::new("value", DataType::BinaryView, false), + ] + .into(), + vec![ + values, + Arc::new(BinaryViewArray::from_iter_values( + (0..LARGE_ROWS).map(|row| text(row).into_bytes()), + )), + ], + None, + ); + let maps = MapArray::new( + Arc::new(Field::new("entries", entries.data_type().clone(), false)), + offsets, + entries, + None, + false, + ); + let columns: Vec = vec![Arc::new(lists), Arc::new(large_lists), Arc::new(maps)]; + let schema = Arc::new(Schema::new( + columns + .iter() + .enumerate() + .map(|(index, column)| { + Field::new(format!("value{index}"), column.data_type().clone(), false) + }) + .collect::>(), + )); + let batch = RecordBatch::try_new(schema.clone(), columns).unwrap(); + let (responses, status) = execute(BatchEngine { + schema, + batches: vec![Batch::Arrow(batch)], + }) + .await; + assert!(status.is_none(), "{status:?}"); + assert!(completed(&responses)); + let batches = output_batches(&responses); + assert!(batches.len() > 1); + let mut offset = 0; + for batch in batches { + let lists = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let large_lists = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let maps = batch.column(2).as_any().downcast_ref::().unwrap(); + for row in 0..batch.num_rows() { + let expected = text(offset + row); + let list = lists.value(row); + assert_eq!( + list.as_any() + .downcast_ref::() + .unwrap() + .value(0), + expected + ); + let large_list = large_lists.value(row); + assert_eq!( + large_list + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + expected + ); + let entries = maps.value(row); + assert_eq!( + entries + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + expected + ); + assert_eq!( + entries + .column(1) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + expected.as_bytes() + ); + } + offset += batch.num_rows(); + } + assert_eq!(offset, LARGE_ROWS); +} diff --git a/crates/spark-connect-server/tests/support/server.rs b/crates/spark-connect-server/tests/support/server.rs new file mode 100644 index 00000000..0ec88720 --- /dev/null +++ b/crates/spark-connect-server/tests/support/server.rs @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use spark_connect_proto::spark_connect_service_client::SparkConnectServiceClient; +use spark_connect_server::engine::Engine; +use tokio::sync::oneshot; +use tokio::task::JoinHandle; +use tonic::transport::Channel; + +pub struct Server { + // Some test binaries need a second connection to the same listener. + #[allow(dead_code)] + pub address: SocketAddr, + pub client: SparkConnectServiceClient, + pub shutdown: Option>, + task: Option>, +} + +impl Server { + pub async fn start_with_engine(engine: Arc) -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (shutdown, stopped) = oneshot::channel(); + let task = tokio::spawn(async move { + spark_connect_server::serve_with_engine(listener, engine, async { + let _ = stopped.await; + }) + .await + .unwrap(); + }); + let client = SparkConnectServiceClient::connect(format!("http://{address}")) + .await + .unwrap(); + Self { + address, + client, + shutdown: Some(shutdown), + task: Some(task), + } + } + + pub async fn stop(mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + tokio::time::timeout(Duration::from_secs(5), self.task.take().unwrap()) + .await + .unwrap() + .unwrap(); + } +} + +impl Drop for Server { + fn drop(&mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + if let Some(task) = self.task.take() { + task.abort(); + } + } +} diff --git a/dev/design/ci.md b/dev/design/ci.md index e2f68161..d30dcf3b 100644 --- a/dev/design/ci.md +++ b/dev/design/ci.md @@ -16,8 +16,8 @@ The CI consists of these main workflows: and macOS universal2 wheels using the release recipe 4. **Coverage** (`.github/workflows/coverage.yml`) - Rust and Python coverage, with regression floors -5. **Experimental server frontend** (`.github/workflows/server-frontend.yml`) - Test and lint - the optional Rust protocol frontend with stable Rust and MSRV 1.88 +5. **Experimental server frontend** (`.github/workflows/server-frontend.yml`) - Test the + optional Rust protocol frontend with stable Rust and MSRV 1.88 The other workflows are `lint.yml` (rustfmt, clippy, rustdoc, ruff), `generated-drift.yml`, `vendor-drift.yml`, `docs.yml`, `pandas-parity.yml`, `pipelines-parity.yml` and `release.yml`. @@ -75,8 +75,10 @@ This job should always pass; failures indicate incomplete implementation or test This focused workflow runs on frontend, protocol, workspace manifest, lockfile, Cargo configuration, toolchain and workflow changes. Each stable Rust and Rust -1.88.0 job has a 45-minute limit and runs `cargo test --locked -p spark-connect-server` -and strict Clippy for all frontend targets. Ubuntu 24.04 supplies a protobuf compiler with +1.88.0 job has a 45-minute limit and runs `cargo test --locked -p spark-connect-server`. +The existing workspace lint job covers the frontend with its pinned Rust toolchain; +test jobs do not promote warnings from a floating or older compiler to errors. +Ubuntu 24.04 supplies a protobuf compiler with proto3 optional support. Tests use injected engine fixtures and require no execution backend; they cover frontend protocol behavior rather than Spark execution compatibility. From 9441b6cd3a08d9f82bf258bf506587bcf9aedc59 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 15:14:03 -0800 Subject: [PATCH 04/18] Fix frontend slicing, identifiers and shutdown lifecycle --- Cargo.lock | 1 + Cargo.toml | 3 + crates/spark-connect-server/Cargo.toml | 9 +- crates/spark-connect-server/README.md | 19 +- crates/spark-connect-server/build.rs | 45 +- crates/spark-connect-server/src/engine/mod.rs | 14 +- crates/spark-connect-server/src/lib.rs | 44 +- crates/spark-connect-server/src/service.rs | 151 ++++++- .../tests/review_regressions.rs | 425 ++++++++++++++++++ .../tests/server_shutdown.rs | 192 ++++++++ 10 files changed, 865 insertions(+), 38 deletions(-) create mode 100644 crates/spark-connect-server/tests/review_regressions.rs create mode 100644 crates/spark-connect-server/tests/server_shutdown.rs diff --git a/Cargo.lock b/Cargo.lock index 61cee6fd..7e32c1e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4057,6 +4057,7 @@ dependencies = [ "futures", "h2", "prost", + "prost-types", "tokio", "tokio-stream", "tonic", diff --git a/Cargo.toml b/Cargo.toml index 473aebeb..9ba59607 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,11 +90,14 @@ should_implement_trait = "allow" tonic = { version = "0.14", default-features = false, features = ["channel", "codegen"] } tonic-prost = "0.14" tonic-prost-build = "0.14" +tonic-build = "0.14" +h2 = "0.4" prost = "0.14" prost-types = "0.14" # async runtime tokio = { version = "1", features = ["rt-multi-thread", "sync", "macros", "time"] } +tokio-stream = "0.1" futures = "0.3" # Arrow (result decoding). diff --git a/crates/spark-connect-server/Cargo.toml b/crates/spark-connect-server/Cargo.toml index a27d0413..a923d66e 100644 --- a/crates/spark-connect-server/Cargo.toml +++ b/crates/spark-connect-server/Cargo.toml @@ -33,13 +33,16 @@ arrow.workspace = true futures.workspace = true prost.workspace = true tokio = { workspace = true, features = ["net"] } -tokio-stream = { version = "0.1", features = ["net"] } +tokio-stream = { workspace = true, features = ["net"] } tonic = { workspace = true, features = ["server", "router"] } tonic-prost.workspace = true uuid.workspace = true [build-dependencies] -tonic-build = "0.14" +tonic-build.workspace = true +spark-connect-proto = { package = "apache-spark-connect-proto", path = "../spark-connect-proto" } +prost.workspace = true +prost-types.workspace = true [dev-dependencies] -h2 = "0.4" +h2.workspace = true diff --git a/crates/spark-connect-server/README.md b/crates/spark-connect-server/README.md index b00cdd0c..564b6531 100644 --- a/crates/spark-connect-server/README.md +++ b/crates/spark-connect-server/README.md @@ -40,8 +40,9 @@ Completed responses are not retained. Reattachment, artifact upload/status, `Interrupt`, enriched error details, session cloning and status queries return `UNIMPLEMENTED`. -The frontend validates UUID session and operation IDs, rejects duplicate active -operation IDs within a session, and checks client-observed server session IDs. +The frontend requires lowercase, hyphenated UUID session and operation IDs, +rejects duplicate active operation IDs within a session, and checks +client-observed server session IDs. Client user IDs distinguish session keys but are not authenticated identities. Protobuf nesting follows the selected Prost features; dependency feature unification can disable its recursion limit, and the frontend adds no independent @@ -74,7 +75,8 @@ carries a cumulative row offset, and empty results still include an empty batch. Engine methods and streams must cancel their owned work when dropped. The frontend cancels them when a client disconnects, an operation is released, a -session is released or server shutdown begins. Results pass through a bounded +session is released, server shutdown begins or the serving future is dropped. +Results pass through a bounded channel, so unread responses eventually stop polling the engine. Blocking Arrow encoding retains operation ownership until its worker finishes. @@ -84,6 +86,17 @@ for transport drain; returning from `serve_with_engine` does not prove that an engine's independently scheduled or blocking work has finished. The caller must honor the engine's cleanup contract before releasing its resources. +`ReleaseSession(allow_reconnect=true)` permits a new backend session with the +same protocol ID while the previous incarnation still drains. Engines must keep +the two incarnations' resources independent. Commands may commit effects before +frontend response validation fails, for example on an oversized payload; that +failure provides no rollback or safe-retry guarantee. + +With panic unwinding enabled, panics in `prepare`, `analyze`, `config` and query +execution are translated to `INTERNAL`. Synchronous session creation and query +schema access are outside that boundary and must not panic. Resource cleanup +must not panic either. + This is an experiment for trusted local clients and trusted engine outputs. Keep the supplied listener on loopback: this frontend supplies no authentication, authorization or TLS. Requests can be up to 64 MiB. Active sessions and operations diff --git a/crates/spark-connect-server/build.rs b/crates/spark-connect-server/build.rs index 3ffc7807..5041d662 100644 --- a/crates/spark-connect-server/build.rs +++ b/crates/spark-connect-server/build.rs @@ -15,13 +15,14 @@ // specific language governing permissions and limitations // under the License. +use prost::Message; use tonic_build::manual::{Builder, Method, Service}; fn main() { // The official protocol crate already generates messages and client bindings. // Generate only the standard service routes, using those exact message types. - // Keep this RPC list and its streaming signatures synchronized with - // spark-connect-proto/proto/spark/connect/base.proto on each protocol bump. + // Check this table against the generated protocol descriptor before emitting + // routes: streaming signatures are part of wire compatibility. let methods = [ ( "execute_plan", @@ -120,6 +121,46 @@ fn main() { false, ), ]; + let descriptors = + prost_types::FileDescriptorSet::decode(spark_connect_proto::FILE_DESCRIPTOR_SET) + .expect("valid Spark Connect descriptor set"); + let descriptor = descriptors + .file + .iter() + .filter(|file| file.package.as_deref() == Some("spark.connect")) + .flat_map(|file| &file.service) + .find(|service| service.name.as_deref() == Some("SparkConnectService")) + .expect("SparkConnectService descriptor"); + assert_eq!(methods.len(), descriptor.method.len(), "RPC table drift"); + let mut routes = std::collections::HashSet::new(); + for &(_, route, input, output, client_stream, server_stream) in &methods { + assert!(routes.insert(route), "duplicate RPC route: {route}"); + let method = descriptor + .method + .iter() + .find(|method| method.name.as_deref() == Some(route)) + .unwrap_or_else(|| panic!("missing protocol RPC: {route}")); + assert_eq!( + method.input_type.as_deref(), + Some(format!(".spark.connect.{input}").as_str()), + "{route} request type drift" + ); + assert_eq!( + method.output_type.as_deref(), + Some(format!(".spark.connect.{output}").as_str()), + "{route} response type drift" + ); + assert_eq!( + method.client_streaming.unwrap_or(false), + client_stream, + "{route} client streaming drift" + ); + assert_eq!( + method.server_streaming.unwrap_or(false), + server_stream, + "{route} server streaming drift" + ); + } let mut service = Service::builder() .name("SparkConnectService") .package("spark.connect"); diff --git a/crates/spark-connect-server/src/engine/mod.rs b/crates/spark-connect-server/src/engine/mod.rs index 26798364..d2f44b46 100644 --- a/crates/spark-connect-server/src/engine/mod.rs +++ b/crates/spark-connect-server/src/engine/mod.rs @@ -43,9 +43,12 @@ pub struct SessionContext { /// Factory for sessions bound to one native engine. pub trait Engine: Send + Sync { - /// Allocate local session state without blocking or performing I/O. + /// Allocate local session state without blocking, performing I/O or panicking. /// Defer asynchronous initialization to the session methods. The frontend - /// serializes creation to ensure one backend session per protocol session. + /// serializes creation to ensure one backend session per active incarnation. + /// A reconnecting session release permits a replacement with the same + /// `session_id` while references to the previous incarnation are still + /// draining. The engine must keep those incarnations' resources independent. fn create_session(&self, context: SessionContext) -> Result, Status>; } @@ -53,6 +56,7 @@ pub trait Engine: Send + Sync { /// /// Dropping a method future must cancel its work. Dropping the final session /// reference releases its resources after in-flight operations have finished. +/// Resource cleanup must not panic. // async_trait repeats Future's must_use annotation on generated trait methods. #[allow(clippy::double_must_use)] #[tonic::async_trait] @@ -78,6 +82,9 @@ pub trait EngineSession: Send + Sync { pub enum PreparedOperation { Query(Box), /// The frontend adds envelope IDs and the final ResultComplete message. + /// Eager command effects may already be committed when later frontend + /// response validation fails, for example when a payload exceeds the size + /// limit. Such a failure does not roll back effects or make retries safe. Command(Vec), } @@ -86,7 +93,8 @@ pub enum PreparedOperation { #[allow(clippy::double_must_use)] #[tonic::async_trait] pub trait PreparedQuery: Send { - /// The declared Spark schema. Batches must have the same field names and + /// The declared Spark schema. This synchronous accessor must not panic. + /// Batches must have the same field names and /// Spark-visible types; equivalent Arrow representations are accepted. /// Batch nullability and metadata may differ from this declaration, but /// emitted IPC batches in one result must share physical types, field names diff --git a/crates/spark-connect-server/src/lib.rs b/crates/spark-connect-server/src/lib.rs index 026e4f6f..dece96a2 100644 --- a/crates/spark-connect-server/src/lib.rs +++ b/crates/spark-connect-server/src/lib.rs @@ -30,29 +30,42 @@ pub mod grpc { )); } +struct CancelOnDrop(std::sync::Arc); + +impl Drop for CancelOnDrop { + fn drop(&mut self) { + self.0.cancel_all(); + } +} + +fn tcp_incoming(listener: tokio::net::TcpListener) -> tonic::transport::server::TcpIncoming { + // Builder socket options do not apply to serve_with_incoming_shutdown. + tonic::transport::server::TcpIncoming::from(listener).with_nodelay(Some(true)) +} + /// Serve with an injected execution engine and a fresh session registry. /// /// The frontend binds each protocol session to the engine session created for it. /// The caller owns the engine and signals shutdown before releasing its resources. +/// Dropping this future also cancels registered engine work, without waiting for +/// the bounded transport drain used when shutdown is signalled. pub async fn serve_with_engine( listener: tokio::net::TcpListener, engine: std::sync::Arc, shutdown: impl std::future::Future + Send + 'static, ) -> Result<(), tonic::transport::Error> { let service = std::sync::Arc::new(service::ConnectService::new(engine)); + let _cancel_on_drop = CancelOnDrop(service.clone()); let router = tonic::transport::Server::builder().add_service( grpc::spark_connect_service_server::SparkConnectServiceServer::from_arc(service.clone()) .max_decoding_message_size(64 * 1024 * 1024), ); let (draining, drain_started) = tokio::sync::oneshot::channel(); - let server = router.serve_with_incoming_shutdown( - tokio_stream::wrappers::TcpListenerStream::new(listener), - async move { - shutdown.await; - service.cancel_all(); - let _ = draining.send(()); - }, - ); + let server = router.serve_with_incoming_shutdown(tcp_incoming(listener), async move { + shutdown.await; + service.cancel_all(); + let _ = draining.send(()); + }); tokio::pin!(server); tokio::select! { result = &mut server => result, @@ -63,3 +76,18 @@ pub async fn serve_with_engine( } } } + +#[cfg(test)] +mod tests { + use futures::StreamExt; + + #[tokio::test] + async fn accepted_connections_disable_nagle() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let mut incoming = super::tcp_incoming(listener); + let _client = tokio::net::TcpStream::connect(address).await.unwrap(); + let accepted = incoming.next().await.unwrap().unwrap(); + assert!(accepted.nodelay().unwrap()); + } +} diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index 87460eed..968f3ffb 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -41,6 +41,25 @@ const MAX_RESULT_BYTES: usize = crate::schema::MAX_RESULT_BYTES; const MAX_RELEASED_SESSIONS: usize = 4096; const RELEASE_WAIT: std::time::Duration = std::time::Duration::from_secs(2); +fn require_canonical_uuid(value: &str, name: &str) -> Result<(), Status> { + if Uuid::parse_str(value).is_ok_and(|id| id.hyphenated().to_string() == value) { + Ok(()) + } else { + Err(Status::invalid_argument(format!( + "{name} must be a canonical lowercase hyphenated UUID" + ))) + } +} + +async fn catch_engine_panic( + action: impl std::future::Future> + Send, +) -> Result { + AssertUnwindSafe(action) + .catch_unwind() + .await + .unwrap_or_else(|_| Err(Status::internal("engine request panicked"))) +} + struct Session { id: String, backend: Arc, @@ -172,7 +191,7 @@ impl ConnectService { } fn key(id: &str, user: Option<&proto::UserContext>) -> Result { - Uuid::parse_str(id).map_err(|_| Status::invalid_argument("session_id must be a UUID"))?; + require_canonical_uuid(id, "session_id")?; Ok(( user.map(|u| u.user_id.clone()).unwrap_or_default(), id.into(), @@ -375,7 +394,7 @@ fn encode_batch(batch: &RecordBatch) -> Result, Status> { // PySpark assembles one table from all IPC batches. Its physical types and // nullability must agree across batches, while field/schema metadata may differ. -fn check_output_schema(data: &[u8], expected: &mut Option) -> Result<(), Status> { +fn check_output_schema(schema: &Schema, expected: &mut Option) -> Result<(), Status> { use arrow::datatypes::{DataType, Field, FieldRef}; fn field(input: &FieldRef) -> FieldRef { @@ -396,9 +415,6 @@ fn check_output_schema(data: &[u8], expected: &mut Option) -> Result<(), } } - let schema = StreamReader::try_new(data, None) - .map_err(|error| Status::internal(format!("invalid output IPC: {error}")))? - .schema(); let actual = Schema::new(schema.fields().iter().map(field).collect::>()); match expected { Some(expected) if *expected != actual => Err(Status::internal( @@ -413,17 +429,43 @@ fn check_output_schema(data: &[u8], expected: &mut Option) -> Result<(), } fn estimated_rows_per_batch(batch: &RecordBatch) -> usize { - let bytes = batch.get_array_memory_size(); + use arrow::datatypes::DataType; + + fn contains_view(data_type: &DataType) -> bool { + match data_type { + DataType::Utf8View | DataType::BinaryView => true, + DataType::List(field) | DataType::LargeList(field) | DataType::Map(field, _) => { + contains_view(field.data_type()) + } + DataType::Struct(fields) => fields.iter().any(|field| contains_view(field.data_type())), + _ => false, + } + } + + let bytes = batch.columns().iter().fold(0usize, |bytes, column| { + // ArrayData's slice estimate omits variadic view buffers. Keep the + // conservative allocation estimate for those until encoding measures + // their materialized payload; ordinary slices need only their own bytes. + let size = if contains_view(column.data_type()) { + column.get_array_memory_size() + } else { + column + .to_data() + .get_slice_memory_size() + .unwrap_or_else(|_| column.get_array_memory_size()) + }; + bytes.saturating_add(size) + }); // Start below the exact IPC limit to allow schema/offset overhead. This is // only an estimate: skewed rows and annotations are checked after encoding. - // Include retained backing allocations so even sliced/view inputs begin - // conservatively without materializing an entire oversized engine batch. - rows_for_bytes(batch.num_rows(), bytes) + // Nested children may still retain unsliced buffers, making the initial + // estimate conservative. Successful encodings can grow the next row budget. + rows_for_bytes(batch.num_rows(), bytes).min(batch.num_rows().max(1)) } fn rows_for_bytes(rows: usize, bytes: usize) -> usize { (rows as u128 * (MAX_RESULT_BYTES / 2) as u128 / bytes.max(1) as u128) - .clamp(1, rows.max(1) as u128) as usize + .clamp(1, usize::MAX as u128) as usize } // Fields drop in declaration order, including when encoding panics or a @@ -445,6 +487,7 @@ async fn send_batch( id: &str, operation: &str, batch: ResultBatch, + batch_schema: &Schema, offset: &mut i64, output_schema: &mut Option, guard: &Arc, @@ -473,7 +516,7 @@ async fn send_batch( )), ); if result.encoded_len() <= MAX_RESULT_BYTES { - check_output_schema(&batch.data, output_schema)?; + check_output_schema(batch_schema, output_schema)?; *offset = offset .checked_add(row_count) .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; @@ -586,7 +629,17 @@ async fn send_arrow_batch( pending.push((batch.clone(), rows_per_batch)); continue; } - check_output_schema(&payload.data, output_schema)?; + let emitted_schema = StreamReader::try_new(&payload.data[..], None) + .map_err(|error| Status::internal(format!("invalid output IPC: {error}")))? + .schema(); + check_output_schema(emitted_schema.as_ref(), output_schema)?; + if let Some((_, next_rows)) = pending.last_mut() { + // A retained backing allocation can dwarf this slice's IPC. Adapt + // from actual output, but limit growth so one tiny sample cannot + // request an arbitrarily large/skewed materialization next. + *next_rows = rows_for_bytes(batch.num_rows(), result_bytes) + .min(batch.num_rows().saturating_mul(4).max(1)); + } *offset = offset .checked_add(batch.num_rows() as i64) .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; @@ -610,8 +663,7 @@ impl SparkConnectService for ConnectService { let operation = req .operation_id .unwrap_or_else(|| Uuid::new_v4().to_string()); - Uuid::parse_str(&operation) - .map_err(|_| Status::invalid_argument("operation_id must be a UUID"))?; + require_canonical_uuid(&operation, "operation_id")?; let op = req .plan .and_then(|p| p.op_type) @@ -631,7 +683,7 @@ impl SparkConnectService for ConnectService { biased; _ = cancelled.changed() => return Err(Status::cancelled("session released")), _ = operation_cancelled.changed() => return Err(Status::cancelled("operation released")), - prepared = session.backend.prepare(op) => prepared?, + prepared = catch_engine_panic(async { session.backend.prepare(op).await }) => prepared?, }; let query = match prepared { PreparedOperation::Query(query) => query, @@ -697,6 +749,7 @@ impl SparkConnectService for ConnectService { &req.session_id, &operation, batch, + batch_schema.as_ref(), &mut offset, &mut output_schema, &guard, @@ -712,11 +765,13 @@ impl SparkConnectService for ConnectService { drop(batches); if !sent { let batch = RecordBatch::new_empty(arrow_schema); + let batch_schema = batch.schema(); if !send_batch( &sender, &req.session_id, &operation, batch.into(), + batch_schema.as_ref(), &mut offset, &mut output_schema, &guard, @@ -771,7 +826,11 @@ impl SparkConnectService for ConnectService { req.user_context.as_ref(), req.client_observed_server_side_session_id.as_deref(), )?; - let result = session.while_open(session.backend.analyze(input)).await?; + let result = session + .while_open(catch_engine_panic(async { + session.backend.analyze(input).await + })) + .await?; if let proto::analyze_plan_response::Result::Schema(result) = &result { if let Some(schema) = &result.schema { schema::validate_schema_size(schema)?; @@ -798,7 +857,11 @@ impl SparkConnectService for ConnectService { req.user_context.as_ref(), req.client_observed_server_side_session_id.as_deref(), )?; - let result = session.while_open(session.backend.config(input)).await?; + let result = session + .while_open(catch_engine_panic(async { + session.backend.config(input).await + })) + .await?; Ok(Response::new(proto::ConfigResponse { session_id: req.session_id, server_side_session_id: session.id.clone(), @@ -812,8 +875,7 @@ impl SparkConnectService for ConnectService { request: Request, ) -> Result, Status> { let req = request.into_inner(); - Uuid::parse_str(&req.operation_id) - .map_err(|_| Status::invalid_argument("operation_id must be a UUID"))?; + require_canonical_uuid(&req.operation_id, "operation_id")?; let session = self.find_session( &req.session_id, req.user_context.as_ref(), @@ -1036,6 +1098,54 @@ mod tests { ); } + #[tokio::test] + async fn successful_encoding_grows_a_conservative_row_budget() { + use arrow::array::Int64Array; + use arrow::datatypes::{DataType, Field, Schema}; + + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), + vec![Arc::new(Int64Array::from_iter_values(0..1000))], + ) + .unwrap(); + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("adaptive").unwrap(); + let (sender, mut receiver) = mpsc::channel(1024); + let mut offset = 0; + let mut output_schema = None; + assert!(send_arrow_batch( + &sender, + "session", + "adaptive", + batch, + 1, + &mut offset, + &mut output_schema, + &guard, + ) + .await + .unwrap()); + let (mut batches, mut rows) = (0, 0); + while let Ok(result) = receiver.try_recv() { + let result = result.unwrap(); + assert!(result.encoded_len() <= MAX_RESULT_BYTES); + let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + result.response_type + else { + panic!("expected Arrow response"); + }; + assert_eq!(batch.start_offset, Some(rows)); + batches += 1; + rows += batch.row_count; + } + assert_eq!(rows, 1000); + assert_eq!(offset, rows); + assert!( + batches <= 8, + "a conservative estimate must adapt, got {batches} batches" + ); + } + #[tokio::test] async fn existing_ipc_is_forwarded_unchanged() { use arrow::array::Int64Array; @@ -1071,6 +1181,7 @@ mod tests { data: data.clone(), row_count: 10 }, + batch.schema().as_ref(), &mut offset, &mut output_schema, &guard, @@ -1125,6 +1236,7 @@ mod tests { data: data.clone(), row_count: 50_000 }, + batch.schema().as_ref(), &mut offset, &mut output_schema, &guard, @@ -1170,6 +1282,7 @@ mod tests { data: data.clone(), row_count, }, + batch.schema().as_ref(), &mut offset, &mut output_schema, &guard, diff --git a/crates/spark-connect-server/tests/review_regressions.rs b/crates/spark-connect-server/tests/review_regressions.rs new file mode 100644 index 00000000..e821bd81 --- /dev/null +++ b/crates/spark-connect-server/tests/review_regressions.rs @@ -0,0 +1,425 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Regressions for sliced output, protocol IDs and backend panic boundaries. + +#[path = "support/server.rs"] +mod server; + +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::Int64Array; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::ipc::reader::StreamReader; +use arrow::record_batch::RecordBatch; +use prost::Message; +use server::Server; +use spark_connect_proto as proto; +use spark_connect_server::engine::{ + BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, + ResultBatch, SessionContext, +}; +use tonic::{Code, Status}; +use uuid::Uuid; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Call { + Prepare, + Analyze, + Config, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum PanicAt { + Construction, + Poll, +} + +struct State { + batches: Vec, + panic: Option<(Call, PanicAt)>, + panicked: AtomicBool, + sessions: AtomicUsize, + prepares: AtomicUsize, +} + +impl State { + fn new(batches: Vec, panic: Option<(Call, PanicAt)>) -> Arc { + Arc::new(Self { + batches, + panic, + panicked: AtomicBool::new(false), + sessions: AtomicUsize::new(0), + prepares: AtomicUsize::new(0), + }) + } + + fn panic_once(&self, call: Call) -> Option { + self.panic + .filter(|(target, _)| *target == call) + .filter(|_| !self.panicked.swap(true, Ordering::SeqCst)) + .map(|(_, at)| at) + } +} + +struct Fixture(Arc); + +impl Engine for Fixture { + fn create_session(&self, _: SessionContext) -> Result, Status> { + self.0.sessions.fetch_add(1, Ordering::SeqCst); + Ok(Arc::new(Self(self.0.clone()))) + } +} + +type EngineFuture<'a, T> = Pin> + Send + 'a>>; + +// Implement the expanded async-trait signatures directly so the fixture can +// panic both before returning a future and while that future is polled. +impl EngineSession for Fixture { + fn prepare<'life0, 'async_trait>( + &'life0 self, + _: proto::plan::OpType, + ) -> EngineFuture<'async_trait, PreparedOperation> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + self.0.prepares.fetch_add(1, Ordering::SeqCst); + let panic = self.0.panic_once(Call::Prepare); + assert!(panic != Some(PanicAt::Construction), "prepare construction"); + Box::pin(async move { + assert!(panic != Some(PanicAt::Poll), "prepare poll"); + Ok(PreparedOperation::Query(Box::new(Query( + self.0.batches.clone(), + )))) + }) + } + + fn analyze<'life0, 'async_trait>( + &'life0 self, + _: proto::analyze_plan_request::Analyze, + ) -> EngineFuture<'async_trait, proto::analyze_plan_response::Result> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + let panic = self.0.panic_once(Call::Analyze); + assert!(panic != Some(PanicAt::Construction), "analyze construction"); + Box::pin(async move { + assert!(panic != Some(PanicAt::Poll), "analyze poll"); + Ok(proto::analyze_plan_response::Result::SparkVersion( + proto::analyze_plan_response::SparkVersion { + version: "fixture".into(), + }, + )) + }) + } + + fn config<'life0, 'async_trait>( + &'life0 self, + _: proto::config_request::operation::OpType, + ) -> EngineFuture<'async_trait, ConfigResult> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + let panic = self.0.panic_once(Call::Config); + assert!(panic != Some(PanicAt::Construction), "config construction"); + Box::pin(async move { + assert!(panic != Some(PanicAt::Poll), "config poll"); + Ok(ConfigResult::default()) + }) + } +} + +struct Query(Vec); + +#[tonic::async_trait] +impl PreparedQuery for Query { + fn schema(&self) -> SchemaRef { + self.0[0].schema() + } + + async fn execute(self: Box) -> Result { + Ok(Box::pin(futures::stream::iter( + self.0 + .into_iter() + .map(|batch| Ok(ResultBatch::Arrow(batch))), + ))) + } +} + +fn integers(rows: usize) -> RecordBatch { + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), + vec![Arc::new(Int64Array::from_iter_values(0..rows as i64))], + ) + .unwrap() +} + +fn execute(session: &str, operation: &str) -> proto::ExecutePlanRequest { + proto::ExecutePlanRequest { + session_id: session.into(), + operation_id: Some(operation.into()), + plan: Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(proto::Relation::default())), + }), + ..Default::default() + } +} + +fn config(session: &str) -> proto::ConfigRequest { + proto::ConfigRequest { + session_id: session.into(), + operation: Some(proto::config_request::Operation { + op_type: Some(proto::config_request::operation::OpType::GetAll( + Default::default(), + )), + }), + ..Default::default() + } +} + +fn analyze(session: &str) -> proto::AnalyzePlanRequest { + proto::AnalyzePlanRequest { + session_id: session.into(), + analyze: Some(proto::analyze_plan_request::Analyze::SparkVersion( + Default::default(), + )), + ..Default::default() + } +} + +fn release(session: &str, operation: &str) -> proto::ReleaseExecuteRequest { + proto::ReleaseExecuteRequest { + session_id: session.into(), + operation_id: operation.into(), + release: Some(proto::release_execute_request::Release::ReleaseAll( + Default::default(), + )), + ..Default::default() + } +} + +async fn collect( + server: &mut Server, + session: &str, + operation: &str, +) -> Vec { + tokio::time::timeout(Duration::from_secs(15), async { + let mut stream = server + .client + .execute_plan(execute(session, operation)) + .await + .unwrap() + .into_inner(); + let mut output = Vec::new(); + while let Some(message) = stream.message().await.unwrap() { + output.push(message); + } + assert!(matches!( + output.last().unwrap().response_type.as_ref(), + Some(proto::execute_plan_response::ResponseType::ResultComplete( + _ + )) + )); + output + }) + .await + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn slices_of_a_large_allocation_remain_one_response_per_small_batch() { + let backing = integers(10_000_000); + let batches = (0..20) + .map(|index| backing.slice(index * 1000, 1000)) + .collect(); + let state = State::new(batches, None); + let mut server = Server::start_with_engine(Arc::new(Fixture(state))).await; + let output = collect( + &mut server, + &Uuid::new_v4().to_string(), + &Uuid::new_v4().to_string(), + ) + .await; + let (mut batches, mut rows) = (0, 0); + for message in output { + assert!(message.encoded_len() <= 3 * 1024 * 1024); + if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + message.response_type + { + batches += 1; + assert_eq!(batch.start_offset, Some(rows)); + assert_eq!(batch.row_count, 1000); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + let decoded = reader.next().unwrap().unwrap(); + let values = decoded + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(values.values().iter().copied().eq(rows..rows + 1000)); + assert!(reader.next().is_none()); + rows += batch.row_count; + } + } + assert_eq!(batches, 20); + assert_eq!(rows, 20_000); + server.stop().await; +} + +#[tokio::test] +async fn noncanonical_ids_are_rejected_without_creating_or_releasing_state() { + let state = State::new(vec![integers(1)], None); + let mut server = Server::start_with_engine(Arc::new(Fixture(state.clone()))).await; + let uuid = Uuid::parse_str("abcdef12-3456-4789-abcd-0123456789ab").unwrap(); + let session = uuid.to_string(); + let operation = Uuid::new_v4().to_string(); + let original = server + .client + .config(config(&session)) + .await + .unwrap() + .into_inner() + .server_side_session_id; + for alias in [ + uuid.simple().to_string(), + uuid.braced().to_string(), + uuid.urn().to_string(), + session.to_uppercase(), + ] { + assert_eq!( + server + .client + .config(config(&alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .analyze_plan(analyze(&alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .execute_plan(execute(&alias, &operation)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_execute(release(&alias, &operation)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_session(proto::ReleaseSessionRequest { + session_id: alias.clone(), + ..Default::default() + }) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .execute_plan(execute(&Uuid::new_v4().to_string(), &alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_execute(release(&session, &alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .config(config(&session)) + .await + .unwrap() + .into_inner() + .server_side_session_id, + original + ); + } + assert_eq!(state.sessions.load(Ordering::SeqCst), 1); + assert_eq!(state.prepares.load(Ordering::SeqCst), 0); + collect(&mut server, &session, &operation).await; + server.stop().await; +} + +#[tokio::test] +async fn backend_request_panics_have_internal_status_and_leave_the_session_usable() { + for call in [Call::Prepare, Call::Analyze, Call::Config] { + for at in [PanicAt::Construction, PanicAt::Poll] { + let state = State::new(vec![integers(1)], Some((call, at))); + let mut server = Server::start_with_engine(Arc::new(Fixture(state.clone()))).await; + let session = Uuid::new_v4().to_string(); + let operation = Uuid::new_v4().to_string(); + let status = match call { + Call::Prepare => server + .client + .execute_plan(execute(&session, &operation)) + .await + .unwrap_err(), + Call::Analyze => server + .client + .analyze_plan(analyze(&session)) + .await + .unwrap_err(), + Call::Config => server.client.config(config(&session)).await.unwrap_err(), + }; + assert_eq!(status.code(), Code::Internal); + assert_eq!(status.message(), "engine request panicked"); + server.client.config(config(&session)).await.unwrap(); + server.client.analyze_plan(analyze(&session)).await.unwrap(); + // Reuse the operation ID after a preparation panic: its reservation + // must have been released before the failed RPC returned. + collect(&mut server, &session, &operation).await; + assert_eq!(state.sessions.load(Ordering::SeqCst), 1); + server.stop().await; + } + } +} diff --git a/crates/spark-connect-server/tests/server_shutdown.rs b/crates/spark-connect-server/tests/server_shutdown.rs new file mode 100644 index 00000000..ac624c93 --- /dev/null +++ b/crates/spark-connect-server/tests/server_shutdown.rs @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Duration; + +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use futures::Stream; +use spark_connect_proto as proto; +use spark_connect_server::engine::{ + BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, + ResultBatch, SessionContext, +}; +use tonic::Status; + +#[derive(Default)] +struct Spy { + streams_started: AtomicUsize, + streams_dropped: AtomicUsize, + configs_started: AtomicUsize, + configs_dropped: AtomicUsize, + sessions_dropped: AtomicUsize, +} + +struct PendingEngine(Arc); + +impl Engine for PendingEngine { + fn create_session(&self, _context: SessionContext) -> Result, Status> { + Ok(Arc::new(PendingSession(self.0.clone()))) + } +} + +struct PendingSession(Arc); + +impl Drop for PendingSession { + fn drop(&mut self) { + self.0.sessions_dropped.fetch_add(1, Ordering::SeqCst); + } +} + +#[tonic::async_trait] +impl EngineSession for PendingSession { + async fn prepare(&self, _operation: proto::plan::OpType) -> Result { + Ok(PreparedOperation::Query(Box::new(PendingQuery( + self.0.clone(), + )))) + } + + async fn analyze( + &self, + _input: proto::analyze_plan_request::Analyze, + ) -> Result { + Err(Status::unimplemented("shutdown fixture does not analyze")) + } + + async fn config( + &self, + _input: proto::config_request::operation::OpType, + ) -> Result { + let _call = PendingConfig(self.0.clone()); + self.0.configs_started.fetch_add(1, Ordering::SeqCst); + std::future::pending().await + } +} + +struct PendingConfig(Arc); + +impl Drop for PendingConfig { + fn drop(&mut self) { + self.0.configs_dropped.fetch_add(1, Ordering::SeqCst); + } +} + +struct PendingQuery(Arc); + +#[tonic::async_trait] +impl PreparedQuery for PendingQuery { + fn schema(&self) -> SchemaRef { + Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])) + } + + async fn execute(self: Box) -> Result { + self.0.streams_started.fetch_add(1, Ordering::SeqCst); + Ok(Box::pin(PendingResults(self.0.clone()))) + } +} + +struct PendingResults(Arc); + +impl Stream for PendingResults { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Pending + } +} + +impl Drop for PendingResults { + fn drop(&mut self) { + self.0.streams_dropped.fetch_add(1, Ordering::SeqCst); + } +} + +async fn wait_for_count(counter: &AtomicUsize, expected: usize) { + tokio::time::timeout(Duration::from_secs(5), async { + while counter.load(Ordering::SeqCst) != expected { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("backend work must finish without disconnecting its client"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn aborting_serve_cancels_backend_work_before_clients_disconnect() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let spy = Arc::new(Spy::default()); + let server = tokio::spawn(spark_connect_server::serve_with_engine( + listener, + Arc::new(PendingEngine(spy.clone())), + std::future::pending(), + )); + let mut client = proto::spark_connect_service_client::SparkConnectServiceClient::connect( + format!("http://{address}"), + ) + .await + .unwrap(); + let session = uuid::Uuid::new_v4().to_string(); + let mut result = client + .execute_plan(proto::ExecutePlanRequest { + session_id: session.clone(), + operation_id: Some(uuid::Uuid::new_v4().to_string()), + plan: Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(proto::Relation::default())), + }), + ..Default::default() + }) + .await + .unwrap() + .into_inner(); + assert!(result.message().await.unwrap().unwrap().schema.is_some()); + wait_for_count(&spy.streams_started, 1).await; + + let mut config_client = client.clone(); + let config = tokio::spawn(async move { + config_client + .config(proto::ConfigRequest { + session_id: session, + operation: Some(proto::config_request::Operation { + op_type: Some(proto::config_request::operation::OpType::GetAll( + Default::default(), + )), + }), + ..Default::default() + }) + .await + }); + wait_for_count(&spy.configs_started, 1).await; + server.abort(); + assert!(server.await.unwrap_err().is_cancelled()); + + // Retain both the streamed response and its client connection: cancellation + // must come from dropping serve_with_engine, not from a peer disconnect. + wait_for_count(&spy.streams_dropped, 1).await; + wait_for_count(&spy.configs_dropped, 1).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert!(tokio::time::timeout(Duration::from_secs(5), config) + .await + .unwrap() + .unwrap() + .is_err()); + drop(result); + drop(client); +} From 4cadf0705f7ca3d042c8d5bb2a389866b63438ca Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 15:40:04 -0800 Subject: [PATCH 05/18] Size nested slices before Arrow encoding --- crates/spark-connect-core/Cargo.toml | 2 +- crates/spark-connect-server/Cargo.toml | 2 +- crates/spark-connect-server/src/engine/mod.rs | 3 + crates/spark-connect-server/src/service.rs | 264 ++++++++++++++++-- .../tests/review_regressions.rs | 189 ++++++++++++- 5 files changed, 415 insertions(+), 45 deletions(-) diff --git a/crates/spark-connect-core/Cargo.toml b/crates/spark-connect-core/Cargo.toml index a34fb898..9d4e6c9d 100644 --- a/crates/spark-connect-core/Cargo.toml +++ b/crates/spark-connect-core/Cargo.toml @@ -32,7 +32,7 @@ prost-types.workspace = true crc32fast = "1" sha2 = "0.10" # Transport errors that tonic keeps as a status's source, for `RetryPolicy::can_retry`. -h2 = "0.4" +h2.workspace = true hyper = "1" # The Basic credential and the percent-decoded userinfo of a proxy URI, already built for tonic. base64 = "0.22" diff --git a/crates/spark-connect-server/Cargo.toml b/crates/spark-connect-server/Cargo.toml index a923d66e..5fe56bcc 100644 --- a/crates/spark-connect-server/Cargo.toml +++ b/crates/spark-connect-server/Cargo.toml @@ -33,7 +33,7 @@ arrow.workspace = true futures.workspace = true prost.workspace = true tokio = { workspace = true, features = ["net"] } -tokio-stream = { workspace = true, features = ["net"] } +tokio-stream.workspace = true tonic = { workspace = true, features = ["server", "router"] } tonic-prost.workspace = true uuid.workspace = true diff --git a/crates/spark-connect-server/src/engine/mod.rs b/crates/spark-connect-server/src/engine/mod.rs index d2f44b46..64f6a8a1 100644 --- a/crates/spark-connect-server/src/engine/mod.rs +++ b/crates/spark-connect-server/src/engine/mod.rs @@ -57,6 +57,9 @@ pub trait Engine: Send + Sync { /// Dropping a method future must cancel its work. Dropping the final session /// reference releases its resources after in-flight operations have finished. /// Resource cleanup must not panic. +/// With unwinding enabled, panics in these methods become `INTERNAL` errors; +/// the frontend keeps the session available for later requests. Implementations +/// must leave their state usable when unwinding. // async_trait repeats Future's must_use annotation on generated trait methods. #[allow(clippy::double_must_use)] #[tonic::async_trait] diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index 968f3ffb..e57bbcc8 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -70,6 +70,8 @@ struct Session { struct Operation { cancelled: watch::Sender, done: watch::Sender, + #[cfg(test)] + encoding_attempts: std::sync::atomic::AtomicUsize, } impl Session { @@ -107,6 +109,8 @@ impl Session { let running = Arc::new(Operation { cancelled, done: watch::channel(false).0, + #[cfg(test)] + encoding_attempts: std::sync::atomic::AtomicUsize::new(0), }); let mut operations = self .operations @@ -428,38 +432,70 @@ fn check_output_schema(schema: &Schema, expected: &mut Option) -> Result } } -fn estimated_rows_per_batch(batch: &RecordBatch) -> usize { +fn estimated_array_bytes(array: &dyn arrow::array::Array) -> usize { + use arrow::array::{Array, AsArray, OffsetSizeTrait}; use arrow::datatypes::DataType; - fn contains_view(data_type: &DataType) -> bool { - match data_type { - DataType::Utf8View | DataType::BinaryView => true, - DataType::List(field) | DataType::LargeList(field) | DataType::Map(field, _) => { - contains_view(field.data_type()) - } - DataType::Struct(fields) => fields.iter().any(|field| contains_view(field.data_type())), - _ => false, - } + fn nested_bytes(offsets: &[O], values: &dyn Array) -> usize { + let start = offsets[0].as_usize(); + let end = offsets[offsets.len() - 1].as_usize(); + // List and map slices retain their full child arrays. Like the IPC + // writer, inspect only the child range referenced by these offsets. + offsets + .len() + .saturating_mul(std::mem::size_of::()) + .saturating_add(estimated_array_bytes( + values.slice(start, end - start).as_ref(), + )) } - let bytes = batch.columns().iter().fold(0usize, |bytes, column| { - // ArrayData's slice estimate omits variadic view buffers. Keep the - // conservative allocation estimate for those until encoding measures - // their materialized payload; ordinary slices need only their own bytes. - let size = if contains_view(column.data_type()) { - column.get_array_memory_size() - } else { - column + fn view_bytes(len: usize, lengths: impl Iterator) -> usize { + // Encoding materializes views with 64-bit offsets. Count repeated + // references separately, including null-slot lengths: Arrow's cast + // reserves their bytes even though it emits no value for a null slot. + lengths.fold(len.saturating_add(1).saturating_mul(8), |bytes, len| { + bytes.saturating_add(len as usize) + }) + } + + let bytes = match array.data_type() { + DataType::List(_) => { + let list = array.as_list::(); + nested_bytes(list.value_offsets(), list.values().as_ref()) + } + DataType::LargeList(_) => { + let list = array.as_list::(); + nested_bytes(list.value_offsets(), list.values().as_ref()) + } + DataType::Map(_, _) => { + let map = array.as_map(); + nested_bytes(map.value_offsets(), map.entries()) + } + DataType::Struct(_) => array + .as_struct() + .columns() + .iter() + .fold(0usize, |bytes, child| { + bytes.saturating_add(estimated_array_bytes(child.as_ref())) + }), + DataType::Utf8View => view_bytes(array.len(), array.as_string_view().lengths()), + DataType::BinaryView => view_bytes(array.len(), array.as_binary_view().lengths()), + _ => { + return array .to_data() .get_slice_memory_size() - .unwrap_or_else(|_| column.get_array_memory_size()) - }; - bytes.saturating_add(size) + .unwrap_or_else(|_| array.get_array_memory_size()); + } + }; + bytes.saturating_add(array.nulls().map_or(0, |_| array.len().div_ceil(8))) +} + +fn estimated_rows_per_batch(batch: &RecordBatch) -> usize { + let bytes = batch.columns().iter().fold(0usize, |bytes, column| { + bytes.saturating_add(estimated_array_bytes(column.as_ref())) }); // Start below the exact IPC limit to allow schema/offset overhead. This is // only an estimate: skewed rows and annotations are checked after encoding. - // Nested children may still retain unsliced buffers, making the initial - // estimate conservative. Successful encodings can grow the next row budget. rows_for_bytes(batch.num_rows(), bytes).min(batch.num_rows().max(1)) } @@ -477,6 +513,11 @@ struct EncodingBatch { impl EncodingBatch { fn encode(self) -> Result<(Self, Vec), Status> { + #[cfg(test)] + self._guard + .running + .encoding_attempts + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); let data = encode_batch(&self.batch)?; Ok((self, data)) } @@ -577,9 +618,29 @@ async fn send_arrow_batch( ) -> Result { let mut pending = vec![(batch, rows_per_batch)]; while let Some((batch, rows_per_batch)) = pending.pop() { + let mut rows_per_batch = rows_per_batch.max(1).min(batch.num_rows().max(1)); + while rows_per_batch > 1 { + let prefix; + let candidate = if rows_per_batch >= batch.num_rows() { + &batch + } else { + prefix = batch.slice(0, rows_per_batch); + &prefix + }; + let estimated = estimated_rows_per_batch(candidate); + if estimated >= candidate.num_rows() { + break; + } + // Growth from a light prefix can reach a dense tail. Inspect its + // logical buffers before encoding rather than repeatedly copying + // an already-known oversized payload. Exact IPC/envelope checks + // below still handle schema overhead and an unfittable single row. + rows_per_batch = estimated; + } // Keep the remainder as one slice instead of allocating a pending entry // per row in a large engine batch. - let batch = if batch.num_rows() > rows_per_batch { + let has_remainder = batch.num_rows() > rows_per_batch; + let batch = if has_remainder { pending.push(( batch.slice(rows_per_batch, batch.num_rows() - rows_per_batch), rows_per_batch, @@ -633,12 +694,14 @@ async fn send_arrow_batch( .map_err(|error| Status::internal(format!("invalid output IPC: {error}")))? .schema(); check_output_schema(emitted_schema.as_ref(), output_schema)?; - if let Some((_, next_rows)) = pending.last_mut() { - // A retained backing allocation can dwarf this slice's IPC. Adapt - // from actual output, but limit growth so one tiny sample cannot - // request an arbitrarily large/skewed materialization next. - *next_rows = rows_for_bytes(batch.num_rows(), result_bytes) - .min(batch.num_rows().saturating_mul(4).max(1)); + if has_remainder { + if let Some((_, next_rows)) = pending.last_mut() { + // Adapt this piece's own remainder, without changing an outer + // sibling when the piece finishes. The row-growth limit is a + // sampling heuristic, not a bound on bytes for skewed rows. + *next_rows = rows_for_bytes(batch.num_rows(), result_bytes) + .min(batch.num_rows().saturating_mul(4).max(1)); + } } *offset = offset .checked_add(batch.num_rows() as i64) @@ -1098,6 +1161,31 @@ mod tests { ); } + #[test] + fn view_estimates_count_repeated_and_null_slot_lengths() { + use arrow::array::{BinaryViewArray, StringViewArray}; + use arrow::buffer::NullBuffer; + + let source = StringViewArray::from(vec!["abcdefghijklmno"]); + let (views, buffers, _) = source.into_parts(); + let repeated = vec![views[0]; 4].into(); + let nulls = Some(NullBuffer::from(vec![true, false, true, false])); + let strings = StringViewArray::new(repeated, buffers, nulls); + // Four logical values reserve four payloads during materialization, + // even though they share one buffer and two slots are null. IPC also + // needs five 64-bit offsets and a validity byte. + let expected = 4 * 15 + 5 * 8 + 1; + assert_eq!(estimated_array_bytes(&strings), expected); + assert_eq!(strings.total_bytes_len(), 2 * 15); + let (views, buffers, nulls) = strings.into_parts(); + let binary = BinaryViewArray::new(views, buffers, nulls); + assert_eq!(estimated_array_bytes(&binary), expected); + assert_eq!( + estimated_array_bytes(&binary.slice(1, 2)), + 2 * 15 + 3 * 8 + 1 + ); + } + #[tokio::test] async fn successful_encoding_grows_a_conservative_row_budget() { use arrow::array::Int64Array; @@ -1146,6 +1234,122 @@ mod tests { ); } + #[tokio::test] + async fn skewed_output_checks_prefixes_and_recovers_light_suffixes() { + use arrow::array::{Int64Array, LargeStringArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use std::sync::atomic::Ordering; + + for shape in ["large_tail", "large_head", "dense_middle"] { + let (prefix_rows, suffix_rows) = match shape { + "large_tail" => (5000, 0), + "large_head" => (0, 5000), + _ => (100_000, 100_000), + }; + let mut sizes = vec![0usize; prefix_rows]; + sizes.extend(std::iter::repeat_n(1024 * 1024, 40)); + sizes.extend(std::iter::repeat_n(0, suffix_rows)); + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::LargeUtf8, false), + ])), + vec![ + Arc::new(Int64Array::from_iter_values(0..sizes.len() as i64)), + Arc::new(LargeStringArray::from_iter_values( + sizes.iter().map(|size| "z".repeat(*size)), + )), + ], + ) + .unwrap(); + let budget = estimated_rows_per_batch(&batch); + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("skewed").unwrap(); + // Even an accidentally permanent one-row budget must not block + // this regression before its response-count assertion can fail. + let (sender, mut receiver) = mpsc::channel(sizes.len()); + let mut offset = 0; + let mut output_schema = None; + assert!(tokio::time::timeout( + std::time::Duration::from_secs(30), + send_arrow_batch( + &sender, + "session", + "skewed", + batch, + budget, + &mut offset, + &mut output_schema, + &guard, + ) + ) + .await + .unwrap() + .unwrap()); + let (mut responses, mut empty_responses, mut rows) = (0usize, 0usize, 0usize); + while let Ok(result) = receiver.try_recv() { + let result = result.unwrap(); + assert!(result.encoded_len() <= MAX_RESULT_BYTES); + let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + result.response_type + else { + panic!("expected Arrow response"); + }; + assert_eq!(batch.start_offset, Some(rows as i64)); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + let decoded = reader.next().unwrap().unwrap(); + assert_eq!(decoded.num_rows() as i64, batch.row_count); + assert!(reader.next().is_none()); + let ids = decoded + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let strings = decoded + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let mut empty = true; + for index in 0..decoded.num_rows() { + assert_eq!(ids.value(index), rows as i64); + let value = strings.value(index); + assert_eq!(value.len(), sizes[rows]); + assert!(value.bytes().all(|byte| byte == b'z')); + empty &= value.is_empty(); + rows += 1; + } + responses += 1; + empty_responses += usize::from(empty); + } + assert_eq!(rows, sizes.len()); + assert_eq!(offset, rows as i64); + let attempts = guard.running.encoding_attempts.load(Ordering::SeqCst); + // Logical prefix checks must avoid repeatedly materializing the + // same unconsumed dense region after successes on empty prefixes. + assert!(attempts <= responses + 4, + "{shape}: {attempts} encodes for {responses} responses repeatedly encoded an oversized region"); + if shape == "dense_middle" { + // A sticky reduced row budget also fragments a light suffix + // inside a failed piece. Let it recover once the dense rows + // are consumed, as well as across outer sibling slices. + assert!( + responses < 80, + "dense-middle output fragmented into {responses} responses" + ); + assert!( + empty_responses <= 35, + "dense-middle output emitted {empty_responses} all-empty responses" + ); + } else if shape == "large_head" { + assert!( + responses < 100, + "large-head output fragmented into {responses} responses" + ); + } + } + } + #[tokio::test] async fn existing_ipc_is_forwarded_unchanged() { use arrow::array::Int64Array; diff --git a/crates/spark-connect-server/tests/review_regressions.rs b/crates/spark-connect-server/tests/review_regressions.rs index e821bd81..992fc8a6 100644 --- a/crates/spark-connect-server/tests/review_regressions.rs +++ b/crates/spark-connect-server/tests/review_regressions.rs @@ -26,7 +26,11 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; -use arrow::array::Int64Array; +use arrow::array::{ + Array, ArrayRef, BinaryViewArray, Int64Array, LargeBinaryArray, LargeListArray, + LargeStringArray, ListArray, MapArray, StringViewArray, StructArray, +}; +use arrow::buffer::OffsetBuffer; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::ipc::reader::StreamReader; use arrow::record_batch::RecordBatch; @@ -247,9 +251,7 @@ async fn collect( .unwrap() } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn slices_of_a_large_allocation_remain_one_response_per_small_batch() { - let backing = integers(10_000_000); +async fn collect_small_slices(backing: RecordBatch) -> Vec { let batches = (0..20) .map(|index| backing.slice(index * 1000, 1000)) .collect(); @@ -261,30 +263,191 @@ async fn slices_of_a_large_allocation_remain_one_response_per_small_batch() { &Uuid::new_v4().to_string(), ) .await; - let (mut batches, mut rows) = (0, 0); + let (mut batches, mut rows) = (Vec::new(), 0); for message in output { assert!(message.encoded_len() <= 3 * 1024 * 1024); if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = message.response_type { - batches += 1; assert_eq!(batch.start_offset, Some(rows)); assert_eq!(batch.row_count, 1000); let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); let decoded = reader.next().unwrap().unwrap(); - let values = decoded - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert!(values.values().iter().copied().eq(rows..rows + 1000)); + assert_eq!(decoded.num_rows() as i64, batch.row_count); + batches.push(decoded); assert!(reader.next().is_none()); rows += batch.row_count; } } - assert_eq!(batches, 20); + assert_eq!(batches.len(), 20); assert_eq!(rows, 20_000); server.stop().await; + batches +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn slices_of_a_large_allocation_remain_one_response_per_small_batch() { + for (index, batch) in collect_small_slices(integers(10_000_000)) + .await + .iter() + .enumerate() + { + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let start = index as i64 * 1000; + assert!(values.values().iter().copied().eq(start..start + 1000)); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn nested_slices_only_budget_the_referenced_child_range() { + // The parent needs only 20,000 rows to retain the full 80 MB child. Exercise + // each offset width and maps sequentially, sharing the same allocation. + let child: ArrayRef = Arc::new(Int64Array::from_iter_values(0..10_000_000)); + for kind in 0..3 { + let item = Arc::new(Field::new("item", DataType::Int64, false)); + let column: ArrayRef = match kind { + 0 => Arc::new(ListArray::new( + item, + OffsetBuffer::new((0..=20_000i32).collect::>().into()), + child.clone(), + None, + )), + 1 => Arc::new(LargeListArray::new( + item, + OffsetBuffer::new((0..=20_000i64).collect::>().into()), + child.clone(), + None, + )), + _ => { + let fields = vec![ + Field::new("key", DataType::Int64, false), + Field::new("value", DataType::Int64, false), + ]; + let entries = StructArray::new( + fields.clone().into(), + vec![child.clone(), child.clone()], + None, + ); + Arc::new(MapArray::new( + Arc::new(Field::new( + "entries", + DataType::Struct(fields.into()), + false, + )), + OffsetBuffer::new((0..=20_000i32).collect::>().into()), + entries, + None, + false, + )) + } + }; + let schema = Arc::new(Schema::new(vec![Field::new( + "nested", + column.data_type().clone(), + false, + )])); + let backing = RecordBatch::try_new(schema, vec![column]).unwrap(); + let batches = collect_small_slices(backing).await; + let mut row = 0i64; + for batch in batches { + for index in 0..batch.num_rows() { + let value = match kind { + 0 => batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(index), + 1 => batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(index), + _ => { + let entry = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(index); + assert_eq!(entry.len(), 1); + let key = entry + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(key.value(0), row); + entry.column(1).clone() + } + }; + let value = value.as_any().downcast_ref::().unwrap(); + assert_eq!(value.len(), 1); + assert_eq!(value.value(0), row); + row += 1; + } + } + assert_eq!(row, 20_000); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_slices_only_budget_the_current_values() { + fn text(row: usize) -> String { + format!("{row:010}{}", "y".repeat(30)) + } + + let values = StringViewArray::from_iter_values((0..1_000_000).map(text)); + let (views, buffers, nulls) = values.into_parts(); + for binary in [false, true] { + let column: ArrayRef = if binary { + Arc::new(BinaryViewArray::new( + views.clone(), + buffers.clone(), + nulls.clone(), + )) + } else { + Arc::new(StringViewArray::new( + views.clone(), + buffers.clone(), + nulls.clone(), + )) + }; + let schema = Arc::new(Schema::new(vec![Field::new( + "view", + column.data_type().clone(), + false, + )])); + let backing = RecordBatch::try_new(schema, vec![column]).unwrap(); + let batches = collect_small_slices(backing).await; + let mut row = 0; + for batch in batches { + for index in 0..batch.num_rows() { + let expected = text(row); + if binary { + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(values.value(index), expected.as_bytes()); + } else { + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(values.value(index), expected); + } + row += 1; + } + } + assert_eq!(row, 20_000); + } } #[tokio::test] From 8c4bea4ca8d194d95708411019e3ab871b727824 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 16:42:30 -0800 Subject: [PATCH 06/18] Account for fixed IPC overhead and reject row count overflow --- crates/spark-connect-server/src/service.rs | 339 +++++++++++++++++- .../tests/review_regressions.rs | 64 ++++ 2 files changed, 390 insertions(+), 13 deletions(-) diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index e57bbcc8..beb0cf81 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -491,17 +491,30 @@ fn estimated_array_bytes(array: &dyn arrow::array::Array) -> usize { } fn estimated_rows_per_batch(batch: &RecordBatch) -> usize { + estimated_rows_for_payload(batch, MAX_RESULT_BYTES / 2) +} + +fn estimated_rows_for_payload(batch: &RecordBatch, payload_bytes: usize) -> usize { let bytes = batch.columns().iter().fold(0usize, |bytes, column| { bytes.saturating_add(estimated_array_bytes(column.as_ref())) }); // Start below the exact IPC limit to allow schema/offset overhead. This is // only an estimate: skewed rows and annotations are checked after encoding. - rows_for_bytes(batch.num_rows(), bytes).min(batch.num_rows().max(1)) + rows_for_bytes(batch.num_rows(), bytes, payload_bytes).min(batch.num_rows().max(1)) } -fn rows_for_bytes(rows: usize, bytes: usize) -> usize { - (rows as u128 * (MAX_RESULT_BYTES / 2) as u128 / bytes.max(1) as u128) - .clamp(1, usize::MAX as u128) as usize +fn rows_for_bytes(rows: usize, bytes: usize, payload_bytes: usize) -> usize { + (rows as u128 * payload_bytes as u128 / bytes.max(1) as u128).clamp(1, usize::MAX as u128) + as usize +} + +fn checked_arrow_row_count(rows: usize, offset: i64) -> Result { + let rows = i64::try_from(rows) + .map_err(|_| Status::resource_exhausted("result batch row count exceeds i64"))?; + offset + .checked_add(rows) + .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; + Ok(rows) } // Fields drop in declaration order, including when encoding panics or a @@ -535,6 +548,7 @@ async fn send_batch( ) -> Result { let (batch, rows_per_batch) = match batch { ResultBatch::Arrow(batch) => { + checked_arrow_row_count(batch.num_rows(), *offset)?; let rows = estimated_rows_per_batch(&batch); (batch, rows) } @@ -542,6 +556,9 @@ async fn send_batch( if row_count < 0 { return Err(Status::internal("negative engine IPC row count")); } + let next_offset = offset + .checked_add(row_count) + .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; let batch = proto::execute_plan_response::ArrowBatch { row_count, data: data.into(), @@ -558,9 +575,7 @@ async fn send_batch( ); if result.encoded_len() <= MAX_RESULT_BYTES { check_output_schema(batch_schema, output_schema)?; - *offset = offset - .checked_add(row_count) - .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; + *offset = next_offset; return Ok(sender.send(Ok(result)).await.is_ok()); } drop(result); @@ -586,7 +601,7 @@ async fn send_batch( } // Decoded columns can share the full IPC allocation. Use its // serialized size instead of counting that allocation per column. - let rows = rows_for_bytes(batch.num_rows(), ipc_bytes); + let rows = rows_for_bytes(batch.num_rows(), ipc_bytes, MAX_RESULT_BYTES / 2); Ok((batch, rows)) }) .await @@ -616,6 +631,35 @@ async fn send_arrow_batch( output_schema: &mut Option, guard: &Arc, ) -> Result { + // Validate the complete engine batch before slicing or IPC encoding. Even + // buffer-free arrays can represent more rows than the signed protocol. + checked_arrow_row_count(batch.num_rows(), *offset)?; + // Schema metadata and wide schemas repeat in every IPC response. Measure + // their fixed cost once, using fresh empty arrays so nested view buffers + // retained by a zero-row slice cannot be materialized here. + let empty = EncodingBatch { + batch: RecordBatch::new_empty(batch.schema()), + _guard: guard.clone(), + }; + let (empty, data) = tokio::task::spawn_blocking(move || empty.encode()) + .await + .map_err(|_| Status::internal("Arrow result encoding task failed"))??; + let fixed_bytes = response( + id, + &guard.session, + operation, + Some(proto::execute_plan_response::ResponseType::ArrowBatch( + proto::execute_plan_response::ArrowBatch { + row_count: 0, + data: data.into(), + start_offset: Some(*offset), + ..Default::default() + }, + )), + ) + .encoded_len(); + drop(empty); + let payload_bytes = (MAX_RESULT_BYTES.saturating_sub(fixed_bytes) / 2).max(1); let mut pending = vec![(batch, rows_per_batch)]; while let Some((batch, rows_per_batch)) = pending.pop() { let mut rows_per_batch = rows_per_batch.max(1).min(batch.num_rows().max(1)); @@ -627,7 +671,7 @@ async fn send_arrow_batch( prefix = batch.slice(0, rows_per_batch); &prefix }; - let estimated = estimated_rows_per_batch(candidate); + let estimated = estimated_rows_for_payload(candidate, payload_bytes); if estimated >= candidate.num_rows() { break; } @@ -649,6 +693,7 @@ async fn send_arrow_batch( } else { batch }; + let row_count = checked_arrow_row_count(batch.num_rows(), *offset)?; // Arrow materialization/IPC encoding is CPU work. Retain operation // ownership if cancellation drops the JoinHandle, so ReleaseAll cannot // claim drain while a result batch is still being encoded. @@ -661,7 +706,7 @@ async fn send_arrow_batch( .map_err(|_| Status::internal("Arrow result encoding task failed"))??; let batch = &encoded.batch; let payload = proto::execute_plan_response::ArrowBatch { - row_count: batch.num_rows() as i64, + row_count, data: data.into(), start_offset: Some(*offset), ..Default::default() @@ -699,12 +744,16 @@ async fn send_arrow_batch( // Adapt this piece's own remainder, without changing an outer // sibling when the piece finishes. The row-growth limit is a // sampling heuristic, not a bound on bytes for skewed rows. - *next_rows = rows_for_bytes(batch.num_rows(), result_bytes) - .min(batch.num_rows().saturating_mul(4).max(1)); + *next_rows = rows_for_bytes( + batch.num_rows(), + result_bytes.saturating_sub(fixed_bytes), + payload_bytes, + ) + .min(batch.num_rows().saturating_mul(4).max(1)); } } *offset = offset - .checked_add(batch.num_rows() as i64) + .checked_add(row_count) .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; if sender.send(Ok(result)).await.is_err() { return Ok(false); @@ -1186,6 +1235,163 @@ mod tests { ); } + #[cfg(target_pointer_width = "64")] + #[tokio::test] + async fn unrepresentable_arrow_row_counts_fail_before_encoding() { + use arrow::array::NullArray; + use arrow::datatypes::{DataType, Field, Schema}; + use arrow::record_batch::RecordBatchOptions; + use std::sync::atomic::Ordering; + + let rows = i64::MAX as usize + 1; + let empty = RecordBatch::try_new_with_options( + Arc::new(Schema::empty()), + vec![], + &RecordBatchOptions::new().with_row_count(Some(rows)), + ) + .unwrap(); + let nulls = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("nulls", DataType::Null, true)])), + vec![Arc::new(NullArray::new(rows))], + ) + .unwrap(); + for batch in [empty, nulls] { + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("row_count").unwrap(); + let (sender, mut receiver) = mpsc::channel(2); + let mut offset = 7; + let mut output_schema = None; + for direct in [false, true] { + let error = if direct { + send_arrow_batch( + &sender, + "session", + "row_count", + batch.clone(), + rows, + &mut offset, + &mut output_schema, + &guard, + ) + .await + .unwrap_err() + } else { + send_batch( + &sender, + "session", + "row_count", + batch.clone().into(), + batch.schema().as_ref(), + &mut offset, + &mut output_schema, + &guard, + ) + .await + .unwrap_err() + }; + assert_eq!(error.code(), tonic::Code::ResourceExhausted); + assert_eq!(error.message(), "result batch row count exceeds i64"); + assert_eq!(offset, 7); + assert!(output_schema.is_none()); + assert_eq!(guard.running.encoding_attempts.load(Ordering::SeqCst), 0); + assert!(receiver.try_recv().is_err()); + } + } + } + + #[tokio::test] + async fn overflowing_engine_batches_do_not_partially_emit() { + use arrow::array::LargeStringArray; + use arrow::datatypes::{DataType, Field, Schema}; + use std::sync::atomic::Ordering; + + let schema = Arc::new(Schema::new(vec![Field::new( + "text", + DataType::LargeUtf8, + false, + )])); + let small = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(LargeStringArray::from(vec![""]))], + ) + .unwrap(); + let large = "z".repeat(2 * 1024 * 1024); + let oversized = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(LargeStringArray::from(vec![large.as_str(); 2]))], + ) + .unwrap(); + for ipc in [false, true] { + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("row_offset").unwrap(); + let (sender, mut receiver) = mpsc::channel(4); + let mut offset = i64::MAX - 2; + let mut output_schema = None; + send_batch( + &sender, + "session", + "row_offset", + small.clone().into(), + schema.as_ref(), + &mut offset, + &mut output_schema, + &guard, + ) + .await + .unwrap(); + let before = guard.running.encoding_attempts.load(Ordering::SeqCst); + let batch = if ipc { + let data = encode_batch(&oversized).unwrap(); + assert!(data.len() > MAX_RESULT_BYTES); + ResultBatch::Ipc { data, row_count: 2 } + } else { + oversized.clone().into() + }; + let error = send_batch( + &sender, + "session", + "row_offset", + batch, + schema.as_ref(), + &mut offset, + &mut output_schema, + &guard, + ) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::ResourceExhausted); + assert_eq!(error.message(), "result row offset exceeds i64"); + assert_eq!(offset, i64::MAX - 1); + assert_eq!( + guard.running.encoding_attempts.load(Ordering::SeqCst), + before + ); + let previous = receiver.try_recv().unwrap().unwrap(); + let Some(proto::execute_plan_response::ResponseType::ArrowBatch(previous)) = + previous.response_type + else { + panic!("expected the preceding valid Arrow response"); + }; + assert_eq!(previous.start_offset, Some(i64::MAX - 2)); + assert_eq!(previous.row_count, 1); + assert!(receiver.try_recv().is_err()); + // A one-row batch still fits exactly at the signed boundary. + send_batch( + &sender, + "session", + "row_offset", + small.clone().into(), + schema.as_ref(), + &mut offset, + &mut output_schema, + &guard, + ) + .await + .unwrap(); + assert_eq!(offset, i64::MAX); + } + } + #[tokio::test] async fn successful_encoding_grows_a_conservative_row_budget() { use arrow::array::Int64Array; @@ -1234,6 +1440,108 @@ mod tests { ); } + #[tokio::test] + async fn fixed_schema_overhead_does_not_collapse_the_payload_budget() { + use arrow::array::{Int64Array, LargeStringArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use std::collections::HashMap; + use std::sync::atomic::Ordering; + + for (metadata_bytes, expected_rows, max_responses) in [ + ((1.49 * 1024.0 * 1024.0) as usize, 1500usize, 18usize), + ((2.6 * 1024.0 * 1024.0) as usize, 300usize, 14usize), + ] { + let schema = Arc::new(Schema::new_with_metadata( + vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::LargeUtf8, false), + ], + HashMap::from([("padding".into(), "x".repeat(metadata_bytes))]), + )); + let text = "z".repeat(8192); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int64Array::from_iter_values(0..expected_rows as i64)), + Arc::new(LargeStringArray::from_iter_values(std::iter::repeat_n( + text.as_str(), + expected_rows, + ))), + ], + ) + .unwrap(); + let budget = estimated_rows_per_batch(&batch); + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("fixed_overhead").unwrap(); + // Bound retained metadata even if the old one-row decay returns; + // the timeout then fails without buffering gigabytes of replies. + let (sender, mut receiver) = mpsc::channel(max_responses + 1); + let mut offset = 0; + let mut output_schema = None; + assert!(tokio::time::timeout( + std::time::Duration::from_secs(15), + send_arrow_batch( + &sender, + "session", + "fixed_overhead", + batch, + budget, + &mut offset, + &mut output_schema, + &guard + ) + ) + .await + .expect("fixed overhead exhausted the bounded response queue") + .unwrap()); + let (mut responses, mut rows) = (0usize, 0usize); + while let Ok(result) = receiver.try_recv() { + let result = result.unwrap(); + assert!(result.encoded_len() <= MAX_RESULT_BYTES); + let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + result.response_type + else { + panic!("expected Arrow response"); + }; + assert_eq!(batch.start_offset, Some(rows as i64)); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + assert_eq!(reader.schema().metadata()["padding"].len(), metadata_bytes); + let decoded = reader.next().unwrap().unwrap(); + assert_eq!(decoded.num_rows() as i64, batch.row_count); + assert!(reader.next().is_none()); + let ids = decoded + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let strings = decoded + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + for index in 0..decoded.num_rows() { + assert_eq!(ids.value(index), rows as i64); + assert_eq!(strings.value(index), text); + rows += 1; + } + responses += 1; + } + assert_eq!(rows, expected_rows); + assert_eq!(offset, expected_rows as i64); + assert!( + responses <= max_responses, + "{metadata_bytes} fixed bytes caused {responses} responses" + ); + // One encode measures the fresh empty schema. Only a small + // constant of exact-envelope retries is acceptable after that. + let attempts = guard.running.encoding_attempts.load(Ordering::SeqCst); + assert!( + attempts <= responses + 2, + "{attempts} encodes for {responses} responses" + ); + } + } + #[tokio::test] async fn skewed_output_checks_prefixes_and_recovers_light_suffixes() { use arrow::array::{Int64Array, LargeStringArray}; @@ -1346,6 +1654,11 @@ mod tests { responses < 100, "large-head output fragmented into {responses} responses" ); + } else { + assert!( + responses < 80, + "large-tail output fragmented into {responses} responses" + ); } } } diff --git a/crates/spark-connect-server/tests/review_regressions.rs b/crates/spark-connect-server/tests/review_regressions.rs index 992fc8a6..43f46c99 100644 --- a/crates/spark-connect-server/tests/review_regressions.rs +++ b/crates/spark-connect-server/tests/review_regressions.rs @@ -450,6 +450,70 @@ async fn view_slices_only_budget_the_current_values() { } } +#[cfg(target_pointer_width = "64")] +#[tokio::test] +async fn unrepresentable_engine_batches_preserve_preceding_results_and_fail_cleanly() { + use arrow::array::NullArray; + use arrow::record_batch::RecordBatchOptions; + + let rows = i64::MAX as usize + 1; + for null_column in [false, true] { + let oversized = if null_column { + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("nulls", DataType::Null, true)])), + vec![Arc::new(NullArray::new(rows))], + ) + .unwrap() + } else { + RecordBatch::try_new_with_options( + Arc::new(Schema::empty()), + vec![], + &RecordBatchOptions::new().with_row_count(Some(rows)), + ) + .unwrap() + }; + let state = State::new(vec![oversized.slice(0, 3), oversized], None); + let mut server = Server::start_with_engine(Arc::new(Fixture(state))).await; + let mut stream = server + .client + .execute_plan(execute( + &Uuid::new_v4().to_string(), + &Uuid::new_v4().to_string(), + )) + .await + .unwrap() + .into_inner(); + let (status, counts) = tokio::time::timeout(Duration::from_secs(15), async { + let mut counts = Vec::new(); + loop { + match stream.message().await { + Ok(Some(message)) => match message.response_type { + Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) => { + assert_eq!(batch.start_offset, Some(0)); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + assert_eq!(reader.next().unwrap().unwrap().num_rows(), 3); + assert!(reader.next().is_none()); + counts.push(batch.row_count); + } + Some(proto::execute_plan_response::ResponseType::ResultComplete(_)) => { + panic!("unrepresentable output must not complete successfully"); + } + _ => {} + }, + Err(status) => break (status, counts), + Ok(None) => panic!("expected row-count overflow status"), + } + } + }) + .await + .unwrap(); + assert_eq!(status.code(), Code::ResourceExhausted); + assert_eq!(status.message(), "result batch row count exceeds i64"); + assert_eq!(counts, vec![3]); + server.stop().await; + } +} + #[tokio::test] async fn noncanonical_ids_are_rejected_without_creating_or_releasing_state() { let state = State::new(vec![integers(1)], None); From dd6069bdbd36bd795cae68d2c0f6c3597c6b8afc Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 17:02:18 -0800 Subject: [PATCH 07/18] Preserve fitting Arrow budgets across padded buffers --- crates/spark-connect-server/src/service.rs | 120 +++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index beb0cf81..ad9e09c3 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -732,6 +732,13 @@ async fn send_arrow_batch( / (result_bytes as u128 * 10)) as usize) .clamp(1, batch.num_rows() - 1); drop(result); + if has_remainder { + if let Some((_, next_rows)) = pending.last_mut() { + // Teach this piece's own remainder about IPC padding or + // other fixed costs absent from the logical estimate. + *next_rows = (*next_rows).min(rows_per_batch); + } + } pending.push((batch.clone(), rows_per_batch)); continue; } @@ -749,6 +756,9 @@ async fn send_arrow_batch( result_bytes.saturating_sub(fixed_bytes), payload_bytes, ) + // A successful size is already known to fit. Fixed buffer + // padding must not make that row budget decay on each send. + .max(batch.num_rows()) .min(batch.num_rows().saturating_mul(4).max(1)); } } @@ -1542,6 +1552,116 @@ mod tests { } } + #[tokio::test] + async fn wide_buffer_padding_does_not_decay_or_repeat_failed_budgets() { + use arrow::array::{Array, ArrayRef, Int64Array, Int8Array}; + use arrow::datatypes::{DataType, Field, Schema}; + use std::sync::atomic::Ordering; + + // Int64 values cross a 64-byte IPC padding boundary at 8 -> 9 rows; + // Int8 values occupy the same padded buffer for up to 64 rows. Nullable + // columns add a separately padded validity buffer absent from an empty + // schema measurement. Share inputs to keep this regression inexpensive. + for (columns, expected_rows, narrow, nullable, max_responses) in [ + (17_293, 64usize, false, false, 18usize), + (17_293, 200usize, true, false, 10usize), + (12_000, 64usize, false, true, 18usize), + ] { + let values: ArrayRef = if narrow { + Arc::new(Int8Array::from_iter_values( + (0..expected_rows).map(|row| (row % 127) as i8), + )) + } else if nullable { + Arc::new(Int64Array::from_iter( + (0..expected_rows).map(|row| (row % 3 != 0).then_some(row as i64)), + )) + } else { + Arc::new(Int64Array::from_iter_values(0..expected_rows as i64)) + }; + let fields = (0..columns) + .map(|column| { + Field::new( + format!("c{column:05}"), + if narrow { + DataType::Int8 + } else { + DataType::Int64 + }, + nullable, + ) + }) + .collect::>(); + let batch = + RecordBatch::try_new(Arc::new(Schema::new(fields)), vec![values; columns]).unwrap(); + let budget = estimated_rows_per_batch(&batch); + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("wide_padding").unwrap(); + let (sender, mut receiver) = mpsc::channel(max_responses + 1); + let mut offset = 0; + let mut output_schema = None; + assert!(tokio::time::timeout( + std::time::Duration::from_secs(30), + send_arrow_batch( + &sender, + "session", + "wide_padding", + batch, + budget, + &mut offset, + &mut output_schema, + &guard + ) + ) + .await + .expect("wide padding exhausted the bounded response queue") + .unwrap()); + let (mut responses, mut rows) = (0usize, 0usize); + while let Ok(result) = receiver.try_recv() { + let result = result.unwrap(); + assert!(result.encoded_len() <= MAX_RESULT_BYTES); + let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + result.response_type + else { + panic!("expected Arrow response"); + }; + assert_eq!(batch.start_offset, Some(rows as i64)); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + let decoded = reader.next().unwrap().unwrap(); + assert_eq!(decoded.num_columns(), columns); + assert_eq!(decoded.num_rows() as i64, batch.row_count); + assert!(reader.next().is_none()); + for column in decoded.columns() { + if narrow { + let values = column.as_any().downcast_ref::().unwrap(); + for index in 0..decoded.num_rows() { + assert_eq!(values.value(index), ((rows + index) % 127) as i8); + } + } else { + let values = column.as_any().downcast_ref::().unwrap(); + for index in 0..decoded.num_rows() { + let row = rows + index; + assert_eq!(values.is_null(index), nullable && row % 3 == 0); + if !values.is_null(index) { + assert_eq!(values.value(index), row as i64); + } + } + } + } + rows += decoded.num_rows(); + responses += 1; + } + assert_eq!(rows, expected_rows); + assert_eq!(offset, expected_rows as i64); + assert!(responses <= max_responses, + "{columns} columns (narrow={narrow}, nullable={nullable}) caused {responses} responses"); + // One measurement plus a small number of exact retries, including + // a nullable slice that crosses the 8 -> 9 row padding boundary. + let attempts = guard.running.encoding_attempts.load(Ordering::SeqCst); + assert!(attempts <= responses + 3, + "{columns} columns (narrow={narrow}, nullable={nullable}): {attempts} encodes for {responses} responses"); + } + } + #[tokio::test] async fn skewed_output_checks_prefixes_and_recovers_light_suffixes() { use arrow::array::{Int64Array, LargeStringArray}; From 8ba53a2d08300cb22e0a6addde037c61057c6d06 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 17:06:30 -0800 Subject: [PATCH 08/18] Avoid redundant logical sizing before Arrow preflight --- crates/spark-connect-server/src/engine/mod.rs | 4 ++-- crates/spark-connect-server/src/service.rs | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/spark-connect-server/src/engine/mod.rs b/crates/spark-connect-server/src/engine/mod.rs index 64f6a8a1..e2285e67 100644 --- a/crates/spark-connect-server/src/engine/mod.rs +++ b/crates/spark-connect-server/src/engine/mod.rs @@ -97,8 +97,8 @@ pub enum PreparedOperation { #[tonic::async_trait] pub trait PreparedQuery: Send { /// The declared Spark schema. This synchronous accessor must not panic. - /// Batches must have the same field names and - /// Spark-visible types; equivalent Arrow representations are accepted. + /// Batches must have the same field names and Spark-visible types; + /// equivalent Arrow representations are accepted. /// Batch nullability and metadata may differ from this declaration, but /// emitted IPC batches in one result must share physical types, field names /// and nullability. Metadata may differ between batches. Encoding materializes diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index ad9e09c3..2e411b09 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -490,6 +490,7 @@ fn estimated_array_bytes(array: &dyn arrow::array::Array) -> usize { bytes.saturating_add(array.nulls().map_or(0, |_| array.len().div_ceil(8))) } +#[cfg(test)] fn estimated_rows_per_batch(batch: &RecordBatch) -> usize { estimated_rows_for_payload(batch, MAX_RESULT_BYTES / 2) } @@ -549,7 +550,7 @@ async fn send_batch( let (batch, rows_per_batch) = match batch { ResultBatch::Arrow(batch) => { checked_arrow_row_count(batch.num_rows(), *offset)?; - let rows = estimated_rows_per_batch(&batch); + let rows = batch.num_rows().max(1); (batch, rows) } ResultBatch::Ipc { data, row_count } => { From 4a1219951693ec407c0efa750a3d85bd66629939 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 17:17:39 -0800 Subject: [PATCH 09/18] Align Arrow encoding regressions with production sizing --- crates/spark-connect-server/src/service.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index 2e411b09..0efb4134 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -549,7 +549,6 @@ async fn send_batch( ) -> Result { let (batch, rows_per_batch) = match batch { ResultBatch::Arrow(batch) => { - checked_arrow_row_count(batch.num_rows(), *offset)?; let rows = batch.num_rows().max(1); (batch, rows) } @@ -600,8 +599,8 @@ async fn send_batch( "engine IPC batch count differs from its declaration", )); } - // Decoded columns can share the full IPC allocation. Use its - // serialized size instead of counting that allocation per column. + // Seed oversized IPC splitting from its serialized size; the + // payload-aware preflight refines this before encoding. let rows = rows_for_bytes(batch.num_rows(), ipc_bytes, MAX_RESULT_BYTES / 2); Ok((batch, rows)) }) @@ -1481,7 +1480,7 @@ mod tests { ], ) .unwrap(); - let budget = estimated_rows_per_batch(&batch); + let budget = batch.num_rows().max(1); let session = Arc::new(Session::new(Arc::new(RegistrySession))); let (guard, _) = session.operation("fixed_overhead").unwrap(); // Bound retained metadata even if the old one-row decay returns; @@ -1594,7 +1593,7 @@ mod tests { .collect::>(); let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), vec![values; columns]).unwrap(); - let budget = estimated_rows_per_batch(&batch); + let budget = batch.num_rows().max(1); let session = Arc::new(Session::new(Arc::new(RegistrySession))); let (guard, _) = session.operation("wide_padding").unwrap(); let (sender, mut receiver) = mpsc::channel(max_responses + 1); @@ -1691,7 +1690,7 @@ mod tests { ], ) .unwrap(); - let budget = estimated_rows_per_batch(&batch); + let budget = batch.num_rows().max(1); let session = Arc::new(Session::new(Arc::new(RegistrySession))); let (guard, _) = session.operation("skewed").unwrap(); // Even an accidentally permanent one-row budget must not block From f1898dcb418353b2c233c02d8acc19e3127b1339 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 20:25:58 -0800 Subject: [PATCH 10/18] Share recursive Arrow schema field transformations --- crates/spark-connect-server/src/schema.rs | 39 +++++++------ crates/spark-connect-server/src/service.rs | 66 ++++++---------------- 2 files changed, 40 insertions(+), 65 deletions(-) diff --git a/crates/spark-connect-server/src/schema.rs b/crates/spark-connect-server/src/schema.rs index a6a37a49..76ab1b15 100644 --- a/crates/spark-connect-server/src/schema.rs +++ b/crates/spark-connect-server/src/schema.rs @@ -19,7 +19,7 @@ use std::sync::Arc; -use arrow::datatypes::{DataType, Field, FieldRef, Schema, TimeUnit}; +use arrow::datatypes::{DataType, Field, FieldRef, Fields, Schema, TimeUnit}; use prost::Message; use spark_connect_proto as proto; use tonic::Status; @@ -41,24 +41,31 @@ pub(crate) fn validate_schema_size(schema: &proto::DataType) -> Result<(), Statu Ok(()) } -/// Compare Spark-visible fields and types, allowing equivalent Arrow physical -/// representations and differences in nullability or field/schema metadata. -pub(crate) fn comparable_schema(schema: &Schema) -> Result { - fn field(input: &FieldRef) -> FieldRef { - Arc::new(Field::new(input.name(), data_type(input.data_type()), true)) - } - - fn data_type(input: &DataType) -> DataType { - match input { - DataType::List(child) => DataType::List(field(child)), - DataType::LargeList(child) => DataType::LargeList(field(child)), - DataType::Struct(fields) => DataType::Struct(fields.iter().map(field).collect()), - DataType::Map(entries, sorted) => DataType::Map(field(entries), *sorted), +/// Transform fields after recursively transforming their supported child fields. +pub(crate) fn map_fields( + fields: &Fields, + transform: &impl Fn(&FieldRef, DataType) -> FieldRef, +) -> Fields { + fn field(input: &FieldRef, transform: &impl Fn(&FieldRef, DataType) -> FieldRef) -> FieldRef { + let data_type = match input.data_type() { + DataType::List(child) => DataType::List(field(child, transform)), + DataType::LargeList(child) => DataType::LargeList(field(child, transform)), + DataType::Struct(children) => DataType::Struct(map_fields(children, transform)), + DataType::Map(entries, sorted) => DataType::Map(field(entries, transform), *sorted), other => other.clone(), - } + }; + transform(input, data_type) } - let fields: Vec<_> = schema.fields().iter().map(field).collect(); + fields.iter().map(|input| field(input, transform)).collect() +} + +/// Compare Spark-visible fields and types, allowing equivalent Arrow physical +/// representations and differences in nullability or field/schema metadata. +pub(crate) fn comparable_schema(schema: &Schema) -> Result { + let fields = map_fields(schema.fields(), &|field, data_type| { + Arc::new(Field::new(field.name(), data_type, true)) + }); spark_schema(&Schema::new(fields)) } diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index 0efb4134..11baf810 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -312,42 +312,26 @@ fn complete(id: &str, session: &Session, operation: &str) -> proto::ExecutePlanR } fn encode_batch(batch: &RecordBatch) -> Result, Status> { - use arrow::datatypes::{DataType, FieldRef, Schema}; + use arrow::datatypes::DataType; use arrow::record_batch::RecordBatchOptions; - fn materialized_field(field: &FieldRef) -> FieldRef { - let data_type = materialized_type(field.data_type()); - if data_type == *field.data_type() { - field.clone() - } else { - Arc::new(field.as_ref().clone().with_data_type(data_type)) - } - } - - fn materialized_type(data_type: &DataType) -> DataType { - match data_type { - DataType::Utf8View => DataType::LargeUtf8, - DataType::BinaryView => DataType::LargeBinary, - DataType::List(field) => DataType::List(materialized_field(field)), - DataType::LargeList(field) => DataType::LargeList(materialized_field(field)), - DataType::Struct(fields) => { - DataType::Struct(fields.iter().map(materialized_field).collect()) - } - DataType::Map(field, sorted) => DataType::Map(materialized_field(field), *sorted), - other => other.clone(), - } - } - // IPC writes all backing buffers of view arrays even after a row slice. Materialize // their bytes, including nested children, so splitting measures only retained rows. // Use 64-bit offsets so materializing a batch cannot overflow 32-bit byte offsets // before the result-size check has a chance to split it. let source_schema = batch.schema(); - let fields: Vec<_> = source_schema - .fields() - .iter() - .map(materialized_field) - .collect(); + let fields = schema::map_fields(source_schema.fields(), &|field, data_type| { + let data_type = match data_type { + DataType::Utf8View => DataType::LargeUtf8, + DataType::BinaryView => DataType::LargeBinary, + other => other, + }; + if data_type == *field.data_type() { + field.clone() + } else { + Arc::new(field.as_ref().clone().with_data_type(data_type)) + } + }); let changed = fields .iter() .zip(source_schema.fields()) @@ -399,27 +383,11 @@ fn encode_batch(batch: &RecordBatch) -> Result, Status> { // PySpark assembles one table from all IPC batches. Its physical types and // nullability must agree across batches, while field/schema metadata may differ. fn check_output_schema(schema: &Schema, expected: &mut Option) -> Result<(), Status> { - use arrow::datatypes::{DataType, Field, FieldRef}; - - fn field(input: &FieldRef) -> FieldRef { - Arc::new(Field::new( - input.name(), - data_type(input.data_type()), - input.is_nullable(), - )) - } - - fn data_type(input: &DataType) -> DataType { - match input { - DataType::List(child) => DataType::List(field(child)), - DataType::LargeList(child) => DataType::LargeList(field(child)), - DataType::Struct(fields) => DataType::Struct(fields.iter().map(field).collect()), - DataType::Map(entries, sorted) => DataType::Map(field(entries), *sorted), - other => other.clone(), - } - } + use arrow::datatypes::Field; - let actual = Schema::new(schema.fields().iter().map(field).collect::>()); + let actual = Schema::new(schema::map_fields(schema.fields(), &|field, data_type| { + Arc::new(Field::new(field.name(), data_type, field.is_nullable())) + })); match expected { Some(expected) if *expected != actual => Err(Status::internal( "Arrow output schema changed within one result", From 94ab6108d3074bcdfae66359f56988d2ab43dd7a Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 20:25:59 -0800 Subject: [PATCH 11/18] Generate optional server bindings from the protocol crate --- .github/workflows/server-frontend.yml | 5 + Cargo.lock | 3 - Cargo.toml | 1 - crates/spark-connect-proto/Cargo.toml | 3 + crates/spark-connect-proto/build.rs | 4 +- crates/spark-connect-proto/src/lib.rs | 2 + crates/spark-connect-server/Cargo.toml | 9 +- crates/spark-connect-server/README.md | 3 +- crates/spark-connect-server/build.rs | 186 ------------------------- crates/spark-connect-server/src/lib.rs | 9 +- 10 files changed, 18 insertions(+), 207 deletions(-) delete mode 100644 crates/spark-connect-server/build.rs diff --git a/.github/workflows/server-frontend.yml b/.github/workflows/server-frontend.yml index e088800f..6b77267f 100644 --- a/.github/workflows/server-frontend.yml +++ b/.github/workflows/server-frontend.yml @@ -86,3 +86,8 @@ jobs: - name: Test frontend # Protocol and injected-engine tests use local fixtures without an execution backend. run: cargo test --locked -p spark-connect-server + + - name: Check protocol feature combinations + run: | + cargo check --locked -p apache-spark-connect-proto --no-default-features + cargo check --locked -p apache-spark-connect-proto --no-default-features --features server diff --git a/Cargo.lock b/Cargo.lock index 7e32c1e5..41a9b1b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4057,12 +4057,9 @@ dependencies = [ "futures", "h2", "prost", - "prost-types", "tokio", "tokio-stream", "tonic", - "tonic-build", - "tonic-prost", "uuid", ] diff --git a/Cargo.toml b/Cargo.toml index 9ba59607..189b8bab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,7 +90,6 @@ should_implement_trait = "allow" tonic = { version = "0.14", default-features = false, features = ["channel", "codegen"] } tonic-prost = "0.14" tonic-prost-build = "0.14" -tonic-build = "0.14" h2 = "0.4" prost = "0.14" prost-types = "0.14" diff --git a/crates/spark-connect-proto/Cargo.toml b/crates/spark-connect-proto/Cargo.toml index efc4c833..330fdfa0 100644 --- a/crates/spark-connect-proto/Cargo.toml +++ b/crates/spark-connect-proto/Cargo.toml @@ -15,6 +15,9 @@ workspace = true [lib] name = "spark_connect_proto" +[features] +server = [] + [dependencies] tonic.workspace = true tonic-prost.workspace = true diff --git a/crates/spark-connect-proto/build.rs b/crates/spark-connect-proto/build.rs index 048ae15f..dd85bafe 100644 --- a/crates/spark-connect-proto/build.rs +++ b/crates/spark-connect-proto/build.rs @@ -35,8 +35,10 @@ fn main() -> Result<(), Box> { let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").ok_or("OUT_DIR is not set")?); let full_set = out_dir.join("spark_connect_descriptor_set_full.bin"); tonic_prost_build::configure() - .build_server(false) + .build_server(cfg!(feature = "server")) .build_client(true) + // async_trait repeats Future's must_use on generated server methods. + .trait_attribute("SparkConnectService", "#[allow(clippy::double_must_use)]") .bytes(".") .file_descriptor_set_path(&full_set) .compile_protos(&protos, &[proto_root])?; diff --git a/crates/spark-connect-proto/src/lib.rs b/crates/spark-connect-proto/src/lib.rs index 35689ba4..58eb65a9 100644 --- a/crates/spark-connect-proto/src/lib.rs +++ b/crates/spark-connect-proto/src/lib.rs @@ -3,6 +3,8 @@ //! All messages live in the `spark.connect` protobuf package and are re-exported //! here at the crate root. The generated gRPC client is //! [`spark_connect_service_client::SparkConnectServiceClient`]. +//! Enable the optional `server` feature for `spark_connect_service_server` +//! bindings. Embeddings configure tonic's transport server features separately. //! //! Baseline protocol version: see `proto/PROTO_VERSION.txt` / `proto/SPARK_SHA.txt`. diff --git a/crates/spark-connect-server/Cargo.toml b/crates/spark-connect-server/Cargo.toml index 5fe56bcc..96adcd4a 100644 --- a/crates/spark-connect-server/Cargo.toml +++ b/crates/spark-connect-server/Cargo.toml @@ -28,21 +28,14 @@ description = "Experimental Spark Connect server frontend with an injectable exe workspace = true [dependencies] -spark-connect-proto = { package = "apache-spark-connect-proto", path = "../spark-connect-proto" } +spark-connect-proto = { package = "apache-spark-connect-proto", path = "../spark-connect-proto", features = ["server"] } arrow.workspace = true futures.workspace = true prost.workspace = true tokio = { workspace = true, features = ["net"] } tokio-stream.workspace = true tonic = { workspace = true, features = ["server", "router"] } -tonic-prost.workspace = true uuid.workspace = true -[build-dependencies] -tonic-build.workspace = true -spark-connect-proto = { package = "apache-spark-connect-proto", path = "../spark-connect-proto" } -prost.workspace = true -prost-types.workspace = true - [dev-dependencies] h2.workspace = true diff --git a/crates/spark-connect-server/README.md b/crates/spark-connect-server/README.md index 564b6531..620a7df6 100644 --- a/crates/spark-connect-server/README.md +++ b/crates/spark-connect-server/README.md @@ -20,7 +20,8 @@ # Experimental Spark Connect server frontend This unpublished library serves a subset of the Spark Connect protocol using the -workspace's `apache-spark-connect-proto` messages. The caller supplies an +workspace's `apache-spark-connect-proto` messages and its optional `server` +bindings. The caller supplies an `Engine` implementation and a bound Tokio TCP listener to `serve_with_engine`. No execution engine or standalone server executable is included. diff --git a/crates/spark-connect-server/build.rs b/crates/spark-connect-server/build.rs deleted file mode 100644 index 5041d662..00000000 --- a/crates/spark-connect-server/build.rs +++ /dev/null @@ -1,186 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use prost::Message; -use tonic_build::manual::{Builder, Method, Service}; - -fn main() { - // The official protocol crate already generates messages and client bindings. - // Generate only the standard service routes, using those exact message types. - // Check this table against the generated protocol descriptor before emitting - // routes: streaming signatures are part of wire compatibility. - let methods = [ - ( - "execute_plan", - "ExecutePlan", - "ExecutePlanRequest", - "ExecutePlanResponse", - false, - true, - ), - ( - "analyze_plan", - "AnalyzePlan", - "AnalyzePlanRequest", - "AnalyzePlanResponse", - false, - false, - ), - ( - "config", - "Config", - "ConfigRequest", - "ConfigResponse", - false, - false, - ), - ( - "add_artifacts", - "AddArtifacts", - "AddArtifactsRequest", - "AddArtifactsResponse", - true, - false, - ), - ( - "artifact_status", - "ArtifactStatus", - "ArtifactStatusesRequest", - "ArtifactStatusesResponse", - false, - false, - ), - ( - "interrupt", - "Interrupt", - "InterruptRequest", - "InterruptResponse", - false, - false, - ), - ( - "reattach_execute", - "ReattachExecute", - "ReattachExecuteRequest", - "ExecutePlanResponse", - false, - true, - ), - ( - "release_execute", - "ReleaseExecute", - "ReleaseExecuteRequest", - "ReleaseExecuteResponse", - false, - false, - ), - ( - "release_session", - "ReleaseSession", - "ReleaseSessionRequest", - "ReleaseSessionResponse", - false, - false, - ), - ( - "fetch_error_details", - "FetchErrorDetails", - "FetchErrorDetailsRequest", - "FetchErrorDetailsResponse", - false, - false, - ), - ( - "clone_session", - "CloneSession", - "CloneSessionRequest", - "CloneSessionResponse", - false, - false, - ), - ( - "get_status", - "GetStatus", - "GetStatusRequest", - "GetStatusResponse", - false, - false, - ), - ]; - let descriptors = - prost_types::FileDescriptorSet::decode(spark_connect_proto::FILE_DESCRIPTOR_SET) - .expect("valid Spark Connect descriptor set"); - let descriptor = descriptors - .file - .iter() - .filter(|file| file.package.as_deref() == Some("spark.connect")) - .flat_map(|file| &file.service) - .find(|service| service.name.as_deref() == Some("SparkConnectService")) - .expect("SparkConnectService descriptor"); - assert_eq!(methods.len(), descriptor.method.len(), "RPC table drift"); - let mut routes = std::collections::HashSet::new(); - for &(_, route, input, output, client_stream, server_stream) in &methods { - assert!(routes.insert(route), "duplicate RPC route: {route}"); - let method = descriptor - .method - .iter() - .find(|method| method.name.as_deref() == Some(route)) - .unwrap_or_else(|| panic!("missing protocol RPC: {route}")); - assert_eq!( - method.input_type.as_deref(), - Some(format!(".spark.connect.{input}").as_str()), - "{route} request type drift" - ); - assert_eq!( - method.output_type.as_deref(), - Some(format!(".spark.connect.{output}").as_str()), - "{route} response type drift" - ); - assert_eq!( - method.client_streaming.unwrap_or(false), - client_stream, - "{route} client streaming drift" - ); - assert_eq!( - method.server_streaming.unwrap_or(false), - server_stream, - "{route} server streaming drift" - ); - } - let mut service = Service::builder() - .name("SparkConnectService") - .package("spark.connect"); - for (name, route, input, output, client_stream, server_stream) in methods { - let mut method = Method::builder() - .name(name) - .route_name(route) - .input_type(format!("spark_connect_proto::{input}")) - .output_type(format!("spark_connect_proto::{output}")) - .codec_path("tonic_prost::ProstCodec"); - if client_stream { - method = method.client_streaming(); - } - if server_stream { - method = method.server_streaming(); - } - service = service.method(method.build()); - } - Builder::new() - .build_client(false) - .compile(&[service.build()]); - println!("cargo:rerun-if-changed=build.rs"); -} diff --git a/crates/spark-connect-server/src/lib.rs b/crates/spark-connect-server/src/lib.rs index dece96a2..64db976b 100644 --- a/crates/spark-connect-server/src/lib.rs +++ b/crates/spark-connect-server/src/lib.rs @@ -20,14 +20,9 @@ mod schema; mod service; /// Generated server bindings for custom embedding. This experimental API follows -/// the RPC table in `build.rs` and may change with the protocol. -// tonic's generated async-trait methods duplicate Future's must_use annotation. -#[allow(clippy::double_must_use)] +/// the protocol schema and may change with the protocol. pub mod grpc { - include!(concat!( - env!("OUT_DIR"), - "/spark.connect.SparkConnectService.rs" - )); + pub use spark_connect_proto::spark_connect_service_server; } struct CancelOnDrop(std::sync::Arc); From 2edcde3ff3973e3d443346fb419fa73092706d22 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 20:26:00 -0800 Subject: [PATCH 12/18] Consolidate frontend fixtures and overlapping lifecycle tests --- .../tests/engine_injection.rs | 246 +++++++----- crates/spark-connect-server/tests/protocol.rs | 375 +----------------- .../tests/schema_consistency.rs | 91 +---- .../spark-connect-server/tests/support/mod.rs | 175 +++----- .../tests/support/requests.rs | 111 ++++++ 5 files changed, 333 insertions(+), 665 deletions(-) create mode 100644 crates/spark-connect-server/tests/support/requests.rs diff --git a/crates/spark-connect-server/tests/engine_injection.rs b/crates/spark-connect-server/tests/engine_injection.rs index 739a8124..a5797ed9 100644 --- a/crates/spark-connect-server/tests/engine_injection.rs +++ b/crates/spark-connect-server/tests/engine_injection.rs @@ -17,6 +17,8 @@ //! Real gRPC tests of the injected backend boundary. The fake is independent of Spark execution. +#[path = "support/requests.rs"] +mod requests; #[path = "support/server.rs"] mod server; @@ -37,6 +39,9 @@ use futures::{Stream, StreamExt}; use prost::Message; use proto::execute_plan_response::ResponseType; use proto::spark_connect_service_client::SparkConnectServiceClient; +use requests::{ + assert_cancelled, config_request, relation, release, release_session, request, user, +}; use server::Server; use spark_connect_proto as proto; use spark_connect_server::engine::{ @@ -514,36 +519,6 @@ async fn start_server(spy: Arc) -> Server { Server::start_with_engine(Arc::new(FakeEngine(spy))).await } -fn user(id: &str) -> Option { - Some(proto::UserContext { - user_id: id.into(), - ..Default::default() - }) -} - -fn relation(end: i64) -> proto::Relation { - proto::Relation { - rel_type: Some(proto::relation::RelType::Range(proto::Range { - end, - step: 1, - ..Default::default() - })), - ..Default::default() - } -} - -fn request(session: &str, end: i64) -> proto::ExecutePlanRequest { - proto::ExecutePlanRequest { - session_id: session.into(), - user_context: user("alice"), - operation_id: Some(Uuid::new_v4().to_string()), - plan: Some(proto::Plan { - op_type: Some(proto::plan::OpType::Root(relation(end))), - }), - ..Default::default() - } -} - fn command_request( session: &str, command_type: proto::command::CommandType, @@ -557,20 +532,6 @@ fn command_request( request } -fn config_request( - session: &str, - input: proto::config_request::operation::OpType, -) -> proto::ConfigRequest { - proto::ConfigRequest { - session_id: session.into(), - user_context: user("alice"), - operation: Some(proto::config_request::Operation { - op_type: Some(input), - }), - ..Default::default() - } -} - fn get_config(session: &str) -> proto::ConfigRequest { config_request( session, @@ -580,26 +541,6 @@ fn get_config(session: &str) -> proto::ConfigRequest { ) } -fn release(session: &str, operation: &str) -> proto::ReleaseExecuteRequest { - proto::ReleaseExecuteRequest { - session_id: session.into(), - user_context: user("alice"), - operation_id: operation.into(), - release: Some(proto::release_execute_request::Release::ReleaseAll( - proto::release_execute_request::ReleaseAll {}, - )), - ..Default::default() - } -} - -fn release_session(session: &str) -> proto::ReleaseSessionRequest { - proto::ReleaseSessionRequest { - session_id: session.into(), - user_context: user("alice"), - ..Default::default() - } -} - async fn wait_for_count(counter: &AtomicUsize, expected: usize) { tokio::time::timeout(Duration::from_secs(5), async { while counter.load(Ordering::SeqCst) < expected { @@ -639,26 +580,6 @@ fn values(responses: &[proto::ExecutePlanResponse]) -> Vec { values } -async fn assert_cancelled(stream: &mut tonic::Streaming) { - tokio::time::timeout(Duration::from_secs(5), async { - loop { - match stream.message().await { - Err(status) => { - assert_eq!(status.code(), Code::Cancelled); - break; - } - Ok(Some(response)) => assert!(!matches!( - response.response_type, - Some(ResponseType::ResultComplete(_)) - )), - Ok(None) => panic!("cancelled fake execution must report cancellation"), - } - } - }) - .await - .expect("cancelled fake execution must terminate"); -} - async fn start_live( server: &mut Server, request: proto::ExecutePlanRequest, @@ -862,23 +783,89 @@ async fn invalid_requests_do_not_open_backends_and_factory_failures_can_retry() assert_eq!( server .client - .execute_plan(request("invalid", 99)) + .execute_plan(request("not-a-uuid", 0)) .await .unwrap_err() .code(), Code::InvalidArgument ); - let mut invalid = request(&session, 99); - invalid.operation_id = Some("invalid".into()); + let mut invalid_operation = request(&session, 0); + invalid_operation.operation_id = Some("not-a-uuid".into()); assert_eq!( server .client - .execute_plan(invalid) + .execute_plan(invalid_operation) .await .unwrap_err() .code(), Code::InvalidArgument ); + for plan in [None, Some(proto::Plan { op_type: None })] { + let mut request = request(&session, 0); + request.plan = plan; + assert_eq!( + server + .client + .execute_plan(request) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + } + assert_eq!( + server + .client + .analyze_plan(proto::AnalyzePlanRequest { + session_id: session.clone(), + ..Default::default() + }) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + for operation in [ + None, + Some(proto::config_request::Operation { op_type: None }), + ] { + assert_eq!( + server + .client + .config(proto::ConfigRequest { + session_id: session.clone(), + operation, + ..Default::default() + }) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + } + // A stale observed ID cannot match a session that does not exist. Repeated + // requests must not consume backend session capacity while being rejected. + for _ in 0..3 { + let mut stale = request(&session, 0); + stale.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); + assert_eq!( + server.client.execute_plan(stale).await.unwrap_err().code(), + Code::FailedPrecondition + ); + } + let mut stale = config_request( + &session, + proto::config_request::operation::OpType::GetOption(proto::config_request::GetOption { + keys: vec!["missing".into()], + }), + ); + stale.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); + assert_eq!( + server.client.config(stale).await.unwrap_err().code(), + Code::FailedPrecondition + ); + // Identifier and envelope validation must precede backend session allocation. + assert_eq!(spy.factory_calls.lock().unwrap().len(), 0); assert_eq!( server .client @@ -888,7 +875,35 @@ async fn invalid_requests_do_not_open_backends_and_factory_failures_can_retry() .code(), Code::NotFound ); - assert!(spy.factory_calls.lock().unwrap().is_empty()); + assert_eq!( + server + .client + .release_execute(release("not-a-uuid", &session)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_execute(release(&session, "not-a-uuid")) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + let unknown = Uuid::new_v4().to_string(); + assert_eq!( + server + .client + .release_execute(release(&unknown, &session)) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + assert_eq!(spy.factory_calls.lock().unwrap().len(), 0); spy.fail_next.store(true, Ordering::SeqCst); assert_eq!( server @@ -924,6 +939,15 @@ async fn native_streams_drop_on_release_disconnect_session_release_and_shutdown( let live = request(&session, LIVE); let operation = live.operation_id.clone().unwrap(); let mut stream = start_live(&mut server, live.clone()).await; + assert_eq!( + server + .client + .execute_plan(live.clone()) + .await + .unwrap_err() + .code(), + Code::AlreadyExists + ); let mut duplicate = command_request( &session, proto::command::CommandType::SqlCommand(proto::SqlCommand { @@ -943,19 +967,51 @@ async fn native_streams_drop_on_release_disconnect_session_release_and_shutdown( ); assert_eq!(spy.commands.load(Ordering::SeqCst), 0); assert_eq!(spy.prepares.load(Ordering::SeqCst), 1); - server - .client - .release_execute(release(&session, &operation)) - .await - .unwrap(); + let mut release_request = release(&session, &operation); + release_request.client_observed_server_side_session_id = Some( + server + .client + .config(get_config(&session)) + .await + .unwrap() + .into_inner() + .server_side_session_id, + ); + let released = tokio::time::timeout( + Duration::from_secs(3), + server.client.release_execute(release_request.clone()), + ) + .await + .expect("ReleaseAll must finish without consuming the result") + .unwrap() + .into_inner(); + assert_eq!(released.operation_id.as_deref(), Some(operation.as_str())); assert_eq!( spy.streams_dropped.load(Ordering::SeqCst), 1, "release must wait for backend stream teardown" ); + let absent = server + .client + .release_execute(release_request) + .await + .unwrap() + .into_inner(); + assert_eq!( + absent.operation_id, None, + "teardown must remove the operation" + ); assert_cancelled(&mut stream).await; - let abandoned = start_live(&mut server, live.clone()).await; + let mut abandoned = server + .client + .execute_plan(live.clone()) + .await + .unwrap() + .into_inner(); + assert!(abandoned.message().await.unwrap().unwrap().schema.is_some()); + // Keep the result batch unread while proving the backend stream exists. + wait_for_count(&spy.executions, 2).await; drop(abandoned); wait_for_count(&spy.streams_dropped, 2).await; let mut restarted = tokio::time::timeout(Duration::from_secs(5), async { diff --git a/crates/spark-connect-server/tests/protocol.rs b/crates/spark-connect-server/tests/protocol.rs index 1377bcc2..38bad66f 100644 --- a/crates/spark-connect-server/tests/protocol.rs +++ b/crates/spark-connect-server/tests/protocol.rs @@ -17,25 +17,25 @@ //! Verify observable Spark Connect wire behavior independently of DataFrame conveniences. +#[path = "support/requests.rs"] +mod requests; #[path = "support/server.rs"] mod server; mod support; use std::collections::HashSet; use std::io::Cursor; -use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -use std::time::Duration; use arrow::array::Int64Array; use arrow::ipc::reader::StreamReader; use prost::Message; use proto::execute_plan_response::ResponseType; use spark_connect_proto as proto; -use spark_connect_server::engine::{Engine, EngineSession, SessionContext}; +use requests::request as range; use server::Server; -use support::{FixtureEngine, LIVE_FIXTURE}; +use support::FixtureEngine; use tonic::Code; use uuid::Uuid; @@ -43,87 +43,6 @@ async fn start_server() -> Server { Server::start_with_engine(Arc::new(FixtureEngine::default())).await } -#[derive(Default)] -struct CountingEngine { - sessions_created: AtomicUsize, -} - -impl Engine for CountingEngine { - fn create_session( - &self, - context: SessionContext, - ) -> Result, tonic::Status> { - self.sessions_created.fetch_add(1, Ordering::SeqCst); - FixtureEngine::default().create_session(context) - } -} - -fn range(session: &str, end: i64) -> proto::ExecutePlanRequest { - proto::ExecutePlanRequest { - session_id: session.into(), - user_context: Some(proto::UserContext::default()), - operation_id: Some(Uuid::new_v4().to_string()), - plan: Some(proto::Plan { - op_type: Some(proto::plan::OpType::Root(proto::Relation { - rel_type: Some(proto::relation::RelType::Range(proto::Range { - start: Some(0), - end, - step: 1, - num_partitions: None, - })), - ..Default::default() - })), - }), - ..Default::default() - } -} - -fn release(session: &str, operation: &str) -> proto::ReleaseExecuteRequest { - proto::ReleaseExecuteRequest { - session_id: session.into(), - user_context: Some(proto::UserContext::default()), - operation_id: operation.into(), - release: Some(proto::release_execute_request::Release::ReleaseAll( - proto::release_execute_request::ReleaseAll {}, - )), - ..Default::default() - } -} - -fn config_request( - session: &str, - operation: proto::config_request::operation::OpType, -) -> proto::ConfigRequest { - proto::ConfigRequest { - session_id: session.into(), - user_context: Some(proto::UserContext::default()), - operation: Some(proto::config_request::Operation { - op_type: Some(operation), - }), - ..Default::default() - } -} - -async fn assert_cancelled(stream: &mut tonic::Streaming) { - tokio::time::timeout(Duration::from_secs(5), async { - loop { - match stream.message().await { - Ok(Some(response)) => assert!(!matches!( - response.response_type, - Some(ResponseType::ResultComplete(_)) - )), - Ok(None) => panic!("cancelled execution must report cancellation"), - Err(status) => { - assert_eq!(status.code(), Code::Cancelled); - break; - } - } - } - }) - .await - .expect("cancelled stream must terminate"); -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn arrow_stream_preserves_schema_ids_offsets_and_empty_results() { let mut server = start_server().await; @@ -223,7 +142,7 @@ async fn observed_session_mismatch_and_released_sessions_are_rejected() { assert_eq!(error.code(), Code::FailedPrecondition); let release = proto::ReleaseSessionRequest { session_id: session.clone(), - user_context: Some(proto::UserContext::default()), + user_context: requests::user("alice"), ..Default::default() }; let released = server @@ -251,64 +170,6 @@ async fn observed_session_mismatch_and_released_sessions_are_rejected() { server.stop().await; } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn release_all_cancels_a_pending_backend_stream_and_waits_for_teardown() { - let engine = Arc::new(FixtureEngine::default()); - let mut server = Server::start_with_engine(engine.clone()).await; - let session = Uuid::new_v4().to_string(); - let request = range(&session, LIVE_FIXTURE); - let operation = request.operation_id.clone().unwrap(); - let mut stream = server - .client - .execute_plan(request) - .await - .unwrap() - .into_inner(); - let header = stream.message().await.unwrap().unwrap(); - assert!(header.schema.is_some()); - let batch = stream.message().await.unwrap().unwrap(); - assert!(matches!( - batch.response_type, - Some(ResponseType::ArrowBatch(_)) - )); - let release = proto::ReleaseExecuteRequest { - session_id: session.clone(), - user_context: Some(proto::UserContext::default()), - operation_id: operation.clone(), - client_observed_server_side_session_id: Some(header.server_side_session_id), - release: Some(proto::release_execute_request::Release::ReleaseAll( - proto::release_execute_request::ReleaseAll {}, - )), - ..Default::default() - }; - let released = tokio::time::timeout( - Duration::from_secs(3), - server.client.release_execute(release.clone()), - ) - .await - .expect("ReleaseAll must finish without consuming the result") - .unwrap() - .into_inner(); - assert_eq!(released.operation_id.as_deref(), Some(operation.as_str())); - assert_eq!( - engine.streams_dropped.load(Ordering::SeqCst), - 1, - "ReleaseAll must wait until the backend stream is dropped" - ); - let absent = server - .client - .release_execute(release) - .await - .unwrap() - .into_inner(); - assert_eq!( - absent.operation_id, None, - "completed teardown must remove the operation" - ); - assert_cancelled(&mut stream).await; - server.stop().await; -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn arrow_payloads_are_byte_limited_and_oversized_rows_fail() { use arrow::array::{LargeStringArray, StringArray, StringViewArray}; @@ -415,232 +276,6 @@ async fn arrow_payloads_are_byte_limited_and_oversized_rows_fail() { server.stop().await; } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn release_session_cancels_live_stream_and_live_operation_ids_are_unique() { - let mut server = start_server().await; - let session = Uuid::new_v4().to_string(); - let request = range(&session, LIVE_FIXTURE); - let mut stream = server - .client - .execute_plan(request.clone()) - .await - .unwrap() - .into_inner(); - assert!(stream.message().await.unwrap().unwrap().schema.is_some()); - let duplicate = server.client.execute_plan(request).await.unwrap_err(); - assert_eq!(duplicate.code(), Code::AlreadyExists); - server - .client - .release_session(proto::ReleaseSessionRequest { - session_id: session.clone(), - user_context: Some(proto::UserContext::default()), - ..Default::default() - }) - .await - .unwrap(); - assert_cancelled(&mut stream).await; - assert_eq!( - server - .client - .execute_plan(range(&session, 0)) - .await - .unwrap_err() - .code(), - Code::FailedPrecondition - ); - server.stop().await; -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn invalid_requests_do_not_allocate_sessions_and_unknown_release_is_rejected() { - let engine = Arc::new(CountingEngine::default()); - let mut server = Server::start_with_engine(engine.clone()).await; - let valid = Uuid::new_v4().to_string(); - assert_eq!( - server - .client - .execute_plan(range("not-a-uuid", 0)) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - let mut invalid_operation = range(&valid, 0); - invalid_operation.operation_id = Some("not-a-uuid".into()); - assert_eq!( - server - .client - .execute_plan(invalid_operation) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - for plan in [None, Some(proto::Plan { op_type: None })] { - let mut request = range(&valid, 0); - request.plan = plan; - assert_eq!( - server - .client - .execute_plan(request) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - } - assert_eq!( - server - .client - .analyze_plan(proto::AnalyzePlanRequest { - session_id: valid.clone(), - ..Default::default() - }) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - for operation in [ - None, - Some(proto::config_request::Operation { op_type: None }), - ] { - assert_eq!( - server - .client - .config(proto::ConfigRequest { - session_id: valid.clone(), - operation, - ..Default::default() - }) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - } - // A stale observed ID cannot match a session that does not exist. Repeated - // requests must not consume backend session capacity while being rejected. - for _ in 0..3 { - let mut stale = range(&valid, 0); - stale.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); - assert_eq!( - server.client.execute_plan(stale).await.unwrap_err().code(), - Code::FailedPrecondition - ); - } - let mut stale = config_request( - &valid, - proto::config_request::operation::OpType::GetOption(proto::config_request::GetOption { - keys: vec!["missing".into()], - }), - ); - stale.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); - assert_eq!( - server.client.config(stale).await.unwrap_err().code(), - Code::FailedPrecondition - ); - // Identifier and envelope validation must precede backend session allocation. - assert_eq!(engine.sessions_created.load(Ordering::SeqCst), 0); - assert_eq!( - server - .client - .release_execute(release(&valid, &Uuid::new_v4().to_string())) - .await - .unwrap_err() - .code(), - Code::NotFound - ); - assert_eq!( - server - .client - .release_execute(release("not-a-uuid", &valid)) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - assert_eq!( - server - .client - .release_execute(release(&valid, "not-a-uuid")) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - let unknown = Uuid::new_v4().to_string(); - assert_eq!( - server - .client - .release_execute(release(&unknown, &valid)) - .await - .unwrap_err() - .code(), - Code::NotFound - ); - assert_eq!(engine.sessions_created.load(Ordering::SeqCst), 0); - server - .client - .config(config_request( - &valid, - proto::config_request::operation::OpType::GetOption(proto::config_request::GetOption { - keys: vec!["spark.sql.session.timeZone".into()], - }), - )) - .await - .unwrap(); - assert_eq!(engine.sessions_created.load(Ordering::SeqCst), 1); - server.stop().await; -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn disconnect_removes_operation_and_shutdown_cancels_live_streams() { - let mut server = start_server().await; - let session = Uuid::new_v4().to_string(); - let request = range(&session, LIVE_FIXTURE); - let operation = request.operation_id.clone().unwrap(); - let mut abandoned = server - .client - .execute_plan(request.clone()) - .await - .unwrap() - .into_inner(); - assert!(abandoned.message().await.unwrap().unwrap().schema.is_some()); - drop(abandoned); - let mut resumed = tokio::time::timeout(Duration::from_secs(5), async { - loop { - match server.client.execute_plan(request.clone()).await { - Ok(response) => break response.into_inner(), - Err(status) if status.code() == Code::AlreadyExists => { - tokio::time::sleep(Duration::from_millis(20)).await; - } - Err(status) => panic!("unexpected retry failure: {status}"), - } - } - }) - .await - .expect("disconnect must remove the operation registry entry"); - assert!(resumed.message().await.unwrap().unwrap().schema.is_some()); - server - .client - .release_execute(release(&session, &operation)) - .await - .unwrap(); - assert_cancelled(&mut resumed).await; - - let mut live = server - .client - .execute_plan(range(&session, LIVE_FIXTURE)) - .await - .unwrap() - .into_inner(); - assert!(live.message().await.unwrap().unwrap().schema.is_some()); - server.shutdown.take().unwrap().send(()).unwrap(); - assert_cancelled(&mut live).await; - server.stop().await; -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn backend_status_codes_survive_prepare_and_stream_failures() { let engine = FixtureEngine::default() diff --git a/crates/spark-connect-server/tests/schema_consistency.rs b/crates/spark-connect-server/tests/schema_consistency.rs index efb9e8df..24f9bde5 100644 --- a/crates/spark-connect-server/tests/schema_consistency.rs +++ b/crates/spark-connect-server/tests/schema_consistency.rs @@ -17,8 +17,11 @@ //! Result schemas remain consumable as one Arrow table across output batches. +#[path = "support/requests.rs"] +mod requests; #[path = "support/server.rs"] mod server; +mod support; use std::collections::HashMap; use std::io::Cursor; @@ -35,75 +38,20 @@ use arrow::ipc::{reader::StreamReader, writer::StreamWriter}; use arrow::record_batch::RecordBatch; use prost::Message; use spark_connect_proto as proto; -use spark_connect_server::engine::{ - BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, - ResultBatch, SessionContext, -}; use tonic::{Code, Status}; use uuid::Uuid; use server::Server; +use support::{Batch, FixtureEngine}; const RESULT_LIMIT: usize = 3 * 1024 * 1024; const LARGE_ROWS: usize = 4096; -#[derive(Clone)] -enum Batch { - Arrow(RecordBatch), - Ipc(Vec, i64), -} - -#[derive(Clone)] -struct BatchEngine { +struct BatchCase { schema: SchemaRef, batches: Vec, } -impl Engine for BatchEngine { - fn create_session(&self, _: SessionContext) -> Result, Status> { - Ok(Arc::new(self.clone())) - } -} - -#[tonic::async_trait] -impl EngineSession for BatchEngine { - async fn prepare(&self, _: proto::plan::OpType) -> Result { - Ok(PreparedOperation::Query(Box::new(self.clone()))) - } - - async fn analyze( - &self, - _: proto::analyze_plan_request::Analyze, - ) -> Result { - Err(Status::unimplemented("schema fixture has no analysis")) - } - - async fn config( - &self, - _: proto::config_request::operation::OpType, - ) -> Result { - Err(Status::unimplemented("schema fixture has no configuration")) - } -} - -#[tonic::async_trait] -impl PreparedQuery for BatchEngine { - fn schema(&self) -> SchemaRef { - self.schema.clone() - } - - async fn execute(self: Box) -> Result { - Ok(Box::pin(futures::stream::iter( - self.batches.into_iter().map(|batch| { - Ok(match batch { - Batch::Arrow(batch) => ResultBatch::Arrow(batch), - Batch::Ipc(data, row_count) => ResultBatch::Ipc { data, row_count }, - }) - }), - ))) - } -} - fn ipc(batch: &RecordBatch) -> Vec { let mut data = Vec::new(); let mut writer = StreamWriter::try_new(&mut data, batch.schema().as_ref()).unwrap(); @@ -113,23 +61,10 @@ fn ipc(batch: &RecordBatch) -> Vec { data } -async fn execute(engine: BatchEngine) -> (Vec, Option) { +async fn execute(engine: BatchCase) -> (Vec, Option) { + let engine = FixtureEngine::default().with_results(1, engine.schema, engine.batches); let mut server = Server::start_with_engine(Arc::new(engine)).await; - let request = proto::ExecutePlanRequest { - session_id: Uuid::new_v4().to_string(), - operation_id: Some(Uuid::new_v4().to_string()), - plan: Some(proto::Plan { - op_type: Some(proto::plan::OpType::Root(proto::Relation { - rel_type: Some(proto::relation::RelType::Range(proto::Range { - end: 1, - step: 1, - ..Default::default() - })), - ..Default::default() - })), - }), - ..Default::default() - }; + let request = requests::request(&Uuid::new_v4().to_string(), 1); let mut stream = server .client .execute_plan(request) @@ -215,7 +150,7 @@ async fn changed_physical_types_and_nullability_fail_before_the_second_batch() { strings(DataType::LargeUtf8, false), strings(DataType::Utf8, true), ] { - let (responses, status) = execute(BatchEngine { + let (responses, status) = execute(BatchCase { schema: first.schema(), batches: vec![Batch::Arrow(first.clone()), Batch::Arrow(second)], }) @@ -254,7 +189,7 @@ async fn metadata_can_change_and_batch_schema_can_differ_from_the_declaration() DataType::List(Arc::new(Field::new("item", DataType::Utf8, false))), false, )])); - let (responses, status) = execute(BatchEngine { + let (responses, status) = execute(BatchCase { schema: declared, batches: vec![ Batch::Arrow(metadata_batch("first")), @@ -304,7 +239,7 @@ async fn view_ipc_is_unchanged_when_small_and_materialized_when_oversized() { let batch = views(rows); let data = ipc(&batch); assert_eq!(data.len() > RESULT_LIMIT, rows == LARGE_ROWS); - let (responses, status) = execute(BatchEngine { + let (responses, status) = execute(BatchCase { schema: batch.schema(), batches: vec![Batch::Ipc(data.clone(), rows as i64)], }) @@ -354,7 +289,7 @@ async fn mixing_small_and_oversized_view_ipc_rejects_the_changed_output_schema() let small = Batch::Ipc(ipc(&small), 2); let large = Batch::Ipc(ipc(&large), LARGE_ROWS as i64); for large_first in [false, true] { - let (responses, status) = execute(BatchEngine { + let (responses, status) = execute(BatchCase { schema: views(0).schema(), batches: if large_first { vec![large.clone(), small.clone()] @@ -425,7 +360,7 @@ async fn view_children_in_lists_large_lists_and_maps_are_materialized_when_split .collect::>(), )); let batch = RecordBatch::try_new(schema.clone(), columns).unwrap(); - let (responses, status) = execute(BatchEngine { + let (responses, status) = execute(BatchCase { schema, batches: vec![Batch::Arrow(batch)], }) diff --git a/crates/spark-connect-server/tests/support/mod.rs b/crates/spark-connect-server/tests/support/mod.rs index 0df3a439..6723e0aa 100644 --- a/crates/spark-connect-server/tests/support/mod.rs +++ b/crates/spark-connect-server/tests/support/mod.rs @@ -15,22 +15,20 @@ // specific language governing permissions and limitations // under the License. -//! Explicit backend outputs for frontend protocol tests. +//! Explicit backend outputs for frontend protocol and schema tests. //! //! Range.end is only a fixture lookup key. This backend neither interprets SQL -//! nor implements Spark operators; each case supplies Arrow batches, a pending -//! stream, or a chosen failure. These tests do not verify query execution semantics. +//! nor implements Spark operators; each case supplies batches or a chosen error. +//! Test binaries use different subsets of the fixture constructors. + +#![allow(dead_code)] use std::collections::HashMap; -use std::pin::Pin; -use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -use std::task::{Context, Poll}; use arrow::array::Int64Array; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; -use futures::{Stream, StreamExt}; use spark_connect_proto as proto; use spark_connect_server::engine::{ BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, @@ -38,28 +36,25 @@ use spark_connect_server::engine::{ }; use tonic::{Code, Status}; -pub const LIVE_FIXTURE: i64 = 1_000_000_000_000; - #[derive(Clone)] -enum Ending { - Complete, - Pending, - Error(Code), +pub enum Batch { + Arrow(RecordBatch), + Ipc(Vec, i64), } #[derive(Clone)] enum Output { Query { schema: SchemaRef, - batches: Vec, - ending: Ending, + batches: Vec, + error: Option, }, Rejected(Code), } +#[derive(Clone)] pub struct FixtureEngine { outputs: HashMap, - pub streams_dropped: Arc, } impl Default for FixtureEngine { @@ -72,59 +67,35 @@ impl Default for FixtureEngine { ) .unwrap() }; - let mut engine = Self { + Self { outputs: HashMap::new(), - streams_dropped: Arc::new(AtomicUsize::new(0)), - }; - engine.outputs.insert( - 0, - Output::Query { - schema: schema.clone(), - batches: Vec::new(), - ending: Ending::Complete, - }, - ); - engine.outputs.insert( - 1, - Output::Query { - schema: schema.clone(), - batches: vec![batch(0, 1)], - ending: Ending::Complete, - }, - ); - engine.outputs.insert( + } + .with_batches(0, schema.clone(), Vec::new()) + .with_batches(1, schema.clone(), vec![batch(0, 1)]) + .with_batches( 25_000, - Output::Query { - schema: schema.clone(), - batches: vec![batch(0, 8192), batch(8192, 16_384), batch(16_384, 25_000)], - ending: Ending::Complete, - }, - ); - engine.outputs.insert( - LIVE_FIXTURE, - Output::Query { - schema: schema.clone(), - batches: vec![batch(0, 8192)], - ending: Ending::Pending, - }, - ); - engine + schema.clone(), + vec![batch(0, 8192), batch(8192, 16_384), batch(16_384, 25_000)], + ) } } impl FixtureEngine { - pub fn with_batches( - mut self, - selector: i64, - schema: SchemaRef, - batches: Vec, - ) -> Self { + pub fn with_batches(self, selector: i64, schema: SchemaRef, batches: Vec) -> Self { + self.with_results( + selector, + schema, + batches.into_iter().map(Batch::Arrow).collect(), + ) + } + + pub fn with_results(mut self, selector: i64, schema: SchemaRef, batches: Vec) -> Self { self.outputs.insert( selector, Output::Query { schema, batches, - ending: Ending::Complete, + error: None, }, ); self @@ -141,7 +112,7 @@ impl FixtureEngine { Output::Query { schema: Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), batches: Vec::new(), - ending: Ending::Error(code), + error: Some(code), }, ); self @@ -149,21 +120,13 @@ impl FixtureEngine { } impl Engine for FixtureEngine { - fn create_session(&self, _context: SessionContext) -> Result, Status> { - Ok(Arc::new(FixtureSession { - outputs: self.outputs.clone(), - streams_dropped: self.streams_dropped.clone(), - })) + fn create_session(&self, _: SessionContext) -> Result, Status> { + Ok(Arc::new(self.clone())) } } -struct FixtureSession { - outputs: HashMap, - streams_dropped: Arc, -} - #[tonic::async_trait] -impl EngineSession for FixtureSession { +impl EngineSession for FixtureEngine { async fn prepare(&self, operation: proto::plan::OpType) -> Result { let proto::plan::OpType::Root(relation) = operation else { return Err(Status::unimplemented("fixture accepts root selectors only")); @@ -175,12 +138,11 @@ impl EngineSession for FixtureSession { Some(Output::Query { schema, batches, - ending, + error, }) => Ok(PreparedOperation::Query(Box::new(FixtureQuery { schema: schema.clone(), batches: batches.clone(), - ending: ending.clone(), - streams_dropped: self.streams_dropped.clone(), + error: *error, }))), Some(Output::Rejected(code)) => Err(Status::new(*code, "prepared fixture error")), None => Err(Status::unimplemented("unknown fixture selector")), @@ -189,7 +151,7 @@ impl EngineSession for FixtureSession { async fn analyze( &self, - _input: proto::analyze_plan_request::Analyze, + _: proto::analyze_plan_request::Analyze, ) -> Result { Err(Status::unimplemented( "fixture does not analyze query semantics", @@ -198,29 +160,18 @@ impl EngineSession for FixtureSession { async fn config( &self, - input: proto::config_request::operation::OpType, + _: proto::config_request::operation::OpType, ) -> Result { - let proto::config_request::operation::OpType::GetOption(input) = input else { - return Err(Status::unimplemented( - "fixture supports optional config lookup only", - )); - }; - Ok(ConfigResult { - pairs: input - .keys - .into_iter() - .map(|key| proto::KeyValue { key, value: None }) - .collect(), - warnings: Vec::new(), - }) + Err(Status::unimplemented( + "fixture does not configure query semantics", + )) } } struct FixtureQuery { schema: SchemaRef, - batches: Vec, - ending: Ending, - streams_dropped: Arc, + batches: Vec, + error: Option, } #[tonic::async_trait] @@ -230,36 +181,16 @@ impl PreparedQuery for FixtureQuery { } async fn execute(self: Box) -> Result { - let first = futures::stream::iter(self.batches.into_iter().map(|batch| Ok(batch.into()))); - let inner: BatchStream = match self.ending { - Ending::Complete => Box::pin(first), - Ending::Pending => Box::pin(first.chain(futures::stream::pending())), - Ending::Error(code) => Box::pin(first.chain(futures::stream::once(async move { - Err(Status::new(code, "streamed fixture error")) - }))), - }; - Ok(Box::pin(TrackedStream { - inner, - streams_dropped: self.streams_dropped, - })) - } -} - -struct TrackedStream { - inner: BatchStream, - streams_dropped: Arc, -} - -impl Stream for TrackedStream { - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { - self.inner.as_mut().poll_next(context) - } -} - -impl Drop for TrackedStream { - fn drop(&mut self) { - self.streams_dropped.fetch_add(1, Ordering::SeqCst); + let batches = self.batches.into_iter().map(|batch| { + Ok(match batch { + Batch::Arrow(batch) => ResultBatch::Arrow(batch), + Batch::Ipc(data, row_count) => ResultBatch::Ipc { data, row_count }, + }) + }); + let failure = self + .error + .into_iter() + .map(|code| Err(Status::new(code, "streamed fixture error"))); + Ok(Box::pin(futures::stream::iter(batches.chain(failure)))) } } diff --git a/crates/spark-connect-server/tests/support/requests.rs b/crates/spark-connect-server/tests/support/requests.rs new file mode 100644 index 00000000..6b054034 --- /dev/null +++ b/crates/spark-connect-server/tests/support/requests.rs @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared request envelopes and terminal-stream assertions. Individual test +//! binaries use different subsets of these helpers. + +#![allow(dead_code)] + +use std::time::Duration; + +use spark_connect_proto as proto; +use tonic::Code; +use uuid::Uuid; + +pub fn user(id: &str) -> Option { + Some(proto::UserContext { + user_id: id.into(), + ..Default::default() + }) +} + +pub fn relation(end: i64) -> proto::Relation { + proto::Relation { + rel_type: Some(proto::relation::RelType::Range(proto::Range { + end, + step: 1, + ..Default::default() + })), + ..Default::default() + } +} + +pub fn request(session: &str, end: i64) -> proto::ExecutePlanRequest { + proto::ExecutePlanRequest { + session_id: session.into(), + user_context: user("alice"), + operation_id: Some(Uuid::new_v4().to_string()), + plan: Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(relation(end))), + }), + ..Default::default() + } +} + +pub fn config_request( + session: &str, + input: proto::config_request::operation::OpType, +) -> proto::ConfigRequest { + proto::ConfigRequest { + session_id: session.into(), + user_context: user("alice"), + operation: Some(proto::config_request::Operation { + op_type: Some(input), + }), + ..Default::default() + } +} + +pub fn release(session: &str, operation: &str) -> proto::ReleaseExecuteRequest { + proto::ReleaseExecuteRequest { + session_id: session.into(), + user_context: user("alice"), + operation_id: operation.into(), + release: Some(proto::release_execute_request::Release::ReleaseAll( + proto::release_execute_request::ReleaseAll {}, + )), + ..Default::default() + } +} + +pub fn release_session(session: &str) -> proto::ReleaseSessionRequest { + proto::ReleaseSessionRequest { + session_id: session.into(), + user_context: user("alice"), + ..Default::default() + } +} + +pub async fn assert_cancelled(stream: &mut tonic::Streaming) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + match stream.message().await { + Err(status) => { + assert_eq!(status.code(), Code::Cancelled); + break; + } + Ok(Some(response)) => assert!(!matches!( + response.response_type, + Some(ResponseType::ResultComplete(_)) + )), + Ok(None) => panic!("cancelled execution must report cancellation"), + } + } + }) + .await + .expect("cancelled execution must terminate"); +} From 609f3c43e2e7f67d72809b23f557369eb27c90f6 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 20:28:39 -0800 Subject: [PATCH 13/18] Import shared cancellation response variant --- crates/spark-connect-server/tests/support/requests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/spark-connect-server/tests/support/requests.rs b/crates/spark-connect-server/tests/support/requests.rs index 6b054034..185fce4a 100644 --- a/crates/spark-connect-server/tests/support/requests.rs +++ b/crates/spark-connect-server/tests/support/requests.rs @@ -22,6 +22,7 @@ use std::time::Duration; +use proto::execute_plan_response::ResponseType; use spark_connect_proto as proto; use tonic::Code; use uuid::Uuid; From 28b44e51633821011598c12a8c6933ad813559c1 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 20:44:20 -0800 Subject: [PATCH 14/18] Preserve Arrow IPC with negotiated byte chunking --- crates/spark-connect-server/README.md | 42 +- crates/spark-connect-server/src/engine/mod.rs | 25 +- crates/spark-connect-server/src/lib.rs | 34 +- crates/spark-connect-server/src/service.rs | 1053 ++--------------- .../tests/engine_injection.rs | 581 ++++++--- crates/spark-connect-server/tests/protocol.rs | 723 ++++++++--- .../tests/review_regressions.rs | 652 ---------- .../tests/schema_consistency.rs | 125 +- .../tests/support/arrow.rs | 112 ++ .../tests/support/server.rs | 9 +- 10 files changed, 1383 insertions(+), 1973 deletions(-) delete mode 100644 crates/spark-connect-server/tests/review_regressions.rs create mode 100644 crates/spark-connect-server/tests/support/arrow.rs diff --git a/crates/spark-connect-server/README.md b/crates/spark-connect-server/README.md index 620a7df6..3b4ac6db 100644 --- a/crates/spark-connect-server/README.md +++ b/crates/spark-connect-server/README.md @@ -23,6 +23,8 @@ This unpublished library serves a subset of the Spark Connect protocol using the workspace's `apache-spark-connect-proto` messages and its optional `server` bindings. The caller supplies an `Engine` implementation and a bound Tokio TCP listener to `serve_with_engine`. +Use `serve_with_engine_and_options` with `ServerOptions` to configure Arrow +chunking. No execution engine or standalone server executable is included. Rust owns protocol sessions, operation IDs, response framing and result-stream @@ -60,16 +62,31 @@ assemble them into one Arrow table. Metadata may differ between batches. Declared field metadata and unsupported types fail explicitly. Engines may yield Arrow record batches or complete Arrow IPC streams containing -one batch and its declared row count. The frontend encodes record batches and -forwards IPC that fits the response limit; oversized batches are split by rows. -Encoding materializes string/binary view arrays, including nested children, to -`LargeUtf8`/`LargeBinary`. Small view-typed IPC remains unchanged, so mixing it with -oversized view-typed IPC can produce different physical schemas. The frontend -reports `INTERNAL` before sending a batch whose output schema differs from earlier -batches; it does not complete a result containing mixed schemas. Engines should -use consistent non-view IPC types when mixing batch sizes. -Result envelopes, including schema and batch responses, are limited to 3 MiB. -A single row or schema that cannot fit fails with `RESOURCE_EXHAUSTED`. Each batch +one batch and its actual nonnegative row count. The frontend encodes each record +batch once and forwards existing IPC bytes unchanged, including compressed IPC. +It reads the IPC schema for validation and trusts the producer's batch contents +and declared row count. Record-batch encoding materializes string/binary view +arrays, including nested children, to `LargeUtf8`/`LargeBinary`. Existing IPC +retains its physical types. The frontend reports `INTERNAL` before sending a +batch whose output schema differs from earlier batches; engines mixing record +batches with IPC must use consistent output representations. + +When the client sets `ResultChunkingOptions.allow_arrow_batch_chunking`, the +frontend divides serialized IPC bytes into chunks and sets `chunk_index` and +`num_chunks_in_batch`. Boundaries need not align with rows or Arrow buffers; the +client concatenates a batch's chunks before decoding or decompressing it. A +preferred chunk payload size between 1 KiB and the configured maximum is accepted; +otherwise the maximum is used. `ServerOptions.max_arrow_chunk_size` defaults to +120,795,955 bytes (90% of 128 MiB), matching the JVM server's default chunk payload +limit. It limits IPC payload bytes, excluding the protobuf envelope. Setting it +to zero disables chunking even when the client opts in. Clients must allow room +for the envelope in their gRPC receive limits. + +Without client opt-in, or when chunking is disabled, each IPC batch is sent whole. +Engines remain responsible for bounding batch production, including fitting the +client's gRPC receive limit when batches are sent whole. The frontend applies no +128 MiB batch-production cap. Schema and command response envelopes have a +separate 3 MiB limit and fail with `RESOURCE_EXHAUSTED` when oversized. Each batch carries a cumulative row offset, and empty results still include an empty batch. ## Lifetime and trust requirements @@ -79,7 +96,10 @@ frontend cancels them when a client disconnects, an operation is released, a session is released, server shutdown begins or the serving future is dropped. Results pass through a bounded channel, so unread responses eventually stop polling the engine. Blocking Arrow -encoding retains operation ownership until its worker finishes. +encoding retains operation ownership until its worker finishes. IPC chunks share +their batch's backing allocation, which remains live while any chunk references +it. Larger batches and chunk limits can therefore increase retained memory even +though the response channel is bounded. `ReleaseAll` waits up to two seconds for teardown and reports `DEADLINE_EXCEEDED` if it is still draining. Shutdown cancels sessions and waits up to two seconds diff --git a/crates/spark-connect-server/src/engine/mod.rs b/crates/spark-connect-server/src/engine/mod.rs index e2285e67..edfadfd4 100644 --- a/crates/spark-connect-server/src/engine/mod.rs +++ b/crates/spark-connect-server/src/engine/mod.rs @@ -101,11 +101,11 @@ pub trait PreparedQuery: Send { /// equivalent Arrow representations are accepted. /// Batch nullability and metadata may differ from this declaration, but /// emitted IPC batches in one result must share physical types, field names - /// and nullability. Metadata may differ between batches. Encoding materializes - /// view arrays to large-offset types, including when splitting oversized IPC; - /// mixing those with unchanged view IPC fails explicitly before the changed - /// output schema is sent. Declared fields must not contain metadata, which - /// Spark schema conversion rejects. + /// and nullability. Metadata may differ between batches. Record-batch encoding + /// materializes view arrays to large-offset types; existing IPC retains its + /// physical types. Mixing incompatible output representations fails before + /// the changed output schema is sent. Declared fields must not contain + /// metadata, which Spark schema conversion rejects. fn schema(&self) -> SchemaRef; /// Dropping this future or its returned stream must cancel backend work; @@ -114,14 +114,21 @@ pub trait PreparedQuery: Send { } /// Engine output, optionally already serialized for Spark Connect. +/// +/// Engines must bound batch production. Optional byte chunking bounds network +/// payloads, not the complete batch's allocation. When chunking is disabled or +/// the client does not opt in, each batch must fit the client's gRPC receive limit. pub enum ResultBatch { + /// Encoded once as a complete IPC stream before optional byte chunking. Arrow(RecordBatch), /// A complete IPC stream containing its schema, exactly one record batch, /// and EOS. The producer must supply that batch's actual nonnegative row count. - Ipc { - data: Vec, - row_count: i64, - }, + /// The frontend validates the schema and trusts the producer's batch contents + /// and declared row count. IPC bytes, including compression, are preserved. + /// Clients that opt in receive byte chunks for reassembly before decoding; + /// otherwise the whole stream is sent, so producers must respect the client's + /// gRPC receive limit when selecting batch sizes. + Ipc { data: Vec, row_count: i64 }, } impl From for ResultBatch { diff --git a/crates/spark-connect-server/src/lib.rs b/crates/spark-connect-server/src/lib.rs index 64db976b..f42f1895 100644 --- a/crates/spark-connect-server/src/lib.rs +++ b/crates/spark-connect-server/src/lib.rs @@ -25,6 +25,24 @@ pub mod grpc { pub use spark_connect_proto::spark_connect_service_server; } +/// Transport options for the experimental frontend. +#[derive(Clone, Debug)] +pub struct ServerOptions { + /// Maximum Arrow chunk payload in bytes, excluding the protobuf envelope. + /// Defaults to 90% of 128 MiB, as in Spark's JVM server. A client can request + /// a smaller size. Zero disables chunking, even when the client opts in. + /// Whole batches sent without chunking are not bounded by this option. + pub max_arrow_chunk_size: usize, +} + +impl Default for ServerOptions { + fn default() -> Self { + Self { + max_arrow_chunk_size: 128 * 1024 * 1024 * 9 / 10, + } + } +} + struct CancelOnDrop(std::sync::Arc); impl Drop for CancelOnDrop { @@ -49,7 +67,21 @@ pub async fn serve_with_engine( engine: std::sync::Arc, shutdown: impl std::future::Future + Send + 'static, ) -> Result<(), tonic::transport::Error> { - let service = std::sync::Arc::new(service::ConnectService::new(engine)); + serve_with_engine_and_options(listener, engine, ServerOptions::default(), shutdown).await +} + +/// Serve with an injected engine and explicit transport options. +/// +/// Session lifetime and shutdown follow [`serve_with_engine`]. Producers should +/// bound engine batches independently of this transport chunk size. Clients must +/// allow protobuf overhead in their gRPC receive limits. +pub async fn serve_with_engine_and_options( + listener: tokio::net::TcpListener, + engine: std::sync::Arc, + options: ServerOptions, + shutdown: impl std::future::Future + Send + 'static, +) -> Result<(), tonic::transport::Error> { + let service = std::sync::Arc::new(service::ConnectService::new(engine, options)); let _cancel_on_drop = CancelOnDrop(service.clone()); let router = tonic::transport::Server::builder().add_service( grpc::spark_connect_service_server::SparkConnectServiceServer::from_arc(service.clone()) diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index 11baf810..f2191251 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -70,8 +70,6 @@ struct Session { struct Operation { cancelled: watch::Sender, done: watch::Sender, - #[cfg(test)] - encoding_attempts: std::sync::atomic::AtomicUsize, } impl Session { @@ -109,8 +107,6 @@ impl Session { let running = Arc::new(Operation { cancelled, done: watch::channel(false).0, - #[cfg(test)] - encoding_attempts: std::sync::atomic::AtomicUsize::new(0), }); let mut operations = self .operations @@ -181,14 +177,16 @@ impl SessionRegistry { pub(crate) struct ConnectService { engine: Arc, + options: crate::ServerOptions, sessions: Mutex, closing: std::sync::atomic::AtomicBool, } impl ConnectService { - pub(crate) fn new(engine: Arc) -> Self { + pub(crate) fn new(engine: Arc, options: crate::ServerOptions) -> Self { Self { engine, + options, sessions: Mutex::new(SessionRegistry::default()), closing: std::sync::atomic::AtomicBool::new(false), } @@ -315,10 +313,8 @@ fn encode_batch(batch: &RecordBatch) -> Result, Status> { use arrow::datatypes::DataType; use arrow::record_batch::RecordBatchOptions; - // IPC writes all backing buffers of view arrays even after a row slice. Materialize - // their bytes, including nested children, so splitting measures only retained rows. - // Use 64-bit offsets so materializing a batch cannot overflow 32-bit byte offsets - // before the result-size check has a chance to split it. + // Materialize view arrays, including nested children, into the large-offset + // string/binary representation used for encoded RecordBatch results. let source_schema = batch.schema(); let fields = schema::map_fields(source_schema.fields(), &|field, data_type| { let data_type = match data_type { @@ -400,83 +396,6 @@ fn check_output_schema(schema: &Schema, expected: &mut Option) -> Result } } -fn estimated_array_bytes(array: &dyn arrow::array::Array) -> usize { - use arrow::array::{Array, AsArray, OffsetSizeTrait}; - use arrow::datatypes::DataType; - - fn nested_bytes(offsets: &[O], values: &dyn Array) -> usize { - let start = offsets[0].as_usize(); - let end = offsets[offsets.len() - 1].as_usize(); - // List and map slices retain their full child arrays. Like the IPC - // writer, inspect only the child range referenced by these offsets. - offsets - .len() - .saturating_mul(std::mem::size_of::()) - .saturating_add(estimated_array_bytes( - values.slice(start, end - start).as_ref(), - )) - } - - fn view_bytes(len: usize, lengths: impl Iterator) -> usize { - // Encoding materializes views with 64-bit offsets. Count repeated - // references separately, including null-slot lengths: Arrow's cast - // reserves their bytes even though it emits no value for a null slot. - lengths.fold(len.saturating_add(1).saturating_mul(8), |bytes, len| { - bytes.saturating_add(len as usize) - }) - } - - let bytes = match array.data_type() { - DataType::List(_) => { - let list = array.as_list::(); - nested_bytes(list.value_offsets(), list.values().as_ref()) - } - DataType::LargeList(_) => { - let list = array.as_list::(); - nested_bytes(list.value_offsets(), list.values().as_ref()) - } - DataType::Map(_, _) => { - let map = array.as_map(); - nested_bytes(map.value_offsets(), map.entries()) - } - DataType::Struct(_) => array - .as_struct() - .columns() - .iter() - .fold(0usize, |bytes, child| { - bytes.saturating_add(estimated_array_bytes(child.as_ref())) - }), - DataType::Utf8View => view_bytes(array.len(), array.as_string_view().lengths()), - DataType::BinaryView => view_bytes(array.len(), array.as_binary_view().lengths()), - _ => { - return array - .to_data() - .get_slice_memory_size() - .unwrap_or_else(|_| array.get_array_memory_size()); - } - }; - bytes.saturating_add(array.nulls().map_or(0, |_| array.len().div_ceil(8))) -} - -#[cfg(test)] -fn estimated_rows_per_batch(batch: &RecordBatch) -> usize { - estimated_rows_for_payload(batch, MAX_RESULT_BYTES / 2) -} - -fn estimated_rows_for_payload(batch: &RecordBatch, payload_bytes: usize) -> usize { - let bytes = batch.columns().iter().fold(0usize, |bytes, column| { - bytes.saturating_add(estimated_array_bytes(column.as_ref())) - }); - // Start below the exact IPC limit to allow schema/offset overhead. This is - // only an estimate: skewed rows and annotations are checked after encoding. - rows_for_bytes(batch.num_rows(), bytes, payload_bytes).min(batch.num_rows().max(1)) -} - -fn rows_for_bytes(rows: usize, bytes: usize, payload_bytes: usize) -> usize { - (rows as u128 * payload_bytes as u128 / bytes.max(1) as u128).clamp(1, usize::MAX as u128) - as usize -} - fn checked_arrow_row_count(rows: usize, offset: i64) -> Result { let rows = i64::try_from(rows) .map_err(|_| Status::resource_exhausted("result batch row count exceeds i64"))?; @@ -486,25 +405,46 @@ fn checked_arrow_row_count(rows: usize, offset: i64) -> Result { Ok(rows) } -// Fields drop in declaration order, including when encoding panics or a -// cancelled caller drops a completed JoinHandle without observing its output. +// Keep backend ownership until the batch is dropped, including on encoder panic. struct EncodingBatch { batch: RecordBatch, _guard: Arc, } impl EncodingBatch { - fn encode(self) -> Result<(Self, Vec), Status> { - #[cfg(test)] - self._guard - .running - .encoding_attempts - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - let data = encode_batch(&self.batch)?; - Ok((self, data)) + fn encode(self) -> Result, Status> { + encode_batch(&self.batch) } } +fn arrow_chunk_size( + options: &[proto::execute_plan_request::RequestOption], + maximum: usize, +) -> Option { + use proto::execute_plan_request::request_option::RequestOption; + + let mut options = options + .iter() + .filter_map(|option| match &option.request_option { + Some(RequestOption::ResultChunkingOptions(options)) => Some(options), + _ => None, + }); + if maximum == 0 + || !options + .clone() + .any(|options| options.allow_arrow_batch_chunking) + { + return None; + } + Some( + options + .find_map(|options| options.preferred_arrow_chunk_size) + .and_then(|bytes| usize::try_from(bytes).ok()) + .filter(|bytes| (1024..=maximum).contains(bytes)) + .unwrap_or(maximum), + ) +} + async fn send_batch( sender: &mpsc::Sender>, id: &str, @@ -514,172 +454,74 @@ async fn send_batch( offset: &mut i64, output_schema: &mut Option, guard: &Arc, + chunk_size: Option, ) -> Result { - let (batch, rows_per_batch) = match batch { - ResultBatch::Arrow(batch) => { - let rows = batch.num_rows().max(1); - (batch, rows) + // Reject unrepresentable counts/offsets before encoding or sending any chunk. + let row_count = match &batch { + ResultBatch::Arrow(batch) => checked_arrow_row_count(batch.num_rows(), *offset)?, + ResultBatch::Ipc { row_count, .. } if *row_count < 0 => { + return Err(Status::internal("negative engine IPC row count")); } - ResultBatch::Ipc { data, row_count } => { - if row_count < 0 { - return Err(Status::internal("negative engine IPC row count")); - } - let next_offset = offset - .checked_add(row_count) - .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; - let batch = proto::execute_plan_response::ArrowBatch { - row_count, - data: data.into(), - start_offset: Some(*offset), - ..Default::default() + ResultBatch::Ipc { row_count, .. } => *row_count, + }; + let next_offset = offset + .checked_add(row_count) + .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; + let data: prost::bytes::Bytes = match batch { + ResultBatch::Arrow(batch) => { + // A blocking encoder cannot be cancelled once started. Retain engine + // ownership until its batch is dropped, even if the caller goes away. + let encoding = EncodingBatch { + batch, + _guard: guard.clone(), }; - let result = response( - id, - &guard.session, - operation, - Some(proto::execute_plan_response::ResponseType::ArrowBatch( - batch.clone(), - )), - ); - if result.encoded_len() <= MAX_RESULT_BYTES { - check_output_schema(batch_schema, output_schema)?; - *offset = next_offset; - return Ok(sender.send(Ok(result)).await.is_ok()); - } - drop(result); - // Ordinary IPC is forwarded unchanged. Only oversized payloads need - // vectors so the existing row splitter can enforce the envelope limit. - let guard = guard.clone(); - tokio::task::spawn_blocking(move || { - let _guard = guard; - let ipc_bytes = batch.data.len(); - let mut reader = StreamReader::try_new(&batch.data[..], None) - .map_err(|e| Status::internal(format!("invalid engine IPC: {e}")))?; - let batch = reader - .next() - .transpose() - .map_err(|e| Status::internal(format!("invalid engine IPC: {e}")))? - .ok_or_else(|| Status::internal("engine IPC contains no record batch"))?; - if i64::try_from(batch.num_rows()).ok() != Some(row_count) - || reader.next().is_some() - { - return Err(Status::internal( - "engine IPC batch count differs from its declaration", - )); - } - // Seed oversized IPC splitting from its serialized size; the - // payload-aware preflight refines this before encoding. - let rows = rows_for_bytes(batch.num_rows(), ipc_bytes, MAX_RESULT_BYTES / 2); - Ok((batch, rows)) - }) - .await - .map_err(|_| Status::internal("Arrow IPC decoding task failed"))?? + let data = tokio::task::spawn_blocking(move || encoding.encode()) + .await + .map_err(|_| Status::internal("Arrow result encoding task failed"))??; + let schema = StreamReader::try_new(data.as_slice(), None) + .map_err(|e| Status::internal(format!("invalid output IPC: {e}")))? + .schema(); + check_output_schema(schema.as_ref(), output_schema)?; + data.into() + } + ResultBatch::Ipc { data, .. } => { + check_output_schema(batch_schema, output_schema)?; + data.into() } }; - send_arrow_batch( - sender, - id, - operation, - batch, - rows_per_batch, - offset, - output_schema, - guard, - ) - .await -} - -async fn send_arrow_batch( - sender: &mpsc::Sender>, - id: &str, - operation: &str, - batch: RecordBatch, - rows_per_batch: usize, - offset: &mut i64, - output_schema: &mut Option, - guard: &Arc, -) -> Result { - // Validate the complete engine batch before slicing or IPC encoding. Even - // buffer-free arrays can represent more rows than the signed protocol. - checked_arrow_row_count(batch.num_rows(), *offset)?; - // Schema metadata and wide schemas repeat in every IPC response. Measure - // their fixed cost once, using fresh empty arrays so nested view buffers - // retained by a zero-row slice cannot be materialized here. - let empty = EncodingBatch { - batch: RecordBatch::new_empty(batch.schema()), - _guard: guard.clone(), + let mut payload = proto::execute_plan_response::ArrowBatch { + row_count, + start_offset: Some(*offset), + ..Default::default() }; - let (empty, data) = tokio::task::spawn_blocking(move || empty.encode()) - .await - .map_err(|_| Status::internal("Arrow result encoding task failed"))??; - let fixed_bytes = response( - id, - &guard.session, - operation, - Some(proto::execute_plan_response::ResponseType::ArrowBatch( - proto::execute_plan_response::ArrowBatch { - row_count: 0, - data: data.into(), - start_offset: Some(*offset), - ..Default::default() - }, - )), - ) - .encoded_len(); - drop(empty); - let payload_bytes = (MAX_RESULT_BYTES.saturating_sub(fixed_bytes) / 2).max(1); - let mut pending = vec![(batch, rows_per_batch)]; - while let Some((batch, rows_per_batch)) = pending.pop() { - let mut rows_per_batch = rows_per_batch.max(1).min(batch.num_rows().max(1)); - while rows_per_batch > 1 { - let prefix; - let candidate = if rows_per_batch >= batch.num_rows() { - &batch - } else { - prefix = batch.slice(0, rows_per_batch); - &prefix - }; - let estimated = estimated_rows_for_payload(candidate, payload_bytes); - if estimated >= candidate.num_rows() { - break; - } - // Growth from a light prefix can reach a dense tail. Inspect its - // logical buffers before encoding rather than repeatedly copying - // an already-known oversized payload. Exact IPC/envelope checks - // below still handle schema overhead and an unfittable single row. - rows_per_batch = estimated; - } - // Keep the remainder as one slice instead of allocating a pending entry - // per row in a large engine batch. - let has_remainder = batch.num_rows() > rows_per_batch; - let batch = if has_remainder { - pending.push(( - batch.slice(rows_per_batch, batch.num_rows() - rows_per_batch), - rows_per_batch, - )); - batch.slice(0, rows_per_batch) - } else { - batch - }; - let row_count = checked_arrow_row_count(batch.num_rows(), *offset)?; - // Arrow materialization/IPC encoding is CPU work. Retain operation - // ownership if cancellation drops the JoinHandle, so ReleaseAll cannot - // claim drain while a result batch is still being encoded. - let encoding = EncodingBatch { - batch, - _guard: guard.clone(), - }; - let (encoded, data) = tokio::task::spawn_blocking(move || encoding.encode()) - .await - .map_err(|_| Status::internal("Arrow result encoding task failed"))??; - let batch = &encoded.batch; - let payload = proto::execute_plan_response::ArrowBatch { - row_count, - data: data.into(), - start_offset: Some(*offset), - ..Default::default() - }; - let result = response( + let Some(chunk_bytes) = chunk_size else { + // As on the JVM, clients without chunking opt-in receive the original + // whole IPC batch. Their gRPC receive limit must accommodate it. + payload.data = data; + let message = response( + id, + &guard.session, + operation, + Some(proto::execute_plan_response::ResponseType::ArrowBatch( + payload, + )), + ); + *offset = next_offset; + return Ok(sender.send(Ok(message)).await.is_ok()); + }; + let chunks = data.len().div_ceil(chunk_bytes).max(1); + payload.num_chunks_in_batch = Some( + i64::try_from(chunks) + .map_err(|_| Status::resource_exhausted("Arrow chunk count exceeds i64"))?, + ); + for index in 0..chunks { + let from = index * chunk_bytes; + let to = from.saturating_add(chunk_bytes).min(data.len()); + // Bytes slices share the original IPC allocation, including compressed + // buffers. Clients concatenate the chunks before decoding Arrow. + payload.data = data.slice(from..to); + payload.chunk_index = Some(index as i64); + let message = response( id, &guard.session, operation, @@ -687,56 +529,11 @@ async fn send_arrow_batch( payload.clone(), )), ); - let result_bytes = result.encoded_len(); - if result_bytes > MAX_RESULT_BYTES { - if batch.num_rows() <= 1 { - return Err(Status::resource_exhausted( - "one Arrow row/schema exceeds the 3 MiB result limit", - )); - } - // Recheck every slice including the envelope: skew and schema - // overhead can exceed the initial memory estimate. - let rows_per_batch = ((batch.num_rows() as u128 * MAX_RESULT_BYTES as u128 * 9 - / (result_bytes as u128 * 10)) as usize) - .clamp(1, batch.num_rows() - 1); - drop(result); - if has_remainder { - if let Some((_, next_rows)) = pending.last_mut() { - // Teach this piece's own remainder about IPC padding or - // other fixed costs absent from the logical estimate. - *next_rows = (*next_rows).min(rows_per_batch); - } - } - pending.push((batch.clone(), rows_per_batch)); - continue; - } - let emitted_schema = StreamReader::try_new(&payload.data[..], None) - .map_err(|error| Status::internal(format!("invalid output IPC: {error}")))? - .schema(); - check_output_schema(emitted_schema.as_ref(), output_schema)?; - if has_remainder { - if let Some((_, next_rows)) = pending.last_mut() { - // Adapt this piece's own remainder, without changing an outer - // sibling when the piece finishes. The row-growth limit is a - // sampling heuristic, not a bound on bytes for skewed rows. - *next_rows = rows_for_bytes( - batch.num_rows(), - result_bytes.saturating_sub(fixed_bytes), - payload_bytes, - ) - // A successful size is already known to fit. Fixed buffer - // padding must not make that row budget decay on each send. - .max(batch.num_rows()) - .min(batch.num_rows().saturating_mul(4).max(1)); - } - } - *offset = offset - .checked_add(row_count) - .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; - if sender.send(Ok(result)).await.is_err() { + if sender.send(Ok(message)).await.is_err() { return Ok(false); } } + *offset = next_offset; Ok(true) } @@ -750,6 +547,7 @@ impl SparkConnectService for ConnectService { request: Request, ) -> Result, Status> { let req = request.into_inner(); + let chunk_size = arrow_chunk_size(&req.request_options, self.options.max_arrow_chunk_size); let operation = req .operation_id .unwrap_or_else(|| Uuid::new_v4().to_string()); @@ -843,6 +641,7 @@ impl SparkConnectService for ConnectService { &mut offset, &mut output_schema, &guard, + chunk_size, ) .await? { @@ -865,6 +664,7 @@ impl SparkConnectService for ConnectService { &mut offset, &mut output_schema, &guard, + chunk_size, ) .await? { @@ -1165,61 +965,12 @@ mod tests { ); } - #[test] - fn initial_row_budget_splits_large_batches_before_encoding() { - use arrow::array::Int64Array; - use arrow::datatypes::{DataType, Field, Schema}; - - let batch = RecordBatch::try_new( - Arc::new(Schema::new(vec![Field::new( - "value", - DataType::Int64, - false, - )])), - vec![Arc::new(Int64Array::from_iter_values(0..1_000_000))], - ) - .unwrap(); - let rows = estimated_rows_per_batch(&batch); - assert!(rows > 0 && rows < batch.num_rows()); - assert!(rows * std::mem::size_of::() <= MAX_RESULT_BYTES / 2); - assert_eq!( - estimated_rows_per_batch(&RecordBatch::new_empty(batch.schema())), - 1 - ); - } - - #[test] - fn view_estimates_count_repeated_and_null_slot_lengths() { - use arrow::array::{BinaryViewArray, StringViewArray}; - use arrow::buffer::NullBuffer; - - let source = StringViewArray::from(vec!["abcdefghijklmno"]); - let (views, buffers, _) = source.into_parts(); - let repeated = vec![views[0]; 4].into(); - let nulls = Some(NullBuffer::from(vec![true, false, true, false])); - let strings = StringViewArray::new(repeated, buffers, nulls); - // Four logical values reserve four payloads during materialization, - // even though they share one buffer and two slots are null. IPC also - // needs five 64-bit offsets and a validity byte. - let expected = 4 * 15 + 5 * 8 + 1; - assert_eq!(estimated_array_bytes(&strings), expected); - assert_eq!(strings.total_bytes_len(), 2 * 15); - let (views, buffers, nulls) = strings.into_parts(); - let binary = BinaryViewArray::new(views, buffers, nulls); - assert_eq!(estimated_array_bytes(&binary), expected); - assert_eq!( - estimated_array_bytes(&binary.slice(1, 2)), - 2 * 15 + 3 * 8 + 1 - ); - } - #[cfg(target_pointer_width = "64")] #[tokio::test] async fn unrepresentable_arrow_row_counts_fail_before_encoding() { use arrow::array::NullArray; use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatchOptions; - use std::sync::atomic::Ordering; let rows = i64::MAX as usize + 1; let empty = RecordBatch::try_new_with_options( @@ -1239,41 +990,24 @@ mod tests { let (sender, mut receiver) = mpsc::channel(2); let mut offset = 7; let mut output_schema = None; - for direct in [false, true] { - let error = if direct { - send_arrow_batch( - &sender, - "session", - "row_count", - batch.clone(), - rows, - &mut offset, - &mut output_schema, - &guard, - ) - .await - .unwrap_err() - } else { - send_batch( - &sender, - "session", - "row_count", - batch.clone().into(), - batch.schema().as_ref(), - &mut offset, - &mut output_schema, - &guard, - ) - .await - .unwrap_err() - }; - assert_eq!(error.code(), tonic::Code::ResourceExhausted); - assert_eq!(error.message(), "result batch row count exceeds i64"); - assert_eq!(offset, 7); - assert!(output_schema.is_none()); - assert_eq!(guard.running.encoding_attempts.load(Ordering::SeqCst), 0); - assert!(receiver.try_recv().is_err()); - } + let error = send_batch( + &sender, + "session", + "row_count", + batch.clone().into(), + batch.schema().as_ref(), + &mut offset, + &mut output_schema, + &guard, + Some(MAX_RESULT_BYTES), + ) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::ResourceExhausted); + assert_eq!(error.message(), "result batch row count exceeds i64"); + assert_eq!(offset, 7); + assert!(output_schema.is_none()); + assert!(receiver.try_recv().is_err()); } } @@ -1281,7 +1015,6 @@ mod tests { async fn overflowing_engine_batches_do_not_partially_emit() { use arrow::array::LargeStringArray; use arrow::datatypes::{DataType, Field, Schema}; - use std::sync::atomic::Ordering; let schema = Arc::new(Schema::new(vec![Field::new( "text", @@ -1314,10 +1047,10 @@ mod tests { &mut offset, &mut output_schema, &guard, + Some(MAX_RESULT_BYTES), ) .await .unwrap(); - let before = guard.running.encoding_attempts.load(Ordering::SeqCst); let batch = if ipc { let data = encode_batch(&oversized).unwrap(); assert!(data.len() > MAX_RESULT_BYTES); @@ -1334,16 +1067,13 @@ mod tests { &mut offset, &mut output_schema, &guard, + Some(MAX_RESULT_BYTES), ) .await .unwrap_err(); assert_eq!(error.code(), tonic::Code::ResourceExhausted); assert_eq!(error.message(), "result row offset exceeds i64"); assert_eq!(offset, i64::MAX - 1); - assert_eq!( - guard.running.encoding_attempts.load(Ordering::SeqCst), - before - ); let previous = receiver.try_recv().unwrap().unwrap(); let Some(proto::execute_plan_response::ResponseType::ArrowBatch(previous)) = previous.response_type @@ -1363,6 +1093,7 @@ mod tests { &mut offset, &mut output_schema, &guard, + Some(MAX_RESULT_BYTES), ) .await .unwrap(); @@ -1370,536 +1101,6 @@ mod tests { } } - #[tokio::test] - async fn successful_encoding_grows_a_conservative_row_budget() { - use arrow::array::Int64Array; - use arrow::datatypes::{DataType, Field, Schema}; - - let batch = RecordBatch::try_new( - Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), - vec![Arc::new(Int64Array::from_iter_values(0..1000))], - ) - .unwrap(); - let session = Arc::new(Session::new(Arc::new(RegistrySession))); - let (guard, _) = session.operation("adaptive").unwrap(); - let (sender, mut receiver) = mpsc::channel(1024); - let mut offset = 0; - let mut output_schema = None; - assert!(send_arrow_batch( - &sender, - "session", - "adaptive", - batch, - 1, - &mut offset, - &mut output_schema, - &guard, - ) - .await - .unwrap()); - let (mut batches, mut rows) = (0, 0); - while let Ok(result) = receiver.try_recv() { - let result = result.unwrap(); - assert!(result.encoded_len() <= MAX_RESULT_BYTES); - let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = - result.response_type - else { - panic!("expected Arrow response"); - }; - assert_eq!(batch.start_offset, Some(rows)); - batches += 1; - rows += batch.row_count; - } - assert_eq!(rows, 1000); - assert_eq!(offset, rows); - assert!( - batches <= 8, - "a conservative estimate must adapt, got {batches} batches" - ); - } - - #[tokio::test] - async fn fixed_schema_overhead_does_not_collapse_the_payload_budget() { - use arrow::array::{Int64Array, LargeStringArray}; - use arrow::datatypes::{DataType, Field, Schema}; - use std::collections::HashMap; - use std::sync::atomic::Ordering; - - for (metadata_bytes, expected_rows, max_responses) in [ - ((1.49 * 1024.0 * 1024.0) as usize, 1500usize, 18usize), - ((2.6 * 1024.0 * 1024.0) as usize, 300usize, 14usize), - ] { - let schema = Arc::new(Schema::new_with_metadata( - vec![ - Field::new("id", DataType::Int64, false), - Field::new("text", DataType::LargeUtf8, false), - ], - HashMap::from([("padding".into(), "x".repeat(metadata_bytes))]), - )); - let text = "z".repeat(8192); - let batch = RecordBatch::try_new( - schema, - vec![ - Arc::new(Int64Array::from_iter_values(0..expected_rows as i64)), - Arc::new(LargeStringArray::from_iter_values(std::iter::repeat_n( - text.as_str(), - expected_rows, - ))), - ], - ) - .unwrap(); - let budget = batch.num_rows().max(1); - let session = Arc::new(Session::new(Arc::new(RegistrySession))); - let (guard, _) = session.operation("fixed_overhead").unwrap(); - // Bound retained metadata even if the old one-row decay returns; - // the timeout then fails without buffering gigabytes of replies. - let (sender, mut receiver) = mpsc::channel(max_responses + 1); - let mut offset = 0; - let mut output_schema = None; - assert!(tokio::time::timeout( - std::time::Duration::from_secs(15), - send_arrow_batch( - &sender, - "session", - "fixed_overhead", - batch, - budget, - &mut offset, - &mut output_schema, - &guard - ) - ) - .await - .expect("fixed overhead exhausted the bounded response queue") - .unwrap()); - let (mut responses, mut rows) = (0usize, 0usize); - while let Ok(result) = receiver.try_recv() { - let result = result.unwrap(); - assert!(result.encoded_len() <= MAX_RESULT_BYTES); - let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = - result.response_type - else { - panic!("expected Arrow response"); - }; - assert_eq!(batch.start_offset, Some(rows as i64)); - let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); - assert_eq!(reader.schema().metadata()["padding"].len(), metadata_bytes); - let decoded = reader.next().unwrap().unwrap(); - assert_eq!(decoded.num_rows() as i64, batch.row_count); - assert!(reader.next().is_none()); - let ids = decoded - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let strings = decoded - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - for index in 0..decoded.num_rows() { - assert_eq!(ids.value(index), rows as i64); - assert_eq!(strings.value(index), text); - rows += 1; - } - responses += 1; - } - assert_eq!(rows, expected_rows); - assert_eq!(offset, expected_rows as i64); - assert!( - responses <= max_responses, - "{metadata_bytes} fixed bytes caused {responses} responses" - ); - // One encode measures the fresh empty schema. Only a small - // constant of exact-envelope retries is acceptable after that. - let attempts = guard.running.encoding_attempts.load(Ordering::SeqCst); - assert!( - attempts <= responses + 2, - "{attempts} encodes for {responses} responses" - ); - } - } - - #[tokio::test] - async fn wide_buffer_padding_does_not_decay_or_repeat_failed_budgets() { - use arrow::array::{Array, ArrayRef, Int64Array, Int8Array}; - use arrow::datatypes::{DataType, Field, Schema}; - use std::sync::atomic::Ordering; - - // Int64 values cross a 64-byte IPC padding boundary at 8 -> 9 rows; - // Int8 values occupy the same padded buffer for up to 64 rows. Nullable - // columns add a separately padded validity buffer absent from an empty - // schema measurement. Share inputs to keep this regression inexpensive. - for (columns, expected_rows, narrow, nullable, max_responses) in [ - (17_293, 64usize, false, false, 18usize), - (17_293, 200usize, true, false, 10usize), - (12_000, 64usize, false, true, 18usize), - ] { - let values: ArrayRef = if narrow { - Arc::new(Int8Array::from_iter_values( - (0..expected_rows).map(|row| (row % 127) as i8), - )) - } else if nullable { - Arc::new(Int64Array::from_iter( - (0..expected_rows).map(|row| (row % 3 != 0).then_some(row as i64)), - )) - } else { - Arc::new(Int64Array::from_iter_values(0..expected_rows as i64)) - }; - let fields = (0..columns) - .map(|column| { - Field::new( - format!("c{column:05}"), - if narrow { - DataType::Int8 - } else { - DataType::Int64 - }, - nullable, - ) - }) - .collect::>(); - let batch = - RecordBatch::try_new(Arc::new(Schema::new(fields)), vec![values; columns]).unwrap(); - let budget = batch.num_rows().max(1); - let session = Arc::new(Session::new(Arc::new(RegistrySession))); - let (guard, _) = session.operation("wide_padding").unwrap(); - let (sender, mut receiver) = mpsc::channel(max_responses + 1); - let mut offset = 0; - let mut output_schema = None; - assert!(tokio::time::timeout( - std::time::Duration::from_secs(30), - send_arrow_batch( - &sender, - "session", - "wide_padding", - batch, - budget, - &mut offset, - &mut output_schema, - &guard - ) - ) - .await - .expect("wide padding exhausted the bounded response queue") - .unwrap()); - let (mut responses, mut rows) = (0usize, 0usize); - while let Ok(result) = receiver.try_recv() { - let result = result.unwrap(); - assert!(result.encoded_len() <= MAX_RESULT_BYTES); - let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = - result.response_type - else { - panic!("expected Arrow response"); - }; - assert_eq!(batch.start_offset, Some(rows as i64)); - let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); - let decoded = reader.next().unwrap().unwrap(); - assert_eq!(decoded.num_columns(), columns); - assert_eq!(decoded.num_rows() as i64, batch.row_count); - assert!(reader.next().is_none()); - for column in decoded.columns() { - if narrow { - let values = column.as_any().downcast_ref::().unwrap(); - for index in 0..decoded.num_rows() { - assert_eq!(values.value(index), ((rows + index) % 127) as i8); - } - } else { - let values = column.as_any().downcast_ref::().unwrap(); - for index in 0..decoded.num_rows() { - let row = rows + index; - assert_eq!(values.is_null(index), nullable && row % 3 == 0); - if !values.is_null(index) { - assert_eq!(values.value(index), row as i64); - } - } - } - } - rows += decoded.num_rows(); - responses += 1; - } - assert_eq!(rows, expected_rows); - assert_eq!(offset, expected_rows as i64); - assert!(responses <= max_responses, - "{columns} columns (narrow={narrow}, nullable={nullable}) caused {responses} responses"); - // One measurement plus a small number of exact retries, including - // a nullable slice that crosses the 8 -> 9 row padding boundary. - let attempts = guard.running.encoding_attempts.load(Ordering::SeqCst); - assert!(attempts <= responses + 3, - "{columns} columns (narrow={narrow}, nullable={nullable}): {attempts} encodes for {responses} responses"); - } - } - - #[tokio::test] - async fn skewed_output_checks_prefixes_and_recovers_light_suffixes() { - use arrow::array::{Int64Array, LargeStringArray}; - use arrow::datatypes::{DataType, Field, Schema}; - use std::sync::atomic::Ordering; - - for shape in ["large_tail", "large_head", "dense_middle"] { - let (prefix_rows, suffix_rows) = match shape { - "large_tail" => (5000, 0), - "large_head" => (0, 5000), - _ => (100_000, 100_000), - }; - let mut sizes = vec![0usize; prefix_rows]; - sizes.extend(std::iter::repeat_n(1024 * 1024, 40)); - sizes.extend(std::iter::repeat_n(0, suffix_rows)); - let batch = RecordBatch::try_new( - Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new("text", DataType::LargeUtf8, false), - ])), - vec![ - Arc::new(Int64Array::from_iter_values(0..sizes.len() as i64)), - Arc::new(LargeStringArray::from_iter_values( - sizes.iter().map(|size| "z".repeat(*size)), - )), - ], - ) - .unwrap(); - let budget = batch.num_rows().max(1); - let session = Arc::new(Session::new(Arc::new(RegistrySession))); - let (guard, _) = session.operation("skewed").unwrap(); - // Even an accidentally permanent one-row budget must not block - // this regression before its response-count assertion can fail. - let (sender, mut receiver) = mpsc::channel(sizes.len()); - let mut offset = 0; - let mut output_schema = None; - assert!(tokio::time::timeout( - std::time::Duration::from_secs(30), - send_arrow_batch( - &sender, - "session", - "skewed", - batch, - budget, - &mut offset, - &mut output_schema, - &guard, - ) - ) - .await - .unwrap() - .unwrap()); - let (mut responses, mut empty_responses, mut rows) = (0usize, 0usize, 0usize); - while let Ok(result) = receiver.try_recv() { - let result = result.unwrap(); - assert!(result.encoded_len() <= MAX_RESULT_BYTES); - let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = - result.response_type - else { - panic!("expected Arrow response"); - }; - assert_eq!(batch.start_offset, Some(rows as i64)); - let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); - let decoded = reader.next().unwrap().unwrap(); - assert_eq!(decoded.num_rows() as i64, batch.row_count); - assert!(reader.next().is_none()); - let ids = decoded - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let strings = decoded - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let mut empty = true; - for index in 0..decoded.num_rows() { - assert_eq!(ids.value(index), rows as i64); - let value = strings.value(index); - assert_eq!(value.len(), sizes[rows]); - assert!(value.bytes().all(|byte| byte == b'z')); - empty &= value.is_empty(); - rows += 1; - } - responses += 1; - empty_responses += usize::from(empty); - } - assert_eq!(rows, sizes.len()); - assert_eq!(offset, rows as i64); - let attempts = guard.running.encoding_attempts.load(Ordering::SeqCst); - // Logical prefix checks must avoid repeatedly materializing the - // same unconsumed dense region after successes on empty prefixes. - assert!(attempts <= responses + 4, - "{shape}: {attempts} encodes for {responses} responses repeatedly encoded an oversized region"); - if shape == "dense_middle" { - // A sticky reduced row budget also fragments a light suffix - // inside a failed piece. Let it recover once the dense rows - // are consumed, as well as across outer sibling slices. - assert!( - responses < 80, - "dense-middle output fragmented into {responses} responses" - ); - assert!( - empty_responses <= 35, - "dense-middle output emitted {empty_responses} all-empty responses" - ); - } else if shape == "large_head" { - assert!( - responses < 100, - "large-head output fragmented into {responses} responses" - ); - } else { - assert!( - responses < 80, - "large-tail output fragmented into {responses} responses" - ); - } - } - } - - #[tokio::test] - async fn existing_ipc_is_forwarded_unchanged() { - use arrow::array::Int64Array; - use arrow::datatypes::{DataType, Field, Schema}; - use arrow::ipc::writer::IpcWriteOptions; - - let batch = RecordBatch::try_new( - Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), - vec![Arc::new(Int64Array::from_iter_values(0..10))], - ) - .unwrap(); - // Legacy IPC framing differs from the frontend encoder's default, so - // byte equality also proves that the payload was not re-encoded. - let options = IpcWriteOptions::try_new(8, true, arrow::ipc::MetadataVersion::V4).unwrap(); - let mut data = Vec::new(); - let mut writer = - StreamWriter::try_new_with_options(&mut data, batch.schema().as_ref(), options) - .unwrap(); - writer.write(&batch).unwrap(); - writer.finish().unwrap(); - drop(writer); - assert_ne!(data, encode_batch(&batch).unwrap()); - let session = Arc::new(Session::new(Arc::new(RegistrySession))); - let (guard, _) = session.operation("ipc").unwrap(); - let (sender, mut receiver) = mpsc::channel(1); - let mut offset = 7; - let mut output_schema = None; - assert!(send_batch( - &sender, - "session", - "ipc", - ResultBatch::Ipc { - data: data.clone(), - row_count: 10 - }, - batch.schema().as_ref(), - &mut offset, - &mut output_schema, - &guard, - ) - .await - .unwrap()); - let result = receiver.recv().await.unwrap().unwrap(); - let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = - result.response_type - else { - panic!("expected Arrow response"); - }; - assert_eq!(&batch.data[..], data.as_slice()); - assert_eq!(batch.row_count, 10); - assert_eq!(batch.start_offset, Some(7)); - assert_eq!(offset, 17); - } - - #[tokio::test] - async fn oversized_ipc_is_split_by_rows_and_checks_its_declared_count() { - use arrow::array::Int64Array; - use arrow::datatypes::{DataType, Field, Schema}; - - let batch = RecordBatch::try_new( - Arc::new(Schema::new( - (0..10) - .map(|column| Field::new(format!("id{column}"), DataType::Int64, false)) - .collect::>(), - )), - (0..10) - .map(|column| { - Arc::new(Int64Array::from_iter_values( - (0..50_000).map(|row| row + column * 50_000), - )) as arrow::array::ArrayRef - }) - .collect(), - ) - .unwrap(); - let data = encode_batch(&batch).unwrap(); - assert!(data.len() > MAX_RESULT_BYTES); - let expected_batches = data.len().div_ceil(MAX_RESULT_BYTES / 2); - let session = Arc::new(Session::new(Arc::new(RegistrySession))); - let (guard, _) = session.operation("ipc").unwrap(); - let (sender, mut receiver) = mpsc::channel(64); - let mut offset = 0; - let mut output_schema = None; - assert!(send_batch( - &sender, - "session", - "ipc", - ResultBatch::Ipc { - data: data.clone(), - row_count: 50_000 - }, - batch.schema().as_ref(), - &mut offset, - &mut output_schema, - &guard, - ) - .await - .unwrap()); - let (mut rows, mut batches) = (0, 0); - while let Ok(result) = receiver.try_recv() { - let result = result.unwrap(); - assert!(result.encoded_len() <= MAX_RESULT_BYTES); - let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = - result.response_type - else { - panic!("expected Arrow response"); - }; - assert_eq!(batch.start_offset, Some(rows)); - let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); - let decoded = reader.next().unwrap().unwrap(); - assert_eq!(decoded.num_columns(), 10); - assert_eq!(decoded.num_rows() as i64, batch.row_count); - for (column, values) in decoded.columns().iter().enumerate() { - let ids = values.as_any().downcast_ref::().unwrap(); - let start = rows + column as i64 * 50_000; - assert!(ids - .values() - .iter() - .copied() - .eq(start..start + batch.row_count)); - } - rows += batch.row_count; - batches += 1; - assert!(reader.next().is_none()); - } - assert_eq!(rows, 50_000); - assert_eq!(offset, rows); - assert!((expected_batches..=expected_batches + 1).contains(&batches)); - for row_count in [-1, 49_999] { - let error = send_batch( - &sender, - "session", - "ipc", - ResultBatch::Ipc { - data: data.clone(), - row_count, - }, - batch.schema().as_ref(), - &mut offset, - &mut output_schema, - &guard, - ) - .await - .unwrap_err(); - assert_eq!(error.code(), tonic::Code::Internal); - assert_eq!(offset, rows); - } - assert!(receiver.try_recv().is_err()); - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cancelled_encoding_wait_retains_operation_until_blocking_task_finishes() { let session = Arc::new(Session::new(Arc::new(RegistrySession))); @@ -1944,7 +1145,8 @@ mod tests { #[tokio::test] async fn released_session_retention_is_bounded_without_evicting_active_sessions() { - let service = ConnectService::new(Arc::new(RegistryEngine)); + let service = + ConnectService::new(Arc::new(RegistryEngine), crate::ServerOptions::default()); let active_id = Uuid::new_v4().to_string(); let active = service.session(&active_id, None, None).unwrap(); @@ -2002,7 +1204,8 @@ mod tests { #[tokio::test] async fn reconnecting_release_of_live_session_allows_a_new_backend() { - let service = ConnectService::new(Arc::new(RegistryEngine)); + let service = + ConnectService::new(Arc::new(RegistryEngine), crate::ServerOptions::default()); let id = Uuid::new_v4().to_string(); let previous = service.session(&id, None, None).unwrap(); release(&service, &id, true).await; diff --git a/crates/spark-connect-server/tests/engine_injection.rs b/crates/spark-connect-server/tests/engine_injection.rs index a5797ed9..661d0450 100644 --- a/crates/spark-connect-server/tests/engine_injection.rs +++ b/crates/spark-connect-server/tests/engine_injection.rs @@ -17,12 +17,15 @@ //! Real gRPC tests of the injected backend boundary. The fake is independent of Spark execution. +#[path = "support/arrow.rs"] +mod arrow_support; #[path = "support/requests.rs"] mod requests; #[path = "support/server.rs"] mod server; use std::collections::HashMap; +use std::future::Future; use std::io::Cursor; use std::pin::Pin; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -35,6 +38,7 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::ipc::reader::StreamReader; use arrow::ipc::writer::StreamWriter; use arrow::record_batch::RecordBatch; +use arrow_support::chunking; use futures::{Stream, StreamExt}; use prost::Message; use proto::execute_plan_response::ResponseType; @@ -61,6 +65,7 @@ const PANICKING_STREAM: i64 = 7_000_000; const PANICKING_STREAM_DROP: i64 = 8_000_000; const MISMATCHED_IPC_BATCH: i64 = 9_000_000; const INVALID_IPC_BATCH: i64 = 10_000_000; +const CHUNKED_RESULT: i64 = 11_000_000; #[derive(Default)] struct Spy { @@ -80,8 +85,35 @@ struct Spy { metadata_drop_observations: Mutex>, slow_drop: Mutex)>>, stream_drop_gate: Mutex>>, + request_panic: Mutex>, } +#[derive(Clone, Copy, PartialEq, Eq)] +enum Call { + Prepare, + Analyze, + Config, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum PanicAt { + Construction, + Poll, +} + +impl Spy { + fn panic_once(&self, call: Call) -> Option { + let mut next = self.request_panic.lock().unwrap(); + if next.as_ref().is_some_and(|(target, _)| *target == call) { + next.take().map(|(_, at)| at) + } else { + None + } + } +} + +type EngineFuture<'a, T> = Pin> + Send + 'a>>; + #[derive(Default)] struct DropGate { released: Mutex, @@ -206,172 +238,208 @@ impl Drop for BlockedMetadata { } } -#[tonic::async_trait] +// Explicit async-trait signatures also exercise a panic before a future is returned. impl EngineSession for FakeSession { - async fn prepare(&self, op: proto::plan::OpType) -> Result { - self.spy.prepares.fetch_add(1, Ordering::SeqCst); - let (value, live, batch_mode) = match op { - proto::plan::OpType::Root(relation) => match relation.rel_type { - Some(proto::relation::RelType::Range(range)) => { - if range.end == BLOCKED_PREPARE { - let _dropped = BlockedPrepare(self.spy.clone()); - std::future::pending::<()>().await; - } - if range.end == -1 { - return Err(Status::invalid_argument("fake rejected the plan")); + fn prepare<'life0, 'async_trait>( + &'life0 self, + op: proto::plan::OpType, + ) -> EngineFuture<'async_trait, PreparedOperation> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + let panic = self.spy.panic_once(Call::Prepare); + assert!(panic != Some(PanicAt::Construction), "prepare construction"); + Box::pin(async move { + assert!(panic != Some(PanicAt::Poll), "prepare poll"); + self.spy.prepares.fetch_add(1, Ordering::SeqCst); + let (value, live, batch_mode) = match op { + proto::plan::OpType::Root(relation) => match relation.rel_type { + Some(proto::relation::RelType::Range(range)) => { + if range.end == BLOCKED_PREPARE { + let _dropped = BlockedPrepare(self.spy.clone()); + std::future::pending::<()>().await; + } + if range.end == -1 { + return Err(Status::invalid_argument("fake rejected the plan")); + } + // Deliberately unlike range semantics: this proves the injected + // backend, rather than the default planner, executed the request. + let batch_mode = match range.end { + MISMATCHED_BATCH => BatchMode::WrongType, + MISMATCHED_IPC_BATCH => BatchMode::WrongTypeIpc, + INVALID_IPC_BATCH => BatchMode::InvalidIpc, + ANNOTATED_BATCH => BatchMode::DifferentAnnotations, + FLOODED_RESULT => BatchMode::Flooded, + CHUNKED_RESULT => BatchMode::Chunked, + PANICKING_EXECUTION => BatchMode::PanicExecute, + PANICKING_STREAM => BatchMode::PanicStream, + PANICKING_STREAM_DROP => BatchMode::PanicDrop, + _ => BatchMode::Declared, + }; + (999, range.end == LIVE, batch_mode) } - // Deliberately unlike range semantics: this proves the injected - // backend, rather than the default planner, executed the request. - let batch_mode = match range.end { - MISMATCHED_BATCH => BatchMode::WrongType, - MISMATCHED_IPC_BATCH => BatchMode::WrongTypeIpc, - INVALID_IPC_BATCH => BatchMode::InvalidIpc, - ANNOTATED_BATCH => BatchMode::DifferentAnnotations, - FLOODED_RESULT => BatchMode::Flooded, - PANICKING_EXECUTION => BatchMode::PanicExecute, - PANICKING_STREAM => BatchMode::PanicStream, - PANICKING_STREAM_DROP => BatchMode::PanicDrop, - _ => BatchMode::Declared, - }; - (999, range.end == LIVE, batch_mode) - } - Some(proto::relation::RelType::Read(read)) => { - let Some(proto::read::ReadType::NamedTable(table)) = read.read_type else { - return Err(Status::unimplemented("fake read")); - }; - let value = self - .views - .lock() - .unwrap() - .get(&table.unparsed_identifier) - .copied() - .ok_or_else(|| Status::not_found("fake view not found"))?; - (value, false, BatchMode::Declared) - } - _ => return Err(Status::unimplemented("fake relation")), - }, - proto::plan::OpType::Command(command) => { - self.spy.commands.fetch_add(1, Ordering::SeqCst); - match command.command_type { - Some(proto::command::CommandType::CreateDataframeView(view)) => { - let Some(proto::relation::RelType::Range(input)) = - view.input.and_then(|input| input.rel_type) - else { - return Err(Status::invalid_argument("fake view requires range input")); + Some(proto::relation::RelType::Read(read)) => { + let Some(proto::read::ReadType::NamedTable(table)) = read.read_type else { + return Err(Status::unimplemented("fake read")); }; - let mut views = self.views.lock().unwrap(); - if !view.replace && views.contains_key(&view.name) { - return Err(Status::already_exists("fake view already exists")); - } - views.insert(view.name, input.end); - return Ok(PreparedOperation::Command(Vec::new())); + let value = self + .views + .lock() + .unwrap() + .get(&table.unparsed_identifier) + .copied() + .ok_or_else(|| Status::not_found("fake view not found"))?; + (value, false, BatchMode::Declared) } - Some(proto::command::CommandType::SqlCommand(command)) => { - return Ok(PreparedOperation::Command(vec![ - ResponseType::SqlCommandResult( - proto::execute_plan_response::SqlCommandResult { - relation: command.input, - }, - ), - ])); + _ => return Err(Status::unimplemented("fake relation")), + }, + proto::plan::OpType::Command(command) => { + self.spy.commands.fetch_add(1, Ordering::SeqCst); + match command.command_type { + Some(proto::command::CommandType::CreateDataframeView(view)) => { + let Some(proto::relation::RelType::Range(input)) = + view.input.and_then(|input| input.rel_type) + else { + return Err(Status::invalid_argument( + "fake view requires range input", + )); + }; + let mut views = self.views.lock().unwrap(); + if !view.replace && views.contains_key(&view.name) { + return Err(Status::already_exists("fake view already exists")); + } + views.insert(view.name, input.end); + return Ok(PreparedOperation::Command(Vec::new())); + } + Some(proto::command::CommandType::SqlCommand(command)) => { + return Ok(PreparedOperation::Command(vec![ + ResponseType::SqlCommandResult( + proto::execute_plan_response::SqlCommandResult { + relation: command.input, + }, + ), + ])); + } + _ => return Err(Status::unimplemented("fake command")), } - _ => return Err(Status::unimplemented("fake command")), } - } - _ => return Err(Status::unimplemented("fake operation")), - }; - Ok(PreparedOperation::Query(Box::new(FakeQuery { - spy: self.spy.clone(), - value, - live, - batch_mode, - }))) + _ => return Err(Status::unimplemented("fake operation")), + }; + Ok(PreparedOperation::Query(Box::new(FakeQuery { + spy: self.spy.clone(), + value, + live, + batch_mode, + }))) + }) } - async fn analyze( - &self, + fn analyze<'life0, 'async_trait>( + &'life0 self, input: proto::analyze_plan_request::Analyze, - ) -> Result { - self.spy.analyses.fetch_add(1, Ordering::SeqCst); - match input { - proto::analyze_plan_request::Analyze::Schema(_) => { - Ok(proto::analyze_plan_response::Result::Schema( - proto::analyze_plan_response::Schema { - schema: Some(proto::DataType { - kind: Some(proto::data_type::Kind::Struct(proto::data_type::Struct { - fields: vec![proto::data_type::StructField { - name: "x".repeat(3 * 1024 * 1024 + 1), - data_type: Some(proto::DataType { - kind: Some(proto::data_type::Kind::Long( - proto::data_type::Long::default(), - )), - }), - nullable: false, - metadata: None, - }], - ..Default::default() - })), - }), - }, - )) + ) -> EngineFuture<'async_trait, proto::analyze_plan_response::Result> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + let panic = self.spy.panic_once(Call::Analyze); + assert!(panic != Some(PanicAt::Construction), "analyze construction"); + Box::pin(async move { + assert!(panic != Some(PanicAt::Poll), "analyze poll"); + self.spy.analyses.fetch_add(1, Ordering::SeqCst); + match input { + proto::analyze_plan_request::Analyze::Schema(_) => { + Ok(proto::analyze_plan_response::Result::Schema( + proto::analyze_plan_response::Schema { + schema: Some(proto::DataType { + kind: Some(proto::data_type::Kind::Struct( + proto::data_type::Struct { + fields: vec![proto::data_type::StructField { + name: "x".repeat(3 * 1024 * 1024 + 1), + data_type: Some(proto::DataType { + kind: Some(proto::data_type::Kind::Long( + proto::data_type::Long::default(), + )), + }), + nullable: false, + metadata: None, + }], + ..Default::default() + }, + )), + }), + }, + )) + } + proto::analyze_plan_request::Analyze::IsStreaming(_) => { + let _dropped = BlockedMetadata { + spy: self.spy.clone(), + call: MetadataCall::Analysis, + }; + std::future::pending().await + } + proto::analyze_plan_request::Analyze::IsLocal(_) => { + Ok(proto::analyze_plan_response::Result::IsLocal( + proto::analyze_plan_response::IsLocal { is_local: true }, + )) + } + _ => Err(Status::unimplemented("fake analysis")), } - proto::analyze_plan_request::Analyze::IsStreaming(_) => { + }) + } + + fn config<'life0, 'async_trait>( + &'life0 self, + input: proto::config_request::operation::OpType, + ) -> EngineFuture<'async_trait, ConfigResult> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + let panic = self.spy.panic_once(Call::Config); + assert!(panic != Some(PanicAt::Construction), "config construction"); + Box::pin(async move { + assert!(panic != Some(PanicAt::Poll), "config poll"); + use proto::config_request::operation::OpType; + self.spy.configs.fetch_add(1, Ordering::SeqCst); + if matches!(&input, OpType::Get(get) if get.keys.iter().any(|key| key == "fake.block")) + { let _dropped = BlockedMetadata { spy: self.spy.clone(), - call: MetadataCall::Analysis, + call: MetadataCall::Config, }; - std::future::pending().await - } - proto::analyze_plan_request::Analyze::IsLocal(_) => { - Ok(proto::analyze_plan_response::Result::IsLocal( - proto::analyze_plan_response::IsLocal { is_local: true }, - )) + return std::future::pending().await; } - _ => Err(Status::unimplemented("fake analysis")), - } - } - - async fn config( - &self, - input: proto::config_request::operation::OpType, - ) -> Result { - use proto::config_request::operation::OpType; - self.spy.configs.fetch_add(1, Ordering::SeqCst); - if matches!(&input, OpType::Get(get) if get.keys.iter().any(|key| key == "fake.block")) { - let _dropped = BlockedMetadata { - spy: self.spy.clone(), - call: MetadataCall::Config, - }; - return std::future::pending().await; - } - let mut configs = self.configs.lock().unwrap(); - match input { - OpType::Set(set) => { - for pair in set.pairs { - configs.insert( - pair.key, - pair.value - .ok_or_else(|| Status::invalid_argument("fake config value"))?, - ); + let mut configs = self.configs.lock().unwrap(); + match input { + OpType::Set(set) => { + for pair in set.pairs { + configs.insert( + pair.key, + pair.value + .ok_or_else(|| Status::invalid_argument("fake config value"))?, + ); + } + Ok(ConfigResult { + pairs: Vec::new(), + warnings: vec!["handled by fake backend".into()], + }) } - Ok(ConfigResult { - pairs: Vec::new(), - warnings: vec!["handled by fake backend".into()], - }) + OpType::GetOption(get) => Ok(ConfigResult { + pairs: get + .keys + .into_iter() + .map(|key| { + let value = configs.get(&key).cloned(); + proto::KeyValue { key, value } + }) + .collect(), + warnings: Vec::new(), + }), + _ => Err(Status::unimplemented("fake config operation")), } - OpType::GetOption(get) => Ok(ConfigResult { - pairs: get - .keys - .into_iter() - .map(|key| { - let value = configs.get(&key).cloned(); - proto::KeyValue { key, value } - }) - .collect(), - warnings: Vec::new(), - }), - _ => Err(Status::unimplemented("fake config operation")), - } + }) } } @@ -389,6 +457,7 @@ enum BatchMode { InvalidIpc, DifferentAnnotations, Flooded, + Chunked, PanicExecute, PanicStream, PanicDrop, @@ -438,6 +507,10 @@ impl PreparedQuery for FakeQuery { )]))])), vec![Arc::new(Int64Array::from(vec![self.value]))], ), + BatchMode::Chunked => RecordBatch::try_new( + self.schema(), + vec![Arc::new(Int64Array::from(vec![self.value; 512 * 1024]))], + ), BatchMode::Flooded => RecordBatch::try_new( self.schema(), vec![Arc::new(Int64Array::from(vec![self.value; 64 * 1024]))], @@ -1535,3 +1608,211 @@ async fn release_all_timeout_keeps_cancellation_and_operation_ownership_until_dr assert_cancelled(&mut recovered).await; server.stop().await; } + +fn identified_request(session: &str, operation: &str) -> proto::ExecutePlanRequest { + let mut request = request(session, 1); + request.operation_id = Some(operation.into()); + request +} + +fn local_analysis(session: &str) -> proto::AnalyzePlanRequest { + proto::AnalyzePlanRequest { + session_id: session.into(), + user_context: requests::user("alice"), + analyze: Some(proto::analyze_plan_request::Analyze::IsLocal( + Default::default(), + )), + ..Default::default() + } +} + +async fn collect_identified(server: &mut Server, session: &str, operation: &str) { + let mut stream = server + .client + .execute_plan(identified_request(session, operation)) + .await + .unwrap() + .into_inner(); + let output = collect(&mut stream).await; + assert!(matches!( + output.last().unwrap().response_type, + Some(ResponseType::ResultComplete(_)) + )); +} + +#[tokio::test] +async fn noncanonical_ids_are_rejected_without_creating_or_releasing_state() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let uuid = Uuid::parse_str("abcdef12-3456-4789-abcd-0123456789ab").unwrap(); + let session = uuid.to_string(); + let operation = Uuid::new_v4().to_string(); + let original = server + .client + .config(get_config(&session)) + .await + .unwrap() + .into_inner() + .server_side_session_id; + for alias in [ + uuid.simple().to_string(), + uuid.braced().to_string(), + uuid.urn().to_string(), + session.to_uppercase(), + ] { + assert_eq!( + server + .client + .config(get_config(&alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .analyze_plan(local_analysis(&alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .execute_plan(identified_request(&alias, &operation)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_execute(release(&alias, &operation)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_session(release_session(&alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .execute_plan(identified_request(&Uuid::new_v4().to_string(), &alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_execute(release(&session, &alias)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .config(get_config(&session)) + .await + .unwrap() + .into_inner() + .server_side_session_id, + original + ); + } + assert_eq!(spy.factory_calls.lock().unwrap().len(), 1); + assert_eq!(spy.prepares.load(Ordering::SeqCst), 0); + collect_identified(&mut server, &session, &operation).await; + server.stop().await; +} + +#[tokio::test] +async fn backend_request_panics_have_internal_status_and_leave_the_session_usable() { + for call in [Call::Prepare, Call::Analyze, Call::Config] { + for at in [PanicAt::Construction, PanicAt::Poll] { + let spy = Arc::new(Spy::default()); + *spy.request_panic.lock().unwrap() = Some((call, at)); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let operation = Uuid::new_v4().to_string(); + let status = match call { + Call::Prepare => server + .client + .execute_plan(identified_request(&session, &operation)) + .await + .unwrap_err(), + Call::Analyze => server + .client + .analyze_plan(local_analysis(&session)) + .await + .unwrap_err(), + Call::Config => server + .client + .config(get_config(&session)) + .await + .unwrap_err(), + }; + assert_eq!(status.code(), Code::Internal); + assert_eq!(status.message(), "engine request panicked"); + server.client.config(get_config(&session)).await.unwrap(); + server + .client + .analyze_plan(local_analysis(&session)) + .await + .unwrap(); + // Reuse the operation ID after a preparation panic: its reservation + // must have been released before the failed RPC returned. + collect_identified(&mut server, &session, &operation).await; + assert_eq!(spy.factory_calls.lock().unwrap().len(), 1); + server.stop().await; + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn release_cancels_an_arrow_batch_between_byte_chunks() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let mut request = request(&session, CHUNKED_RESULT); + let operation = request.operation_id.clone().unwrap(); + request.request_options = vec![chunking(true, Some(1024))]; + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + let first = stream.message().await.unwrap().unwrap(); + let Some(ResponseType::ArrowBatch(chunk)) = first.response_type else { + panic!("first Arrow chunk") + }; + assert_eq!(chunk.chunk_index, Some(0)); + assert!(chunk.num_chunks_in_batch.unwrap() > 4000); + assert_eq!(chunk.row_count, 512 * 1024); + assert_eq!(chunk.start_offset, Some(0)); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_cancelled(&mut stream).await; + wait_for_count(&spy.streams_dropped, 1).await; + assert_eq!(spy.executions.load(Ordering::SeqCst), 1); + server.stop().await; +} diff --git a/crates/spark-connect-server/tests/protocol.rs b/crates/spark-connect-server/tests/protocol.rs index 38bad66f..14c637d2 100644 --- a/crates/spark-connect-server/tests/protocol.rs +++ b/crates/spark-connect-server/tests/protocol.rs @@ -17,6 +17,8 @@ //! Verify observable Spark Connect wire behavior independently of DataFrame conveniences. +#[path = "support/arrow.rs"] +mod arrow_support; #[path = "support/requests.rs"] mod requests; #[path = "support/server.rs"] @@ -26,16 +28,25 @@ mod support; use std::collections::HashSet; use std::io::Cursor; use std::sync::Arc; +use std::time::Duration; -use arrow::array::Int64Array; +use arrow::array::{ + Array, ArrayRef, BinaryViewArray, Int64Array, LargeBinaryArray, LargeListArray, + LargeStringArray, ListArray, MapArray, StringViewArray, StructArray, +}; +use arrow::buffer::OffsetBuffer; +use arrow::datatypes::{DataType, Field, Schema}; use arrow::ipc::reader::StreamReader; +use arrow::ipc::CompressionType; +use arrow::record_batch::RecordBatch; use prost::Message; use proto::execute_plan_response::ResponseType; use spark_connect_proto as proto; +use arrow_support::{assembled_batches, chunking, encoded, DEFAULT_ARROW_CHUNK_SIZE}; use requests::request as range; use server::Server; -use support::FixtureEngine; +use support::{Batch, FixtureEngine}; use tonic::Code; use uuid::Uuid; @@ -170,109 +181,168 @@ async fn observed_session_mismatch_and_released_sessions_are_rejected() { server.stop().await; } +async fn collect_result( + server: &mut Server, + request: proto::ExecutePlanRequest, +) -> Vec { + tokio::time::timeout(Duration::from_secs(30), async { + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + let mut responses = Vec::new(); + while let Some(response) = stream.message().await.unwrap() { + responses.push(response); + } + assert!(matches!( + responses.last().unwrap().response_type, + Some(ResponseType::ResultComplete(_)) + )); + responses + }) + .await + .expect("fixture execution must terminate") +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn arrow_payloads_are_byte_limited_and_oversized_rows_fail() { - use arrow::array::{LargeStringArray, StringArray, StringViewArray}; - use arrow::datatypes::{DataType, Field, Schema}; - use arrow::record_batch::RecordBatch; +async fn negotiated_chunks_preserve_plain_and_compressed_ipc_bytes_and_offsets() { + // Incompressible values keep both compression codecs above the chunk limit. + let mut random = 1u64; + let values = Int64Array::from_iter_values((0..450_000).map(|_| { + random ^= random << 13; + random ^= random >> 7; + random ^= random << 17; + random as i64 + })); + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(values)]).unwrap(); + for compression in [ + None, + Some(CompressionType::ZSTD), + Some(CompressionType::LZ4_FRAME), + ] { + let data = encoded(&batch, compression); + assert!(data.len() > 3 * 1024 * 1024); + let tail = batch.slice(0, 3); + let tail_data = encoded(&tail, compression); + let engine = FixtureEngine::default().with_results( + 42, + schema.clone(), + vec![ + Batch::Ipc(data.clone(), batch.num_rows() as i64), + Batch::Ipc(tail_data.clone(), 3), + ], + ); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + server.client = server + .client + .clone() + .max_decoding_message_size(8 * 1024 * 1024); + for option in [ + None, + Some(chunking(false, Some(1024))), + Some(chunking(true, None)), + Some(chunking(true, Some(3 * 1024 * 1024))), + ] { + let negotiated = option.as_ref().is_some_and(|option| matches!( + &option.request_option, + Some(proto::execute_plan_request::request_option::RequestOption::ResultChunkingOptions(options)) + if options.allow_arrow_batch_chunking + )); + let mut request = range(&Uuid::new_v4().to_string(), 42); + let limited = option.as_ref().is_some_and(|option| matches!( + &option.request_option, + Some(proto::execute_plan_request::request_option::RequestOption::ResultChunkingOptions(options)) + if options.allow_arrow_batch_chunking && options.preferred_arrow_chunk_size.is_some() + )); + request.request_options = option.into_iter().collect(); + let responses = collect_result(&mut server, request).await; + let output = assembled_batches(&responses, negotiated); + assert_eq!(output.len(), 2); + assert_eq!(output[0].data.as_ref(), data.as_slice()); + assert_eq!(output[1].data.as_ref(), tail_data.as_slice()); + assert_eq!(output[0].row_count, 450_000); + assert_eq!(output[1].start_offset, Some(450_000)); + for (result, expected) in output.iter().zip([&batch, &tail]) { + let mut reader = StreamReader::try_new(result.data.as_ref(), None).unwrap(); + assert_eq!(reader.next().unwrap().unwrap(), *expected); + assert!(reader.next().is_none()); + } + let chunks = responses + .iter() + .filter(|response| { + matches!(response.response_type, Some(ResponseType::ArrowBatch(_))) + }) + .count(); + assert_eq!(chunks, if limited { 3 } else { 2 }); + } + server.stop().await; + } +} - let session = Uuid::new_v4().to_string(); - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new("s", DataType::Utf8, false), - ])); - // Skewed string sizes make initial byte-ratio slices too large, forcing - // refinement while preserving every value and row offset. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn preferred_chunk_size_uses_the_protocol_range_and_splits_inside_a_large_row() { + use arrow::array::StringArray; let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int64Array::from_iter_values(0..8192)), - Arc::new(StringArray::from_iter_values( - (0..8192).map(|id| "x".repeat(if id % 64 == 0 { 200000 } else { 1000 })), - )), - ], + Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Utf8, + false, + )])), + vec![Arc::new(StringArray::from(vec![ + "x".repeat(4 * 1024 * 1024) + ]))], ) .unwrap(); - let oversized_schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)])); - let oversized = RecordBatch::try_new( - oversized_schema.clone(), - vec![Arc::new(StringArray::from(vec!["x".repeat(4000000)]))], + let engine = FixtureEngine::default().with_batches(42, batch.schema(), vec![batch.clone()]); + let mut server = Server::start_with_options( + Arc::new(engine), + spark_connect_server::ServerOptions { + max_arrow_chunk_size: 3 * 1024 * 1024, + }, ) - .unwrap(); - let engine = FixtureEngine::default() - .with_batches(2, schema, vec![batch]) - .with_batches(3, oversized_schema, vec![oversized]); - let mut server = Server::start_with_engine(Arc::new(engine)).await; - let mut stream = server - .client - .execute_plan(range(&session, 2)) - .await - .unwrap() - .into_inner(); - let (mut rows, mut batches, mut complete) = (0, 0, false); - while let Some(response) = stream.message().await.unwrap() { - assert!(response.encoded_len() <= 3 * 1024 * 1024); - match response.response_type { - Some(ResponseType::ArrowBatch(batch)) => { - assert_eq!(batch.start_offset, Some(rows)); - let records = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); - let mut decoded = 0; - for record in records { - let record = record.unwrap(); - let ids = record - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let strings = record.column(1); - for index in 0..record.num_rows() { - let id = rows + decoded + index as i64; - assert_eq!(ids.value(index), id); - let value = match strings.data_type() { - DataType::Utf8 => strings - .as_any() - .downcast_ref::() - .unwrap() - .value(index), - DataType::LargeUtf8 => strings - .as_any() - .downcast_ref::() - .unwrap() - .value(index), - DataType::Utf8View => strings - .as_any() - .downcast_ref::() - .unwrap() - .value(index), - other => panic!("unexpected string type: {other:?}"), - }; - assert_eq!(value.len(), if id % 64 == 0 { 200000 } else { 1000 }); - assert!(value.bytes().all(|byte| byte == b'x')); - } - decoded += record.num_rows() as i64; - } - assert_eq!(decoded, batch.row_count); - rows += decoded; - batches += 1; - } - Some(ResponseType::ResultComplete(_)) => complete = true, - None => {} - other => panic!("unexpected result: {other:?}"), + .await; + for preferred in [ + Some(1024), + Some(65_537), + Some(3 * 1024 * 1024), + None, + Some(0), + Some(-1), + Some(1023), + Some(3 * 1024 * 1024 + 1), + Some(i64::MAX), + ] { + let mut request = range(&Uuid::new_v4().to_string(), 42); + request.request_options = vec![chunking(true, preferred)]; + let responses = collect_result(&mut server, request).await; + let payloads: Vec<_> = responses + .iter() + .filter_map(|response| match &response.response_type { + Some(ResponseType::ArrowBatch(batch)) => Some(batch.data.len()), + _ => None, + }) + .collect(); + assert!(payloads.len() >= 2); + if let Some(size) = preferred.filter(|size| (1024..=3 * 1024 * 1024).contains(size)) { + assert!(payloads.iter().all(|len| *len <= size as usize)); + assert_eq!(payloads[0], size as usize); + } else { + assert_eq!( + payloads.len(), + 2, + "invalid preferences use the server maximum" + ); } + let output = assembled_batches(&responses, true); + assert_eq!(output.len(), 1); + assert_eq!(output[0].row_count, 1); + let mut reader = StreamReader::try_new(output[0].data.as_ref(), None).unwrap(); + assert_eq!(reader.next().unwrap().unwrap(), batch); + assert!(reader.next().is_none()); } - assert_eq!(rows, 8192); - assert!(batches > 1); - assert!(complete); - - let mut oversized = server - .client - .execute_plan(range(&session, 3)) - .await - .unwrap() - .into_inner(); - assert!(oversized.message().await.unwrap().unwrap().schema.is_some()); - let status = oversized.message().await.unwrap_err(); - assert_eq!(status.code(), Code::ResourceExhausted); server.stop().await; } @@ -374,7 +444,7 @@ async fn deeply_nested_requests_follow_the_configured_protobuf_decoder_limit() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn view_backing_buffers_are_compacted_before_splitting_including_struct_children() { +async fn view_arrays_are_materialized_once_before_byte_chunking_including_struct_children() { use arrow::array::{BinaryArray, BinaryViewArray, StringArray, StringViewArray, StructArray}; use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatch; @@ -431,67 +501,420 @@ async fn view_backing_buffers_are_compacted_before_splitting_including_struct_ch .unwrap(); let engine = FixtureEngine::default().with_batches(42, batch.schema(), vec![batch]); let mut server = Server::start_with_engine(Arc::new(engine)).await; - let request = range(&session, 42); - let mut stream = server - .client - .execute_plan(request) + let mut request = range(&session, 42); + request.request_options = vec![chunking(true, Some(3 * 1024 * 1024))]; + let responses = collect_result(&mut server, request).await; + let output = assembled_batches(&responses, true); + assert_eq!(output.len(), 1); + assert!(output[0].num_chunks_in_batch.unwrap() > 1); + let (mut rows, mut batches) = (0, 0); + for batch in output { + assert_eq!(batch.start_offset, Some(rows)); + let records = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); + let mut decoded = 0; + for record in records { + let record = record.unwrap(); + let ids = record + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(record.column(1).data_type(), &DataType::LargeUtf8); + assert_eq!(record.column(2).data_type(), &DataType::LargeBinary); + let strings = arrow::compute::cast(record.column(1), &DataType::Utf8).unwrap(); + let strings = strings.as_any().downcast_ref::().unwrap(); + let binaries = arrow::compute::cast(record.column(2), &DataType::Binary).unwrap(); + let binaries = binaries.as_any().downcast_ref::().unwrap(); + let nested = record + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(nested.column(0).data_type(), &DataType::LargeUtf8); + let nested_strings = arrow::compute::cast(nested.column(0), &DataType::Utf8).unwrap(); + let nested_strings = nested_strings + .as_any() + .downcast_ref::() + .unwrap(); + for index in 0..record.num_rows() { + let id = rows + decoded + index as i64; + assert_eq!(ids.value(index), id); + assert_eq!(strings.value(index), text(id)); + assert_eq!(binaries.value(index), binary(id)); + assert_eq!(nested_strings.value(index), nested_text(id)); + } + decoded += record.num_rows() as i64; + } + assert_eq!(decoded, batch.row_count); + rows += decoded; + batches += 1; + } + assert_eq!(rows, 4096); + assert_eq!(batches, 1); + server.stop().await; +} + +fn integers(rows: usize) -> RecordBatch { + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), + vec![Arc::new(Int64Array::from_iter_values(0..rows as i64))], + ) + .unwrap() +} + +#[tokio::test] +async fn repeated_chunk_options_enable_if_any_and_use_the_first_preference() { + let batch = integers(3000); + let engine = FixtureEngine::default().with_batches(42, batch.schema(), vec![batch.clone()]); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + for (options, size) in [ + ( + vec![chunking(false, Some(1024)), chunking(true, None)], + 1024, + ), + ( + vec![chunking(false, None), chunking(true, Some(2048))], + 2048, + ), + ( + vec![chunking(true, Some(1023)), chunking(true, Some(1024))], + DEFAULT_ARROW_CHUNK_SIZE, + ), + ] { + let mut request = range(&Uuid::new_v4().to_string(), 42); + request.request_options = options; + let responses = collect_result(&mut server, request).await; + let payloads: Vec<_> = responses + .iter() + .filter_map(|response| match &response.response_type { + Some(ResponseType::ArrowBatch(batch)) => Some(batch.data.len()), + _ => None, + }) + .collect(); + if size < DEFAULT_ARROW_CHUNK_SIZE { + assert_eq!(payloads[0], size); + } else { + assert_eq!(payloads.len(), 1); + } + assert!(payloads.iter().all(|len| *len <= size)); + let output = assembled_batches(&responses, true); + assert_eq!(output.len(), 1); + let mut reader = StreamReader::try_new(output[0].data.as_ref(), None).unwrap(); + assert_eq!(reader.next().unwrap().unwrap(), batch); + } + server.stop().await; +} + +async fn collect_small_slices(backing: RecordBatch) -> Vec { + let batches = (0..20) + .map(|index| backing.slice(index * 1000, 1000)) + .collect(); + let engine = FixtureEngine::default().with_batches(42, backing.schema(), batches); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let output = collect_result(&mut server, range(&Uuid::new_v4().to_string(), 42)).await; + let (mut batches, mut rows) = (Vec::new(), 0); + for message in output { + assert!(message.encoded_len() <= 3 * 1024 * 1024); + if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + message.response_type + { + assert_eq!(batch.start_offset, Some(rows)); + assert_eq!(batch.row_count, 1000); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + let decoded = reader.next().unwrap().unwrap(); + assert_eq!(decoded.num_rows() as i64, batch.row_count); + batches.push(decoded); + assert!(reader.next().is_none()); + rows += batch.row_count; + } + } + assert_eq!(batches.len(), 20); + assert_eq!(rows, 20_000); + server.stop().await; + batches +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn slices_of_a_large_allocation_remain_one_response_per_small_batch() { + for (index, batch) in collect_small_slices(integers(10_000_000)) .await - .unwrap() - .into_inner(); - let (mut rows, mut batches, mut complete) = (0, 0, false); - while let Some(response) = stream.message().await.unwrap() { - assert!(response.encoded_len() <= 3 * 1024 * 1024); - match response.response_type { - Some(ResponseType::ArrowBatch(batch)) => { - assert_eq!(batch.start_offset, Some(rows)); - let records = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); - let mut decoded = 0; - for record in records { - let record = record.unwrap(); - let ids = record + .iter() + .enumerate() + { + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let start = index as i64 * 1000; + assert!(values.values().iter().copied().eq(start..start + 1000)); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn nested_slices_preserve_referenced_child_values() { + // The parent needs only 20,000 rows to retain the full 80 MB child. Exercise + // each offset width and maps sequentially, sharing the same allocation. + let child: ArrayRef = Arc::new(Int64Array::from_iter_values(0..10_000_000)); + for kind in 0..3 { + let item = Arc::new(Field::new("item", DataType::Int64, false)); + let column: ArrayRef = match kind { + 0 => Arc::new(ListArray::new( + item, + OffsetBuffer::new((0..=20_000i32).collect::>().into()), + child.clone(), + None, + )), + 1 => Arc::new(LargeListArray::new( + item, + OffsetBuffer::new((0..=20_000i64).collect::>().into()), + child.clone(), + None, + )), + _ => { + let fields = vec![ + Field::new("key", DataType::Int64, false), + Field::new("value", DataType::Int64, false), + ]; + let entries = StructArray::new( + fields.clone().into(), + vec![child.clone(), child.clone()], + None, + ); + Arc::new(MapArray::new( + Arc::new(Field::new( + "entries", + DataType::Struct(fields.into()), + false, + )), + OffsetBuffer::new((0..=20_000i32).collect::>().into()), + entries, + None, + false, + )) + } + }; + let schema = Arc::new(Schema::new(vec![Field::new( + "nested", + column.data_type().clone(), + false, + )])); + let backing = RecordBatch::try_new(schema, vec![column]).unwrap(); + let batches = collect_small_slices(backing).await; + let mut row = 0i64; + for batch in batches { + for index in 0..batch.num_rows() { + let value = match kind { + 0 => batch .column(0) .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(record.column(1).data_type(), &DataType::LargeUtf8); - assert_eq!(record.column(2).data_type(), &DataType::LargeBinary); - let strings = arrow::compute::cast(record.column(1), &DataType::Utf8).unwrap(); - let strings = strings.as_any().downcast_ref::().unwrap(); - let binaries = - arrow::compute::cast(record.column(2), &DataType::Binary).unwrap(); - let binaries = binaries.as_any().downcast_ref::().unwrap(); - let nested = record - .column(3) + .downcast_ref::() + .unwrap() + .value(index), + 1 => batch + .column(0) .as_any() - .downcast_ref::() + .downcast_ref::() + .unwrap() + .value(index), + _ => { + let entry = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(index); + assert_eq!(entry.len(), 1); + let key = entry + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(key.value(0), row); + entry.column(1).clone() + } + }; + let value = value.as_any().downcast_ref::().unwrap(); + assert_eq!(value.len(), 1); + assert_eq!(value.value(0), row); + row += 1; + } + } + assert_eq!(row, 20_000); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_slices_preserve_current_values() { + fn text(row: usize) -> String { + format!("{row:010}{}", "y".repeat(30)) + } + + let values = StringViewArray::from_iter_values((0..1_000_000).map(text)); + let (views, buffers, nulls) = values.into_parts(); + for binary in [false, true] { + let column: ArrayRef = if binary { + Arc::new(BinaryViewArray::new( + views.clone(), + buffers.clone(), + nulls.clone(), + )) + } else { + Arc::new(StringViewArray::new( + views.clone(), + buffers.clone(), + nulls.clone(), + )) + }; + let schema = Arc::new(Schema::new(vec![Field::new( + "view", + column.data_type().clone(), + false, + )])); + let backing = RecordBatch::try_new(schema, vec![column]).unwrap(); + let batches = collect_small_slices(backing).await; + let mut row = 0; + for batch in batches { + for index in 0..batch.num_rows() { + let expected = text(row); + if binary { + let values = batch + .column(0) + .as_any() + .downcast_ref::() .unwrap(); - assert_eq!(nested.column(0).data_type(), &DataType::LargeUtf8); - let nested_strings = - arrow::compute::cast(nested.column(0), &DataType::Utf8).unwrap(); - let nested_strings = nested_strings + assert_eq!(values.value(index), expected.as_bytes()); + } else { + let values = batch + .column(0) .as_any() - .downcast_ref::() + .downcast_ref::() .unwrap(); - for index in 0..record.num_rows() { - let id = rows + decoded + index as i64; - assert_eq!(ids.value(index), id); - assert_eq!(strings.value(index), text(id)); - assert_eq!(binaries.value(index), binary(id)); - assert_eq!(nested_strings.value(index), nested_text(id)); - } - decoded += record.num_rows() as i64; + assert_eq!(values.value(index), expected); } - assert_eq!(decoded, batch.row_count); - rows += decoded; - batches += 1; + row += 1; } - Some(ResponseType::ResultComplete(_)) => complete = true, - None => {} - other => panic!("unexpected result: {other:?}"), } + assert_eq!(row, 20_000); + } +} + +#[cfg(target_pointer_width = "64")] +#[tokio::test] +async fn unrepresentable_engine_batches_preserve_preceding_results_and_fail_cleanly() { + use arrow::array::NullArray; + use arrow::record_batch::RecordBatchOptions; + + let rows = i64::MAX as usize + 1; + for null_column in [false, true] { + let oversized = if null_column { + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("nulls", DataType::Null, true)])), + vec![Arc::new(NullArray::new(rows))], + ) + .unwrap() + } else { + RecordBatch::try_new_with_options( + Arc::new(Schema::empty()), + vec![], + &RecordBatchOptions::new().with_row_count(Some(rows)), + ) + .unwrap() + }; + let engine = FixtureEngine::default().with_batches( + 42, + oversized.schema(), + vec![oversized.slice(0, 3), oversized], + ); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let mut stream = server + .client + .execute_plan(range(&Uuid::new_v4().to_string(), 42)) + .await + .unwrap() + .into_inner(); + let (status, counts) = tokio::time::timeout(Duration::from_secs(15), async { + let mut counts = Vec::new(); + loop { + match stream.message().await { + Ok(Some(message)) => match message.response_type { + Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) => { + assert_eq!(batch.start_offset, Some(0)); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + assert_eq!(reader.next().unwrap().unwrap().num_rows(), 3); + assert!(reader.next().is_none()); + counts.push(batch.row_count); + } + Some(proto::execute_plan_response::ResponseType::ResultComplete(_)) => { + panic!("unrepresentable output must not complete successfully"); + } + _ => {} + }, + Err(status) => break (status, counts), + Ok(None) => panic!("expected row-count overflow status"), + } + } + }) + .await + .unwrap(); + assert_eq!(status.code(), Code::ResourceExhausted); + assert_eq!(status.message(), "result batch row count exceeds i64"); + assert_eq!(counts, vec![3]); + server.stop().await; + } +} + +#[tokio::test] +async fn configured_maximum_bounds_payloads_and_zero_disables_chunking() { + assert_eq!( + spark_connect_server::ServerOptions::default().max_arrow_chunk_size, + DEFAULT_ARROW_CHUNK_SIZE + ); + let batch = integers(5000); + let bytes = encoded(&batch, None); + for maximum in [4096, 0] { + let engine = FixtureEngine::default().with_results( + 42, + batch.schema(), + vec![Batch::Ipc(bytes.clone(), 5000)], + ); + let mut server = Server::start_with_options( + Arc::new(engine), + spark_connect_server::ServerOptions { + max_arrow_chunk_size: maximum, + }, + ) + .await; + for preferred in [None, Some(2048), Some(4096), Some(4097)] { + let mut request = range(&Uuid::new_v4().to_string(), 42); + request.request_options = vec![chunking(true, preferred)]; + let responses = collect_result(&mut server, request).await; + let output = assembled_batches(&responses, maximum != 0); + assert_eq!(output.len(), 1); + assert_eq!(output[0].data.as_ref(), bytes.as_slice()); + let chunks: Vec<_> = responses + .iter() + .filter_map(|response| match &response.response_type { + Some(ResponseType::ArrowBatch(batch)) => { + Some((response.encoded_len(), batch.data.len())) + } + _ => None, + }) + .collect(); + if maximum == 0 { + assert_eq!(chunks.len(), 1); + } else { + let limit = preferred + .filter(|size| (1024..=maximum as i64).contains(size)) + .unwrap_or(maximum as i64) as usize; + assert!(chunks.iter().all(|(_, payload)| *payload <= limit)); + assert_eq!(chunks[0].1, limit); + assert!( + chunks[0].0 > limit, + "the chunk limit excludes the protobuf envelope" + ); + } + } + server.stop().await; } - assert_eq!(rows, 4096); - assert!(batches > 1); - assert!(complete); - server.stop().await; } diff --git a/crates/spark-connect-server/tests/review_regressions.rs b/crates/spark-connect-server/tests/review_regressions.rs deleted file mode 100644 index 43f46c99..00000000 --- a/crates/spark-connect-server/tests/review_regressions.rs +++ /dev/null @@ -1,652 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Regressions for sliced output, protocol IDs and backend panic boundaries. - -#[path = "support/server.rs"] -mod server; - -use std::future::Future; -use std::pin::Pin; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::Arc; -use std::time::Duration; - -use arrow::array::{ - Array, ArrayRef, BinaryViewArray, Int64Array, LargeBinaryArray, LargeListArray, - LargeStringArray, ListArray, MapArray, StringViewArray, StructArray, -}; -use arrow::buffer::OffsetBuffer; -use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; -use arrow::ipc::reader::StreamReader; -use arrow::record_batch::RecordBatch; -use prost::Message; -use server::Server; -use spark_connect_proto as proto; -use spark_connect_server::engine::{ - BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, - ResultBatch, SessionContext, -}; -use tonic::{Code, Status}; -use uuid::Uuid; - -#[derive(Clone, Copy, PartialEq, Eq)] -enum Call { - Prepare, - Analyze, - Config, -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum PanicAt { - Construction, - Poll, -} - -struct State { - batches: Vec, - panic: Option<(Call, PanicAt)>, - panicked: AtomicBool, - sessions: AtomicUsize, - prepares: AtomicUsize, -} - -impl State { - fn new(batches: Vec, panic: Option<(Call, PanicAt)>) -> Arc { - Arc::new(Self { - batches, - panic, - panicked: AtomicBool::new(false), - sessions: AtomicUsize::new(0), - prepares: AtomicUsize::new(0), - }) - } - - fn panic_once(&self, call: Call) -> Option { - self.panic - .filter(|(target, _)| *target == call) - .filter(|_| !self.panicked.swap(true, Ordering::SeqCst)) - .map(|(_, at)| at) - } -} - -struct Fixture(Arc); - -impl Engine for Fixture { - fn create_session(&self, _: SessionContext) -> Result, Status> { - self.0.sessions.fetch_add(1, Ordering::SeqCst); - Ok(Arc::new(Self(self.0.clone()))) - } -} - -type EngineFuture<'a, T> = Pin> + Send + 'a>>; - -// Implement the expanded async-trait signatures directly so the fixture can -// panic both before returning a future and while that future is polled. -impl EngineSession for Fixture { - fn prepare<'life0, 'async_trait>( - &'life0 self, - _: proto::plan::OpType, - ) -> EngineFuture<'async_trait, PreparedOperation> - where - 'life0: 'async_trait, - Self: 'async_trait, - { - self.0.prepares.fetch_add(1, Ordering::SeqCst); - let panic = self.0.panic_once(Call::Prepare); - assert!(panic != Some(PanicAt::Construction), "prepare construction"); - Box::pin(async move { - assert!(panic != Some(PanicAt::Poll), "prepare poll"); - Ok(PreparedOperation::Query(Box::new(Query( - self.0.batches.clone(), - )))) - }) - } - - fn analyze<'life0, 'async_trait>( - &'life0 self, - _: proto::analyze_plan_request::Analyze, - ) -> EngineFuture<'async_trait, proto::analyze_plan_response::Result> - where - 'life0: 'async_trait, - Self: 'async_trait, - { - let panic = self.0.panic_once(Call::Analyze); - assert!(panic != Some(PanicAt::Construction), "analyze construction"); - Box::pin(async move { - assert!(panic != Some(PanicAt::Poll), "analyze poll"); - Ok(proto::analyze_plan_response::Result::SparkVersion( - proto::analyze_plan_response::SparkVersion { - version: "fixture".into(), - }, - )) - }) - } - - fn config<'life0, 'async_trait>( - &'life0 self, - _: proto::config_request::operation::OpType, - ) -> EngineFuture<'async_trait, ConfigResult> - where - 'life0: 'async_trait, - Self: 'async_trait, - { - let panic = self.0.panic_once(Call::Config); - assert!(panic != Some(PanicAt::Construction), "config construction"); - Box::pin(async move { - assert!(panic != Some(PanicAt::Poll), "config poll"); - Ok(ConfigResult::default()) - }) - } -} - -struct Query(Vec); - -#[tonic::async_trait] -impl PreparedQuery for Query { - fn schema(&self) -> SchemaRef { - self.0[0].schema() - } - - async fn execute(self: Box) -> Result { - Ok(Box::pin(futures::stream::iter( - self.0 - .into_iter() - .map(|batch| Ok(ResultBatch::Arrow(batch))), - ))) - } -} - -fn integers(rows: usize) -> RecordBatch { - RecordBatch::try_new( - Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), - vec![Arc::new(Int64Array::from_iter_values(0..rows as i64))], - ) - .unwrap() -} - -fn execute(session: &str, operation: &str) -> proto::ExecutePlanRequest { - proto::ExecutePlanRequest { - session_id: session.into(), - operation_id: Some(operation.into()), - plan: Some(proto::Plan { - op_type: Some(proto::plan::OpType::Root(proto::Relation::default())), - }), - ..Default::default() - } -} - -fn config(session: &str) -> proto::ConfigRequest { - proto::ConfigRequest { - session_id: session.into(), - operation: Some(proto::config_request::Operation { - op_type: Some(proto::config_request::operation::OpType::GetAll( - Default::default(), - )), - }), - ..Default::default() - } -} - -fn analyze(session: &str) -> proto::AnalyzePlanRequest { - proto::AnalyzePlanRequest { - session_id: session.into(), - analyze: Some(proto::analyze_plan_request::Analyze::SparkVersion( - Default::default(), - )), - ..Default::default() - } -} - -fn release(session: &str, operation: &str) -> proto::ReleaseExecuteRequest { - proto::ReleaseExecuteRequest { - session_id: session.into(), - operation_id: operation.into(), - release: Some(proto::release_execute_request::Release::ReleaseAll( - Default::default(), - )), - ..Default::default() - } -} - -async fn collect( - server: &mut Server, - session: &str, - operation: &str, -) -> Vec { - tokio::time::timeout(Duration::from_secs(15), async { - let mut stream = server - .client - .execute_plan(execute(session, operation)) - .await - .unwrap() - .into_inner(); - let mut output = Vec::new(); - while let Some(message) = stream.message().await.unwrap() { - output.push(message); - } - assert!(matches!( - output.last().unwrap().response_type.as_ref(), - Some(proto::execute_plan_response::ResponseType::ResultComplete( - _ - )) - )); - output - }) - .await - .unwrap() -} - -async fn collect_small_slices(backing: RecordBatch) -> Vec { - let batches = (0..20) - .map(|index| backing.slice(index * 1000, 1000)) - .collect(); - let state = State::new(batches, None); - let mut server = Server::start_with_engine(Arc::new(Fixture(state))).await; - let output = collect( - &mut server, - &Uuid::new_v4().to_string(), - &Uuid::new_v4().to_string(), - ) - .await; - let (mut batches, mut rows) = (Vec::new(), 0); - for message in output { - assert!(message.encoded_len() <= 3 * 1024 * 1024); - if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = - message.response_type - { - assert_eq!(batch.start_offset, Some(rows)); - assert_eq!(batch.row_count, 1000); - let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); - let decoded = reader.next().unwrap().unwrap(); - assert_eq!(decoded.num_rows() as i64, batch.row_count); - batches.push(decoded); - assert!(reader.next().is_none()); - rows += batch.row_count; - } - } - assert_eq!(batches.len(), 20); - assert_eq!(rows, 20_000); - server.stop().await; - batches -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn slices_of_a_large_allocation_remain_one_response_per_small_batch() { - for (index, batch) in collect_small_slices(integers(10_000_000)) - .await - .iter() - .enumerate() - { - let values = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let start = index as i64 * 1000; - assert!(values.values().iter().copied().eq(start..start + 1000)); - } -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn nested_slices_only_budget_the_referenced_child_range() { - // The parent needs only 20,000 rows to retain the full 80 MB child. Exercise - // each offset width and maps sequentially, sharing the same allocation. - let child: ArrayRef = Arc::new(Int64Array::from_iter_values(0..10_000_000)); - for kind in 0..3 { - let item = Arc::new(Field::new("item", DataType::Int64, false)); - let column: ArrayRef = match kind { - 0 => Arc::new(ListArray::new( - item, - OffsetBuffer::new((0..=20_000i32).collect::>().into()), - child.clone(), - None, - )), - 1 => Arc::new(LargeListArray::new( - item, - OffsetBuffer::new((0..=20_000i64).collect::>().into()), - child.clone(), - None, - )), - _ => { - let fields = vec![ - Field::new("key", DataType::Int64, false), - Field::new("value", DataType::Int64, false), - ]; - let entries = StructArray::new( - fields.clone().into(), - vec![child.clone(), child.clone()], - None, - ); - Arc::new(MapArray::new( - Arc::new(Field::new( - "entries", - DataType::Struct(fields.into()), - false, - )), - OffsetBuffer::new((0..=20_000i32).collect::>().into()), - entries, - None, - false, - )) - } - }; - let schema = Arc::new(Schema::new(vec![Field::new( - "nested", - column.data_type().clone(), - false, - )])); - let backing = RecordBatch::try_new(schema, vec![column]).unwrap(); - let batches = collect_small_slices(backing).await; - let mut row = 0i64; - for batch in batches { - for index in 0..batch.num_rows() { - let value = match kind { - 0 => batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap() - .value(index), - 1 => batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap() - .value(index), - _ => { - let entry = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap() - .value(index); - assert_eq!(entry.len(), 1); - let key = entry - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(key.value(0), row); - entry.column(1).clone() - } - }; - let value = value.as_any().downcast_ref::().unwrap(); - assert_eq!(value.len(), 1); - assert_eq!(value.value(0), row); - row += 1; - } - } - assert_eq!(row, 20_000); - } -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn view_slices_only_budget_the_current_values() { - fn text(row: usize) -> String { - format!("{row:010}{}", "y".repeat(30)) - } - - let values = StringViewArray::from_iter_values((0..1_000_000).map(text)); - let (views, buffers, nulls) = values.into_parts(); - for binary in [false, true] { - let column: ArrayRef = if binary { - Arc::new(BinaryViewArray::new( - views.clone(), - buffers.clone(), - nulls.clone(), - )) - } else { - Arc::new(StringViewArray::new( - views.clone(), - buffers.clone(), - nulls.clone(), - )) - }; - let schema = Arc::new(Schema::new(vec![Field::new( - "view", - column.data_type().clone(), - false, - )])); - let backing = RecordBatch::try_new(schema, vec![column]).unwrap(); - let batches = collect_small_slices(backing).await; - let mut row = 0; - for batch in batches { - for index in 0..batch.num_rows() { - let expected = text(row); - if binary { - let values = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(values.value(index), expected.as_bytes()); - } else { - let values = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(values.value(index), expected); - } - row += 1; - } - } - assert_eq!(row, 20_000); - } -} - -#[cfg(target_pointer_width = "64")] -#[tokio::test] -async fn unrepresentable_engine_batches_preserve_preceding_results_and_fail_cleanly() { - use arrow::array::NullArray; - use arrow::record_batch::RecordBatchOptions; - - let rows = i64::MAX as usize + 1; - for null_column in [false, true] { - let oversized = if null_column { - RecordBatch::try_new( - Arc::new(Schema::new(vec![Field::new("nulls", DataType::Null, true)])), - vec![Arc::new(NullArray::new(rows))], - ) - .unwrap() - } else { - RecordBatch::try_new_with_options( - Arc::new(Schema::empty()), - vec![], - &RecordBatchOptions::new().with_row_count(Some(rows)), - ) - .unwrap() - }; - let state = State::new(vec![oversized.slice(0, 3), oversized], None); - let mut server = Server::start_with_engine(Arc::new(Fixture(state))).await; - let mut stream = server - .client - .execute_plan(execute( - &Uuid::new_v4().to_string(), - &Uuid::new_v4().to_string(), - )) - .await - .unwrap() - .into_inner(); - let (status, counts) = tokio::time::timeout(Duration::from_secs(15), async { - let mut counts = Vec::new(); - loop { - match stream.message().await { - Ok(Some(message)) => match message.response_type { - Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) => { - assert_eq!(batch.start_offset, Some(0)); - let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); - assert_eq!(reader.next().unwrap().unwrap().num_rows(), 3); - assert!(reader.next().is_none()); - counts.push(batch.row_count); - } - Some(proto::execute_plan_response::ResponseType::ResultComplete(_)) => { - panic!("unrepresentable output must not complete successfully"); - } - _ => {} - }, - Err(status) => break (status, counts), - Ok(None) => panic!("expected row-count overflow status"), - } - } - }) - .await - .unwrap(); - assert_eq!(status.code(), Code::ResourceExhausted); - assert_eq!(status.message(), "result batch row count exceeds i64"); - assert_eq!(counts, vec![3]); - server.stop().await; - } -} - -#[tokio::test] -async fn noncanonical_ids_are_rejected_without_creating_or_releasing_state() { - let state = State::new(vec![integers(1)], None); - let mut server = Server::start_with_engine(Arc::new(Fixture(state.clone()))).await; - let uuid = Uuid::parse_str("abcdef12-3456-4789-abcd-0123456789ab").unwrap(); - let session = uuid.to_string(); - let operation = Uuid::new_v4().to_string(); - let original = server - .client - .config(config(&session)) - .await - .unwrap() - .into_inner() - .server_side_session_id; - for alias in [ - uuid.simple().to_string(), - uuid.braced().to_string(), - uuid.urn().to_string(), - session.to_uppercase(), - ] { - assert_eq!( - server - .client - .config(config(&alias)) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - assert_eq!( - server - .client - .analyze_plan(analyze(&alias)) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - assert_eq!( - server - .client - .execute_plan(execute(&alias, &operation)) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - assert_eq!( - server - .client - .release_execute(release(&alias, &operation)) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - assert_eq!( - server - .client - .release_session(proto::ReleaseSessionRequest { - session_id: alias.clone(), - ..Default::default() - }) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - assert_eq!( - server - .client - .execute_plan(execute(&Uuid::new_v4().to_string(), &alias)) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - assert_eq!( - server - .client - .release_execute(release(&session, &alias)) - .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - assert_eq!( - server - .client - .config(config(&session)) - .await - .unwrap() - .into_inner() - .server_side_session_id, - original - ); - } - assert_eq!(state.sessions.load(Ordering::SeqCst), 1); - assert_eq!(state.prepares.load(Ordering::SeqCst), 0); - collect(&mut server, &session, &operation).await; - server.stop().await; -} - -#[tokio::test] -async fn backend_request_panics_have_internal_status_and_leave_the_session_usable() { - for call in [Call::Prepare, Call::Analyze, Call::Config] { - for at in [PanicAt::Construction, PanicAt::Poll] { - let state = State::new(vec![integers(1)], Some((call, at))); - let mut server = Server::start_with_engine(Arc::new(Fixture(state.clone()))).await; - let session = Uuid::new_v4().to_string(); - let operation = Uuid::new_v4().to_string(); - let status = match call { - Call::Prepare => server - .client - .execute_plan(execute(&session, &operation)) - .await - .unwrap_err(), - Call::Analyze => server - .client - .analyze_plan(analyze(&session)) - .await - .unwrap_err(), - Call::Config => server.client.config(config(&session)).await.unwrap_err(), - }; - assert_eq!(status.code(), Code::Internal); - assert_eq!(status.message(), "engine request panicked"); - server.client.config(config(&session)).await.unwrap(); - server.client.analyze_plan(analyze(&session)).await.unwrap(); - // Reuse the operation ID after a preparation panic: its reservation - // must have been released before the failed RPC returned. - collect(&mut server, &session, &operation).await; - assert_eq!(state.sessions.load(Ordering::SeqCst), 1); - server.stop().await; - } - } -} diff --git a/crates/spark-connect-server/tests/schema_consistency.rs b/crates/spark-connect-server/tests/schema_consistency.rs index 24f9bde5..82ed77bd 100644 --- a/crates/spark-connect-server/tests/schema_consistency.rs +++ b/crates/spark-connect-server/tests/schema_consistency.rs @@ -17,6 +17,8 @@ //! Result schemas remain consumable as one Arrow table across output batches. +#[path = "support/arrow.rs"] +mod arrow_support; #[path = "support/requests.rs"] mod requests; #[path = "support/server.rs"] @@ -41,6 +43,7 @@ use spark_connect_proto as proto; use tonic::{Code, Status}; use uuid::Uuid; +use arrow_support::{assembled_batches, chunking}; use server::Server; use support::{Batch, FixtureEngine}; @@ -64,7 +67,8 @@ fn ipc(batch: &RecordBatch) -> Vec { async fn execute(engine: BatchCase) -> (Vec, Option) { let engine = FixtureEngine::default().with_results(1, engine.schema, engine.batches); let mut server = Server::start_with_engine(Arc::new(engine)).await; - let request = requests::request(&Uuid::new_v4().to_string(), 1); + let mut request = requests::request(&Uuid::new_v4().to_string(), 1); + request.request_options = vec![chunking(true, Some(RESULT_LIMIT as i64))]; let mut stream = server .client .execute_plan(request) @@ -76,7 +80,14 @@ async fn execute(engine: BatchCase) -> (Vec, Option< loop { match stream.message().await { Ok(Some(response)) => { - assert!(response.encoded_len() <= RESULT_LIMIT); + if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + &response.response_type + { + assert!(batch.data.len() <= RESULT_LIMIT); + assert!(response.encoded_len() <= batch.data.len() + 512); + } else { + assert!(response.encoded_len() <= RESULT_LIMIT); + } responses.push(response); } Ok(None) => break (responses, None), @@ -91,22 +102,16 @@ async fn execute(engine: BatchCase) -> (Vec, Option< } fn output_batches(responses: &[proto::ExecutePlanResponse]) -> Vec { - let mut offset = 0; - let mut batches = Vec::new(); - for response in responses { - if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = - &response.response_type - { - assert_eq!(batch.start_offset, Some(offset)); - let mut reader = StreamReader::try_new(Cursor::new(&batch.data), None).unwrap(); + assembled_batches(responses, true) + .into_iter() + .map(|batch| { + let mut reader = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); let decoded = reader.next().unwrap().unwrap(); assert_eq!(decoded.num_rows() as i64, batch.row_count); assert!(reader.next().is_none()); - offset += batch.row_count; - batches.push(decoded); - } - } - batches + decoded + }) + .collect() } fn assert_schema_error(responses: &[proto::ExecutePlanResponse], status: Option) { @@ -234,7 +239,7 @@ fn views(rows: usize) -> RecordBatch { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn view_ipc_is_unchanged_when_small_and_materialized_when_oversized() { +async fn view_ipc_remains_unchanged_for_small_and_oversized_batches() { for rows in [2, LARGE_ROWS] { let batch = views(rows); let data = ipc(&batch); @@ -246,78 +251,44 @@ async fn view_ipc_is_unchanged_when_small_and_materialized_when_oversized() { .await; assert!(status.is_none(), "{status:?}"); assert!(completed(&responses)); + let output = assembled_batches(&responses, true); + assert_eq!(output.len(), 1); + assert_eq!(output[0].data.as_ref(), data.as_slice()); let batches = output_batches(&responses); - if rows == 2 { - let payload = responses - .iter() - .find_map(|response| match &response.response_type { - Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) => { - Some(&batch.data) - } - _ => None, - }) - .unwrap(); - assert_eq!(payload.as_ref(), data.as_slice()); - assert_eq!( - batches[0].schema().field(0).data_type(), - &DataType::Utf8View - ); - } else { - assert!(batches.len() > 1); - let mut offset = 0; - for batch in batches { - assert_eq!(batch.schema().field(0).data_type(), &DataType::LargeUtf8); - let values = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - for row in 0..batch.num_rows() { - assert_eq!(values.value(row), text(offset + row)); - } - offset += batch.num_rows(); - } - assert_eq!(offset, rows); - } + assert_eq!(batches, vec![batch]); + assert_eq!( + batches[0].schema().field(0).data_type(), + &DataType::Utf8View + ); } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn mixing_small_and_oversized_view_ipc_rejects_the_changed_output_schema() { +async fn mixing_small_and_oversized_view_ipc_preserves_one_output_schema() { let small = views(2); let large = views(LARGE_ROWS); - let small = Batch::Ipc(ipc(&small), 2); - let large = Batch::Ipc(ipc(&large), LARGE_ROWS as i64); for large_first in [false, true] { + let expected = if large_first { + vec![large.clone(), small.clone()] + } else { + vec![small.clone(), large.clone()] + }; let (responses, status) = execute(BatchCase { - schema: views(0).schema(), - batches: if large_first { - vec![large.clone(), small.clone()] - } else { - vec![small.clone(), large.clone()] - }, + schema: small.schema(), + batches: expected + .iter() + .map(|batch| Batch::Ipc(ipc(batch), batch.num_rows() as i64)) + .collect(), }) .await; - assert_schema_error(&responses, status); - let batches = output_batches(&responses); - let expected_type = if large_first { - DataType::LargeUtf8 - } else { - DataType::Utf8View - }; - let expected_rows = if large_first { LARGE_ROWS } else { 2 }; - assert_eq!( - batches.iter().map(RecordBatch::num_rows).sum::(), - expected_rows - ); - for batch in batches { - assert_eq!(batch.schema().field(0).data_type(), &expected_type); - } + assert!(status.is_none(), "{status:?}"); + assert!(completed(&responses)); + assert_eq!(output_batches(&responses), expected); } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn view_children_in_lists_large_lists_and_maps_are_materialized_when_split() { +async fn arrow_view_children_are_materialized_before_byte_chunking() { let values: ArrayRef = Arc::new(StringViewArray::from_iter_values((0..LARGE_ROWS).map(text))); let offsets = OffsetBuffer::new((0..=LARGE_ROWS as i32).collect::>().into()); let item = Arc::new(Field::new("item", DataType::Utf8View, false)); @@ -368,7 +339,13 @@ async fn view_children_in_lists_large_lists_and_maps_are_materialized_when_split assert!(status.is_none(), "{status:?}"); assert!(completed(&responses)); let batches = output_batches(&responses); - assert!(batches.len() > 1); + assert_eq!(batches.len(), 1); + assert!( + assembled_batches(&responses, true)[0] + .num_chunks_in_batch + .unwrap() + > 1 + ); let mut offset = 0; for batch in batches { let lists = batch diff --git a/crates/spark-connect-server/tests/support/arrow.rs b/crates/spark-connect-server/tests/support/arrow.rs new file mode 100644 index 00000000..c9fd6979 --- /dev/null +++ b/crates/spark-connect-server/tests/support/arrow.rs @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Wire chunk negotiation, reassembly and IPC encoding shared by protocol tests. + +#![allow(dead_code)] + +use std::collections::HashSet; + +use arrow::ipc::writer::{IpcWriteOptions, StreamWriter}; +use arrow::ipc::CompressionType; +use arrow::record_batch::RecordBatch; +use prost::Message; +use proto::execute_plan_response::ResponseType; +use spark_connect_proto as proto; + +pub const DEFAULT_ARROW_CHUNK_SIZE: usize = 128 * 1024 * 1024 * 9 / 10; + +pub fn chunking(allow: bool, preferred: Option) -> proto::execute_plan_request::RequestOption { + proto::execute_plan_request::RequestOption { + request_option: Some( + proto::execute_plan_request::request_option::RequestOption::ResultChunkingOptions( + proto::ResultChunkingOptions { + allow_arrow_batch_chunking: allow, + preferred_arrow_chunk_size: preferred, + }, + ), + ), + } +} + +// Reassemble before Arrow decoding, just as Spark clients do. Every chunk counts +// the whole batch's rows and has its own response ID; row offsets advance once. +pub fn assembled_batches( + responses: &[proto::ExecutePlanResponse], + negotiated: bool, +) -> Vec { + let mut output = Vec::new(); + let mut pending: Option = None; + let (mut offset, mut index) = (0, 0); + let mut data = Vec::new(); + let mut response_ids = HashSet::new(); + for response in responses { + assert!(response_ids.insert(&response.response_id)); + let Some(ResponseType::ArrowBatch(chunk)) = &response.response_type else { + continue; + }; + // The negotiated limit covers IPC payload bytes, matching the JVM. The + // protobuf envelope adds identifiers, field tags and chunk metadata. + assert!(response.encoded_len() <= chunk.data.len() + 512); + if negotiated { + assert!(chunk.data.len() <= DEFAULT_ARROW_CHUNK_SIZE); + } + assert_eq!(chunk.start_offset, Some(offset)); + if !negotiated { + assert_eq!(chunk.chunk_index, None); + assert_eq!(chunk.num_chunks_in_batch, None); + offset += chunk.row_count; + output.push(chunk.clone()); + continue; + } + assert_eq!(chunk.chunk_index, Some(index)); + let count = chunk.num_chunks_in_batch.unwrap(); + assert!(count > index); + if let Some(batch) = &mut pending { + assert_eq!(batch.row_count, chunk.row_count); + assert_eq!(batch.num_chunks_in_batch, Some(count)); + } else { + pending = Some(chunk.clone()); + } + data.extend_from_slice(&chunk.data); + index += 1; + if index == count { + let mut batch = pending.take().unwrap(); + batch.data = std::mem::take(&mut data).into(); + offset += batch.row_count; + output.push(batch); + index = 0; + } + } + assert!(pending.is_none(), "complete must not truncate a batch"); + output +} + +pub fn encoded(batch: &RecordBatch, compression: Option) -> Vec { + let options = IpcWriteOptions::default() + .try_with_compression(compression) + .unwrap(); + let mut data = Vec::new(); + { + let mut writer = + StreamWriter::try_new_with_options(&mut data, batch.schema().as_ref(), options) + .unwrap(); + writer.write(batch).unwrap(); + writer.finish().unwrap(); + } + data +} diff --git a/crates/spark-connect-server/tests/support/server.rs b/crates/spark-connect-server/tests/support/server.rs index 0ec88720..74602859 100644 --- a/crates/spark-connect-server/tests/support/server.rs +++ b/crates/spark-connect-server/tests/support/server.rs @@ -36,11 +36,18 @@ pub struct Server { impl Server { pub async fn start_with_engine(engine: Arc) -> Self { + Self::start_with_options(engine, spark_connect_server::ServerOptions::default()).await + } + + pub async fn start_with_options( + engine: Arc, + options: spark_connect_server::ServerOptions, + ) -> Self { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let (shutdown, stopped) = oneshot::channel(); let task = tokio::spawn(async move { - spark_connect_server::serve_with_engine(listener, engine, async { + spark_connect_server::serve_with_engine_and_options(listener, engine, options, async { let _ = stopped.await; }) .await From 8fcf31b758888253109ff4baa7a82aa1ff18739b Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 21:02:47 -0800 Subject: [PATCH 15/18] Clarify chunk limits and simplify Arrow encoding --- .github/workflows/server-frontend.yml | 15 ++- crates/spark-connect-server/README.md | 6 +- crates/spark-connect-server/src/lib.rs | 4 + crates/spark-connect-server/src/schema.rs | 6 +- crates/spark-connect-server/src/service.rs | 69 +++++------- crates/spark-connect-server/tests/protocol.rs | 102 ++++++++---------- .../tests/schema_consistency.rs | 11 +- 7 files changed, 99 insertions(+), 114 deletions(-) diff --git a/.github/workflows/server-frontend.yml b/.github/workflows/server-frontend.yml index 6b77267f..85cc57e9 100644 --- a/.github/workflows/server-frontend.yml +++ b/.github/workflows/server-frontend.yml @@ -22,7 +22,7 @@ name: Server frontend CI on: pull_request: branches: [master, "branch-*"] - paths: &frontend-paths + paths: - ".github/workflows/server-frontend.yml" - "Cargo.toml" - "Cargo.lock" @@ -32,7 +32,14 @@ on: - "crates/spark-connect-proto/**" push: branches: [master, "branch-*"] - paths: *frontend-paths + paths: + - ".github/workflows/server-frontend.yml" + - "Cargo.toml" + - "Cargo.lock" + - ".cargo/**" + - "rust-toolchain*" + - "crates/spark-connect-server/**" + - "crates/spark-connect-proto/**" workflow_dispatch: permissions: @@ -89,5 +96,5 @@ jobs: - name: Check protocol feature combinations run: | - cargo check --locked -p apache-spark-connect-proto --no-default-features - cargo check --locked -p apache-spark-connect-proto --no-default-features --features server + cargo check --locked -p apache-spark-connect-proto + cargo check --locked -p apache-spark-connect-proto --features server diff --git a/crates/spark-connect-server/README.md b/crates/spark-connect-server/README.md index 3b4ac6db..ce2a327b 100644 --- a/crates/spark-connect-server/README.md +++ b/crates/spark-connect-server/README.md @@ -79,8 +79,10 @@ preferred chunk payload size between 1 KiB and the configured maximum is accepte otherwise the maximum is used. `ServerOptions.max_arrow_chunk_size` defaults to 120,795,955 bytes (90% of 128 MiB), matching the JVM server's default chunk payload limit. It limits IPC payload bytes, excluding the protobuf envelope. Setting it -to zero disables chunking even when the client opts in. Clients must allow room -for the envelope in their gRPC receive limits. +to zero disables chunking even when the client opts in. An opted-in client must +accept the configured maximum plus its protobuf envelope, or request a smaller +chunk size. A 128 MiB receive limit accommodates the default maximum; raw tonic's +4 MiB default does not accommodate chunks above 4 MiB. Without client opt-in, or when chunking is disabled, each IPC batch is sent whole. Engines remain responsible for bounding batch production, including fitting the diff --git a/crates/spark-connect-server/src/lib.rs b/crates/spark-connect-server/src/lib.rs index f42f1895..195d5842 100644 --- a/crates/spark-connect-server/src/lib.rs +++ b/crates/spark-connect-server/src/lib.rs @@ -27,10 +27,14 @@ pub mod grpc { /// Transport options for the experimental frontend. #[derive(Clone, Debug)] +#[non_exhaustive] pub struct ServerOptions { /// Maximum Arrow chunk payload in bytes, excluding the protobuf envelope. /// Defaults to 90% of 128 MiB, as in Spark's JVM server. A client can request /// a smaller size. Zero disables chunking, even when the client opts in. + /// Clients must accept this payload plus the protobuf envelope: a 128 MiB + /// receive limit accommodates the default; tonic's 4 MiB default does not. + /// Alternatively, clients can request a smaller chunk size. /// Whole batches sent without chunking are not bounded by this option. pub max_arrow_chunk_size: usize, } diff --git a/crates/spark-connect-server/src/schema.rs b/crates/spark-connect-server/src/schema.rs index 76ab1b15..daddedbb 100644 --- a/crates/spark-connect-server/src/schema.rs +++ b/crates/spark-connect-server/src/schema.rs @@ -24,7 +24,7 @@ use prost::Message; use spark_connect_proto as proto; use tonic::Status; -pub(crate) const MAX_RESULT_BYTES: usize = 3 * 1024 * 1024; +pub(crate) const MAX_METADATA_RESPONSE_BYTES: usize = 3 * 1024 * 1024; pub(crate) fn result_schema(schema: &Schema) -> Result { let schema = spark_schema(schema)?; @@ -33,9 +33,9 @@ pub(crate) fn result_schema(schema: &Schema) -> Result } pub(crate) fn validate_schema_size(schema: &proto::DataType) -> Result<(), Status> { - if schema.encoded_len() > MAX_RESULT_BYTES { + if schema.encoded_len() > MAX_METADATA_RESPONSE_BYTES { return Err(Status::resource_exhausted( - "Spark schema exceeds the 3 MiB result limit", + "Spark schema exceeds the 3 MiB schema/command response limit", )); } Ok(()) diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index f2191251..93a97017 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -20,7 +20,7 @@ use std::panic::AssertUnwindSafe; use std::pin::Pin; use std::sync::{Arc, Mutex}; -use arrow::datatypes::Schema; +use arrow::datatypes::{Schema, SchemaRef}; use arrow::ipc::{reader::StreamReader, writer::StreamWriter}; use arrow::record_batch::RecordBatch; use futures::{FutureExt, Stream, StreamExt}; @@ -33,11 +33,10 @@ use uuid::Uuid; use crate::engine::{Engine, EngineSession, PreparedOperation, ResultBatch, SessionContext}; use crate::grpc::spark_connect_service_server::SparkConnectService; -use crate::schema::{self, result_schema}; +use crate::schema::{self, result_schema, MAX_METADATA_RESPONSE_BYTES}; type Output = Pin> + Send>>; type SessionKey = (String, String); -const MAX_RESULT_BYTES: usize = crate::schema::MAX_RESULT_BYTES; const MAX_RELEASED_SESSIONS: usize = 4096; const RELEASE_WAIT: std::time::Duration = std::time::Duration::from_secs(2); @@ -290,9 +289,9 @@ fn response( fn checked_response( message: proto::ExecutePlanResponse, ) -> Result { - if message.encoded_len() > MAX_RESULT_BYTES { + if message.encoded_len() > MAX_METADATA_RESPONSE_BYTES { return Err(Status::resource_exhausted( - "ExecutePlan response exceeds the 3 MiB result limit", + "Schema/command response exceeds the 3 MiB limit", )); } Ok(message) @@ -309,7 +308,7 @@ fn complete(id: &str, session: &Session, operation: &str) -> proto::ExecutePlanR ) } -fn encode_batch(batch: &RecordBatch) -> Result, Status> { +fn encode_batch(batch: &RecordBatch) -> Result<(Vec, SchemaRef), Status> { use arrow::datatypes::DataType; use arrow::record_batch::RecordBatchOptions; @@ -373,7 +372,7 @@ fn encode_batch(batch: &RecordBatch) -> Result, Status> { .finish() .map_err(|e| Status::internal(e.to_string()))?; } - Ok(bytes) + Ok((bytes, batch.schema())) } // PySpark assembles one table from all IPC batches. Its physical types and @@ -396,15 +395,6 @@ fn check_output_schema(schema: &Schema, expected: &mut Option) -> Result } } -fn checked_arrow_row_count(rows: usize, offset: i64) -> Result { - let rows = i64::try_from(rows) - .map_err(|_| Status::resource_exhausted("result batch row count exceeds i64"))?; - offset - .checked_add(rows) - .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; - Ok(rows) -} - // Keep backend ownership until the batch is dropped, including on encoder panic. struct EncodingBatch { batch: RecordBatch, @@ -412,7 +402,7 @@ struct EncodingBatch { } impl EncodingBatch { - fn encode(self) -> Result, Status> { + fn encode(self) -> Result<(Vec, SchemaRef), Status> { encode_batch(&self.batch) } } @@ -423,22 +413,23 @@ fn arrow_chunk_size( ) -> Option { use proto::execute_plan_request::request_option::RequestOption; - let mut options = options + let mut chunking = options .iter() .filter_map(|option| match &option.request_option { Some(RequestOption::ResultChunkingOptions(options)) => Some(options), _ => None, }); if maximum == 0 - || !options + || !chunking .clone() .any(|options| options.allow_arrow_batch_chunking) { return None; } Some( - options + chunking .find_map(|options| options.preferred_arrow_chunk_size) + // Validate i64 directly; Scala's toInt can wrap an invalid preference. .and_then(|bytes| usize::try_from(bytes).ok()) .filter(|bytes| (1024..=maximum).contains(bytes)) .unwrap_or(maximum), @@ -458,7 +449,8 @@ async fn send_batch( ) -> Result { // Reject unrepresentable counts/offsets before encoding or sending any chunk. let row_count = match &batch { - ResultBatch::Arrow(batch) => checked_arrow_row_count(batch.num_rows(), *offset)?, + ResultBatch::Arrow(batch) => i64::try_from(batch.num_rows()) + .map_err(|_| Status::resource_exhausted("result batch row count exceeds i64"))?, ResultBatch::Ipc { row_count, .. } if *row_count < 0 => { return Err(Status::internal("negative engine IPC row count")); } @@ -475,12 +467,9 @@ async fn send_batch( batch, _guard: guard.clone(), }; - let data = tokio::task::spawn_blocking(move || encoding.encode()) + let (data, schema) = tokio::task::spawn_blocking(move || encoding.encode()) .await .map_err(|_| Status::internal("Arrow result encoding task failed"))??; - let schema = StreamReader::try_new(data.as_slice(), None) - .map_err(|e| Status::internal(format!("invalid output IPC: {e}")))? - .schema(); check_output_schema(schema.as_ref(), output_schema)?; data.into() } @@ -584,11 +573,7 @@ impl SparkConnectService for ConnectService { checked_response(message) }) .collect::, Status>>()?; - messages.push(checked_response(complete( - &req.session_id, - &session, - &operation, - ))?); + messages.push(complete(&req.session_id, &session, &operation)); return Ok(Response::new(Box::pin(tokio_stream::iter( messages.into_iter().map(Ok), )))); @@ -672,11 +657,7 @@ impl SparkConnectService for ConnectService { } } let _ = sender - .send(Ok(checked_response(complete( - &req.session_id, - &session, - &operation, - ))?)) + .send(Ok(complete(&req.session_id, &session, &operation))) .await; Ok::<_, Status>(()) }; @@ -886,6 +867,8 @@ impl SparkConnectService for ConnectService { mod tests { use super::*; + const CHUNK_SIZE: usize = 1024 * 1024; + struct RegistryEngine; struct RegistrySession; @@ -945,13 +928,13 @@ mod tests { let schema = proto::DataType { kind: Some(proto::data_type::Kind::Struct(proto::data_type::Struct { fields: vec![proto::data_type::StructField { - name: "x".repeat(MAX_RESULT_BYTES - 64), + name: "x".repeat(MAX_METADATA_RESPONSE_BYTES - 64), ..Default::default() }], ..Default::default() })), }; - assert!(schema.encoded_len() < MAX_RESULT_BYTES); + assert!(schema.encoded_len() < MAX_METADATA_RESPONSE_BYTES); let mut header = response( &Uuid::new_v4().to_string(), &session, @@ -999,7 +982,7 @@ mod tests { &mut offset, &mut output_schema, &guard, - Some(MAX_RESULT_BYTES), + Some(CHUNK_SIZE), ) .await .unwrap_err(); @@ -1047,13 +1030,13 @@ mod tests { &mut offset, &mut output_schema, &guard, - Some(MAX_RESULT_BYTES), + Some(CHUNK_SIZE), ) .await .unwrap(); let batch = if ipc { - let data = encode_batch(&oversized).unwrap(); - assert!(data.len() > MAX_RESULT_BYTES); + let (data, _) = encode_batch(&oversized).unwrap(); + assert!(data.len() > MAX_METADATA_RESPONSE_BYTES); ResultBatch::Ipc { data, row_count: 2 } } else { oversized.clone().into() @@ -1067,7 +1050,7 @@ mod tests { &mut offset, &mut output_schema, &guard, - Some(MAX_RESULT_BYTES), + Some(CHUNK_SIZE), ) .await .unwrap_err(); @@ -1093,7 +1076,7 @@ mod tests { &mut offset, &mut output_schema, &guard, - Some(MAX_RESULT_BYTES), + Some(CHUNK_SIZE), ) .await .unwrap(); diff --git a/crates/spark-connect-server/tests/protocol.rs b/crates/spark-connect-server/tests/protocol.rs index 14c637d2..5cf1e886 100644 --- a/crates/spark-connect-server/tests/protocol.rs +++ b/crates/spark-connect-server/tests/protocol.rs @@ -297,13 +297,9 @@ async fn preferred_chunk_size_uses_the_protocol_range_and_splits_inside_a_large_ ) .unwrap(); let engine = FixtureEngine::default().with_batches(42, batch.schema(), vec![batch.clone()]); - let mut server = Server::start_with_options( - Arc::new(engine), - spark_connect_server::ServerOptions { - max_arrow_chunk_size: 3 * 1024 * 1024, - }, - ) - .await; + let mut options = spark_connect_server::ServerOptions::default(); + options.max_arrow_chunk_size = 3 * 1024 * 1024; + let mut server = Server::start_with_options(Arc::new(engine), options).await; for preferred in [ Some(1024), Some(65_537), @@ -444,7 +440,7 @@ async fn deeply_nested_requests_follow_the_configured_protobuf_decoder_limit() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn view_arrays_are_materialized_once_before_byte_chunking_including_struct_children() { +async fn view_arrays_are_materialized_before_byte_chunking_including_struct_children() { use arrow::array::{BinaryArray, BinaryViewArray, StringArray, StringViewArray, StructArray}; use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatch; @@ -461,7 +457,7 @@ async fn view_arrays_are_materialized_once_before_byte_chunking_including_struct let binaries = BinaryViewArray::from_iter_values((0..4096).map(binary)); let nested_strings = StringViewArray::from_iter_values((0..4096).map(nested_text)); // A one-row slice of these view arrays still references backing buffers - // larger than the response limit, although each individual value is 1 KiB. + // larger than the chunk preference, although each individual value is 1 KiB. for size in [ strings .data_buffers() @@ -507,50 +503,45 @@ async fn view_arrays_are_materialized_once_before_byte_chunking_including_struct let output = assembled_batches(&responses, true); assert_eq!(output.len(), 1); assert!(output[0].num_chunks_in_batch.unwrap() > 1); - let (mut rows, mut batches) = (0, 0); - for batch in output { - assert_eq!(batch.start_offset, Some(rows)); - let records = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); - let mut decoded = 0; - for record in records { - let record = record.unwrap(); - let ids = record - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(record.column(1).data_type(), &DataType::LargeUtf8); - assert_eq!(record.column(2).data_type(), &DataType::LargeBinary); - let strings = arrow::compute::cast(record.column(1), &DataType::Utf8).unwrap(); - let strings = strings.as_any().downcast_ref::().unwrap(); - let binaries = arrow::compute::cast(record.column(2), &DataType::Binary).unwrap(); - let binaries = binaries.as_any().downcast_ref::().unwrap(); - let nested = record - .column(3) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(nested.column(0).data_type(), &DataType::LargeUtf8); - let nested_strings = arrow::compute::cast(nested.column(0), &DataType::Utf8).unwrap(); - let nested_strings = nested_strings - .as_any() - .downcast_ref::() - .unwrap(); - for index in 0..record.num_rows() { - let id = rows + decoded + index as i64; - assert_eq!(ids.value(index), id); - assert_eq!(strings.value(index), text(id)); - assert_eq!(binaries.value(index), binary(id)); - assert_eq!(nested_strings.value(index), nested_text(id)); - } - decoded += record.num_rows() as i64; + let batch = &output[0]; + assert_eq!(batch.start_offset, Some(0)); + let records = StreamReader::try_new(Cursor::new(&batch.data), None).unwrap(); + let mut rows = 0; + for record in records { + let record = record.unwrap(); + let ids = record + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(record.column(1).data_type(), &DataType::LargeUtf8); + assert_eq!(record.column(2).data_type(), &DataType::LargeBinary); + let strings = arrow::compute::cast(record.column(1), &DataType::Utf8).unwrap(); + let strings = strings.as_any().downcast_ref::().unwrap(); + let binaries = arrow::compute::cast(record.column(2), &DataType::Binary).unwrap(); + let binaries = binaries.as_any().downcast_ref::().unwrap(); + let nested = record + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(nested.column(0).data_type(), &DataType::LargeUtf8); + let nested_strings = arrow::compute::cast(nested.column(0), &DataType::Utf8).unwrap(); + let nested_strings = nested_strings + .as_any() + .downcast_ref::() + .unwrap(); + for index in 0..record.num_rows() { + let id = rows + index as i64; + assert_eq!(ids.value(index), id); + assert_eq!(strings.value(index), text(id)); + assert_eq!(binaries.value(index), binary(id)); + assert_eq!(nested_strings.value(index), nested_text(id)); } - assert_eq!(decoded, batch.row_count); - rows += decoded; - batches += 1; + rows += record.num_rows() as i64; } + assert_eq!(rows, batch.row_count); assert_eq!(rows, 4096); - assert_eq!(batches, 1); server.stop().await; } @@ -614,6 +605,7 @@ async fn collect_small_slices(backing: RecordBatch) -> Vec { let output = collect_result(&mut server, range(&Uuid::new_v4().to_string(), 42)).await; let (mut batches, mut rows) = (Vec::new(), 0); for message in output { + // A small slice must not serialize the much larger retained backing allocation. assert!(message.encoded_len() <= 3 * 1024 * 1024); if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = message.response_type @@ -878,13 +870,9 @@ async fn configured_maximum_bounds_payloads_and_zero_disables_chunking() { batch.schema(), vec![Batch::Ipc(bytes.clone(), 5000)], ); - let mut server = Server::start_with_options( - Arc::new(engine), - spark_connect_server::ServerOptions { - max_arrow_chunk_size: maximum, - }, - ) - .await; + let mut options = spark_connect_server::ServerOptions::default(); + options.max_arrow_chunk_size = maximum; + let mut server = Server::start_with_options(Arc::new(engine), options).await; for preferred in [None, Some(2048), Some(4096), Some(4097)] { let mut request = range(&Uuid::new_v4().to_string(), 42); request.request_options = vec![chunking(true, preferred)]; diff --git a/crates/spark-connect-server/tests/schema_consistency.rs b/crates/spark-connect-server/tests/schema_consistency.rs index 82ed77bd..11af295a 100644 --- a/crates/spark-connect-server/tests/schema_consistency.rs +++ b/crates/spark-connect-server/tests/schema_consistency.rs @@ -47,7 +47,7 @@ use arrow_support::{assembled_batches, chunking}; use server::Server; use support::{Batch, FixtureEngine}; -const RESULT_LIMIT: usize = 3 * 1024 * 1024; +const CHUNK_PREFERENCE: usize = 3 * 1024 * 1024; const LARGE_ROWS: usize = 4096; struct BatchCase { @@ -68,7 +68,7 @@ async fn execute(engine: BatchCase) -> (Vec, Option< let engine = FixtureEngine::default().with_results(1, engine.schema, engine.batches); let mut server = Server::start_with_engine(Arc::new(engine)).await; let mut request = requests::request(&Uuid::new_v4().to_string(), 1); - request.request_options = vec![chunking(true, Some(RESULT_LIMIT as i64))]; + request.request_options = vec![chunking(true, Some(CHUNK_PREFERENCE as i64))]; let mut stream = server .client .execute_plan(request) @@ -83,10 +83,11 @@ async fn execute(engine: BatchCase) -> (Vec, Option< if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = &response.response_type { - assert!(batch.data.len() <= RESULT_LIMIT); + assert!(batch.data.len() <= CHUNK_PREFERENCE); assert!(response.encoded_len() <= batch.data.len() + 512); } else { - assert!(response.encoded_len() <= RESULT_LIMIT); + // Schema and command envelopes retain their independent 3 MiB cap. + assert!(response.encoded_len() <= 3 * 1024 * 1024); } responses.push(response); } @@ -243,7 +244,7 @@ async fn view_ipc_remains_unchanged_for_small_and_oversized_batches() { for rows in [2, LARGE_ROWS] { let batch = views(rows); let data = ipc(&batch); - assert_eq!(data.len() > RESULT_LIMIT, rows == LARGE_ROWS); + assert_eq!(data.len() > CHUNK_PREFERENCE, rows == LARGE_ROWS); let (responses, status) = execute(BatchCase { schema: batch.schema(), batches: vec![Batch::Ipc(data.clone(), rows as i64)], From c07e3c8e4c3283b0569843620ddb3e2a7d433a21 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 21:03:19 -0800 Subject: [PATCH 16/18] Keep overflow fixture independent of metadata limits --- crates/spark-connect-server/src/service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index 93a97017..71d309db 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -1036,7 +1036,7 @@ mod tests { .unwrap(); let batch = if ipc { let (data, _) = encode_batch(&oversized).unwrap(); - assert!(data.len() > MAX_METADATA_RESPONSE_BYTES); + assert!(data.len() > CHUNK_SIZE); ResultBatch::Ipc { data, row_count: 2 } } else { oversized.clone().into() From 56640d306ec510252d962b77396f7f64ac7484ae Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 21:14:08 -0800 Subject: [PATCH 17/18] Clarify frontend configuration and validation documentation --- crates/spark-connect-server/src/lib.rs | 1 + crates/spark-connect-server/src/service.rs | 2 +- crates/spark-connect-server/tests/protocol.rs | 4 ++-- dev/design/ci.md | 3 ++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/spark-connect-server/src/lib.rs b/crates/spark-connect-server/src/lib.rs index 195d5842..195c0bc6 100644 --- a/crates/spark-connect-server/src/lib.rs +++ b/crates/spark-connect-server/src/lib.rs @@ -26,6 +26,7 @@ pub mod grpc { } /// Transport options for the experimental frontend. +/// Start from [`ServerOptions::default()`] and assign the desired fields. #[derive(Clone, Debug)] #[non_exhaustive] pub struct ServerOptions { diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index 71d309db..00c81c3f 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -291,7 +291,7 @@ fn checked_response( ) -> Result { if message.encoded_len() > MAX_METADATA_RESPONSE_BYTES { return Err(Status::resource_exhausted( - "Schema/command response exceeds the 3 MiB limit", + "schema/command response exceeds the 3 MiB limit", )); } Ok(message) diff --git a/crates/spark-connect-server/tests/protocol.rs b/crates/spark-connect-server/tests/protocol.rs index 5cf1e886..d5f9bfb5 100644 --- a/crates/spark-connect-server/tests/protocol.rs +++ b/crates/spark-connect-server/tests/protocol.rs @@ -456,8 +456,8 @@ async fn view_arrays_are_materialized_before_byte_chunking_including_struct_chil let strings = StringViewArray::from_iter_values((0..4096).map(text)); let binaries = BinaryViewArray::from_iter_values((0..4096).map(binary)); let nested_strings = StringViewArray::from_iter_values((0..4096).map(nested_text)); - // A one-row slice of these view arrays still references backing buffers - // larger than the chunk preference, although each individual value is 1 KiB. + // These view columns exceed the chunk preference, so their encoded batch + // spans several byte chunks. for size in [ strings .data_buffers() diff --git a/dev/design/ci.md b/dev/design/ci.md index d30dcf3b..531267cb 100644 --- a/dev/design/ci.md +++ b/dev/design/ci.md @@ -75,7 +75,8 @@ This job should always pass; failures indicate incomplete implementation or test This focused workflow runs on frontend, protocol, workspace manifest, lockfile, Cargo configuration, toolchain and workflow changes. Each stable Rust and Rust -1.88.0 job has a 45-minute limit and runs `cargo test --locked -p spark-connect-server`. +1.88.0 job has a 45-minute limit, runs `cargo test --locked -p spark-connect-server`, +and checks the protocol crate with its `server` feature both off and on. The existing workspace lint job covers the frontend with its pinned Rust toolchain; test jobs do not promote warnings from a floating or older compiler to errors. Ubuntu 24.04 supplies a protobuf compiler with From 74c8cc2d269c32d46f3abc15d57cc1e31a695444 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 18 Nov 2026 22:14:37 -0800 Subject: [PATCH 18/18] Add embedded JVM engine on the modular Connect frontend Stack the native JNI host, Java adapter, platform packaging and integration coverage on the standalone frontend in PR #870. Preserve executor-produced IPC and exercise negotiated byte chunking through real JNI. Written by OpenAI Codex. --- .github/native-server-protoc.txt | 25 + .github/workflows/lint.yml | 5 + .github/workflows/native-server.yml | 376 +++ .gitignore | 3 + AGENTS.md | 13 +- Cargo.lock | 16 + Cargo.toml | 13 +- README.md | 14 +- SECURITY.md | 50 +- crates/spark-connect-server/Cargo.toml | 1 + crates/spark-connect-server/README.md | 602 +++- .../examples/jvm_client.py | 38 + .../examples/rust_client.rs | 52 + crates/spark-connect-server/jni/Cargo.toml | 47 + crates/spark-connect-server/jni/build.rs | 39 + .../spark-connect-server/jni/native/bridge.c | 1006 ++++++ .../spark-connect-server/jni/native/bridge.h | 77 + crates/spark-connect-server/jni/src/engine.rs | 2934 +++++++++++++++++ crates/spark-connect-server/jni/src/lib.rs | 638 ++++ .../spark-connect-server/jni/test-c-bridge.sh | 50 + .../spark-connect-server/jni/tests/c_bridge.c | 796 +++++ crates/spark-connect-server/jvm/build.sh | 79 + .../jvm/engine_acceptance.py | 794 +++++ .../jvm/native-licenses/LICENSE-binary | 1805 ++++++++++ .../jvm/native-licenses/NOTICE-binary | 26 + .../jvm/native-licenses/dependencies.json | 2123 ++++++++++++ .../jvm/package-native.py | 333 ++ crates/spark-connect-server/jvm/run-engine.sh | 49 + .../jvm/run_acceptance.py | 314 ++ .../engine/ArrowPartitionConverter.java | 64 + .../engine/EmbeddedSparkEngine.java | 161 + .../engine/NativeEngineServer.java | 94 + .../nativebridge/engine/NativeIpcBatch.java | 33 + .../engine/NativeLibraryLoader.java | 239 ++ .../nativebridge/engine/SparkJvmEngine.java | 1018 ++++++ .../engine/CleanupFailureTest.java | 257 ++ .../nativebridge/engine/EngineSmokeTest.java | 1680 ++++++++++ .../engine/IpcBatchSmokeTest.java | 436 +++ .../engine/NativeEngineLifecycleTest.java | 810 +++++ .../engine/NativeJniSmokeTest.java | 295 ++ .../engine/NativeLibraryLoaderTest.java | 267 ++ .../spark-connect-server/jvm/test-engine.sh | 40 + .../jvm/test-jni-smoke.sh | 80 + .../jvm/test-native-loader.sh | 67 + .../jvm/test_acceptance_guards.py | 315 ++ .../jvm/test_package_native.py | 518 +++ dev/design/ci.md | 74 +- docs/contributing.md | 21 + scripts/gen_binary_licenses.py | 31 +- scripts/gen_native_binary_licenses.py | 252 ++ .../tests/test_gen_native_binary_licenses.py | 217 ++ 51 files changed, 19229 insertions(+), 58 deletions(-) create mode 100644 .github/native-server-protoc.txt create mode 100644 .github/workflows/native-server.yml create mode 100644 crates/spark-connect-server/examples/jvm_client.py create mode 100644 crates/spark-connect-server/examples/rust_client.rs create mode 100644 crates/spark-connect-server/jni/Cargo.toml create mode 100644 crates/spark-connect-server/jni/build.rs create mode 100644 crates/spark-connect-server/jni/native/bridge.c create mode 100644 crates/spark-connect-server/jni/native/bridge.h create mode 100644 crates/spark-connect-server/jni/src/engine.rs create mode 100644 crates/spark-connect-server/jni/src/lib.rs create mode 100755 crates/spark-connect-server/jni/test-c-bridge.sh create mode 100644 crates/spark-connect-server/jni/tests/c_bridge.c create mode 100755 crates/spark-connect-server/jvm/build.sh create mode 100755 crates/spark-connect-server/jvm/engine_acceptance.py create mode 100644 crates/spark-connect-server/jvm/native-licenses/LICENSE-binary create mode 100644 crates/spark-connect-server/jvm/native-licenses/NOTICE-binary create mode 100644 crates/spark-connect-server/jvm/native-licenses/dependencies.json create mode 100755 crates/spark-connect-server/jvm/package-native.py create mode 100755 crates/spark-connect-server/jvm/run-engine.sh create mode 100644 crates/spark-connect-server/jvm/run_acceptance.py create mode 100644 crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/ArrowPartitionConverter.java create mode 100644 crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/EmbeddedSparkEngine.java create mode 100644 crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/NativeEngineServer.java create mode 100644 crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/NativeIpcBatch.java create mode 100644 crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/NativeLibraryLoader.java create mode 100644 crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/SparkJvmEngine.java create mode 100644 crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/CleanupFailureTest.java create mode 100644 crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/EngineSmokeTest.java create mode 100644 crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/IpcBatchSmokeTest.java create mode 100644 crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/NativeEngineLifecycleTest.java create mode 100644 crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/NativeJniSmokeTest.java create mode 100644 crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/NativeLibraryLoaderTest.java create mode 100755 crates/spark-connect-server/jvm/test-engine.sh create mode 100755 crates/spark-connect-server/jvm/test-jni-smoke.sh create mode 100755 crates/spark-connect-server/jvm/test-native-loader.sh create mode 100644 crates/spark-connect-server/jvm/test_acceptance_guards.py create mode 100755 crates/spark-connect-server/jvm/test_package_native.py create mode 100644 scripts/gen_native_binary_licenses.py create mode 100644 scripts/tests/test_gen_native_binary_licenses.py diff --git a/.github/native-server-protoc.txt b/.github/native-server-protoc.txt new file mode 100644 index 00000000..f4402dcc --- /dev/null +++ b/.github/native-server-protoc.txt @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Binary wheels for Linux/macOS on x86_64/aarch64 only. +protoc-wheel-0==30.2 \ + --hash=sha256:f03be211d5d8797ac9c376f455200214335cea6d6ccfb9444372959cb37ca496 \ + --hash=sha256:922c711b54e5a2106fb5c510c9a65420325b34349bd8d3b360a2299e238fc0b5 \ + --hash=sha256:de310ef63f2ba9374e052857b4406842c1f2299b656f58a2df2b923e2455d80d \ + --hash=sha256:fd4cbccd84006c51cb344e8cc355fbd0d7ed52dc23b5ed25fb831a7c5dcc5546 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4dd4fa5d..9fe66ee5 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -55,6 +55,11 @@ jobs: # member), whose build script compiles its UDFs to WASM; `tests/wasm-udf-wasi` # compiles its UDFs for WASI. rustup target add wasm32-unknown-unknown wasm32-wasip1 + - name: Set up JDK for workspace JNI headers + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" - name: Cache cargo & target uses: actions/cache@v4 with: diff --git a/.github/workflows/native-server.yml b/.github/workflows/native-server.yml new file mode 100644 index 00000000..617cad14 --- /dev/null +++ b/.github/workflows/native-server.yml @@ -0,0 +1,376 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Native JVM server CI + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +on: + push: + branches: [master, "branch-*"] + paths: + - ".github/workflows/native-server.yml" + - ".github/native-server-protoc.txt" + - "Cargo.toml" + - "Cargo.lock" + - "LICENSE" + - "NOTICE" + - "scripts/gen_binary_licenses.py" + - "scripts/gen_native_binary_licenses.py" + - "scripts/tests/test_gen*binary_licenses.py" + - ".cargo/**" + - "crates/spark-connect-server/**" + - "crates/spark-connect-proto/**" + - "crates/spark-connect-core/**" + - "crates/spark-connect/**" + pull_request: + branches: [master, "branch-*"] + paths: + - ".github/workflows/native-server.yml" + - ".github/native-server-protoc.txt" + - "Cargo.toml" + - "Cargo.lock" + - "LICENSE" + - "NOTICE" + - "scripts/gen_binary_licenses.py" + - "scripts/gen_native_binary_licenses.py" + - "scripts/tests/test_gen*binary_licenses.py" + - ".cargo/**" + - "crates/spark-connect-server/**" + - "crates/spark-connect-proto/**" + - "crates/spark-connect-core/**" + - "crates/spark-connect/**" + workflow_dispatch: + +permissions: + contents: read + +env: + # Bound parallel compilation and debug-artifact size on hosted runners. + CARGO_BUILD_JOBS: "2" + CARGO_PROFILE_TEST_DEBUG: "0" + MACOSX_DEPLOYMENT_TARGET: "11.0" + +jobs: + native-server: + name: Native JAR (${{ matrix.platform }}) + strategy: + fail-fast: false + matrix: + include: + - { runner: ubuntu-22.04, platform: linux-x86_64, library: libspark_connect_jni.so } + - { runner: ubuntu-22.04-arm, platform: linux-aarch64, library: libspark_connect_jni.so } + - { runner: macos-15-intel, platform: macos-x86_64, library: libspark_connect_jni.dylib } + - { runner: macos-15, platform: macos-aarch64, library: libspark_connect_jni.dylib } + runs-on: ${{ matrix.runner }} + timeout-minutes: 75 + steps: + - uses: actions/checkout@v4 + + - name: Install Rust MSRV + # Use runner-provided rustup to keep actions limited to actions/*. + run: | + rustup toolchain install 1.88.0 --profile minimal + rustup default 1.88.0 + rustup component add clippy + + - name: Set up JDK for JNI headers + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + + - name: Set up Python for packaging checks + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Cache cargo and target + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ matrix.platform }}-cargo-jvm-server-1.88.0-${{ hashFiles('.github/native-server-protoc.txt') }}-${{ hashFiles('Cargo.lock') }} + restore-keys: ${{ matrix.platform }}-cargo-jvm-server-1.88.0-${{ hashFiles('.github/native-server-protoc.txt') }}- + + - name: Install protobuf compiler with proto3 optional support + # Ubuntu 22.04's protoc 3.12 requires an extra optional-field flag. + # Pin one compiler on every host while preserving the glibc baseline. + run: | + python3 -m pip install --require-hashes --only-binary=:all: --no-deps \ + -r .github/native-server-protoc.txt + protoc_path="$(python3 -c 'import sysconfig; print(sysconfig.get_path("scripts"))')/protoc" + "$protoc_path" --version + echo "PROTOC=$protoc_path" >> "$GITHUB_ENV" + + - name: Test frontend and JNI crates + # Deterministic engine fixtures; no Spark distribution is needed. + run: cargo test --locked -p spark-connect-server -p spark-connect-jni + + - name: Strict Clippy for native crates + run: cargo clippy --locked -p spark-connect-server -p spark-connect-jni --all-targets --no-deps -- -D warnings + + - name: JNI dependency licenses and notices are up to date + # Resolve the normal static-link closure for all four supported targets; + # Python wheel license assets describe a different dependency graph. + run: | + python3 -B -m unittest discover -s scripts/tests -p 'test_gen*binary_licenses.py' + python3 -B scripts/gen_native_binary_licenses.py --check + + - name: Test acceptance runner guards + run: python3 -B crates/spark-connect-server/jvm/test_acceptance_guards.py + + - name: Test native JAR packaging + run: python3 -B crates/spark-connect-server/jvm/test_package_native.py + + - name: Test C bridge + run: crates/spark-connect-server/jni/test-c-bridge.sh + + - name: Test C bridge with sanitizers + env: + ASAN_OPTIONS: ${{ runner.os == 'Linux' && 'detect_leaks=1:halt_on_error=1' || 'detect_leaks=0:halt_on_error=1' }} + UBSAN_OPTIONS: halt_on_error=1 + run: crates/spark-connect-server/jni/test-c-bridge.sh --sanitize + + - name: Build release native library + run: cargo build --locked --release -p spark-connect-jni + + - name: Package platform native JAR + run: | + python3 -B crates/spark-connect-server/jvm/package-native.py \ + --library "$PWD/target/release/${{ matrix.library }}" \ + --platform "${{ matrix.platform }}" --output-dir native-jars + + - name: Test Java loader and load packaged release library + # This needs only a JDK: no selected Spark assembly or adapter compilation. + run: | + native_jar=(native-jars/spark-connect-native-*-${{ matrix.platform }}.jar) + crates/spark-connect-server/jvm/test-native-loader.sh --native-jar "${native_jar[0]}" + + - name: Upload platform native JAR + uses: actions/upload-artifact@v4 + with: + name: spark-connect-native-platform-${{ matrix.platform }} + path: native-jars/* + if-no-files-found: error + overwrite: true + retention-days: 7 + + jni-smoke: + name: Real JNI start/query/stop (pinned Spark) + needs: native-server + runs-on: ubuntu-22.04 + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Check out pinned Spark runtime source + uses: actions/checkout@v4 + with: + repository: apache/spark + ref: 250ccca3f2b5f043e20b415adf17bda5d202c60f + path: spark-runtime + persist-credentials: false + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Restore pinned Spark assembly + id: spark-assembly + uses: actions/cache/restore@v4 + with: + path: spark-runtime/assembly/target/scala-2.13/jars + # Bump both cache key versions when the command, profiles or JDK change. + key: ${{ runner.os }}-${{ runner.arch }}-jni-smoke-spark-250ccca3f2b5f043e20b415adf17bda5d202c60f-jdk21-v1 + + - name: Restore pinned Spark Maven dependencies + id: spark-maven + if: steps.spark-assembly.outputs.cache-hit != 'true' + uses: actions/cache/restore@v4 + with: + path: ~/.m2/repository + key: linux-jni-smoke-jdk21-spark-250ccca3f2b5f043e20b415adf17bda5d202c60f-v1 + + - name: Build pinned Spark assembly + if: steps.spark-assembly.outputs.cache-hit != 'true' + working-directory: spark-runtime + timeout-minutes: 100 + # This range-query smoke does not need the local ArrowConverters repair + # used by full acceptance. Build the exact upstream development snapshot. + run: ./build/mvn -B -pl assembly -am -DskipTests package + + # Save before testing the adapter, so a JNI regression can reuse this build. + - name: Save pinned Spark assembly + if: steps.spark-assembly.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: spark-runtime/assembly/target/scala-2.13/jars + key: ${{ steps.spark-assembly.outputs.cache-primary-key }} + + - name: Save pinned Spark Maven dependencies + if: steps.spark-assembly.outputs.cache-hit != 'true' && steps.spark-maven.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: ~/.m2/repository + key: ${{ steps.spark-maven.outputs.cache-primary-key }} + + - name: Download validated Linux native JAR + uses: actions/download-artifact@v4 + with: + name: spark-connect-native-platform-linux-x86_64 + path: native-jars + + - name: Extract the validated JNI library + run: | + python3 -B - <<'PY' + import zipfile + from pathlib import Path + + jars = list(Path("native-jars").glob("spark-connect-native-*-linux-x86_64.jar")) + if len(jars) != 1: + raise SystemExit("expected exactly one validated Linux native JAR") + destination = Path("native-library/libspark_connect_jni.so") + destination.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(jars[0]) as jar: + destination.write_bytes(jar.read("META-INF/native/linux-x86_64/libspark_connect_jni.so")) + PY + + - name: Compile Java adapter and run real JNI smoke + timeout-minutes: 10 + run: | + export SPARK_HOME="$PWD/spark-runtime" + crates/spark-connect-server/jvm/test-jni-smoke.sh \ + "$PWD/native-library/libspark_connect_jni.so" + + - name: Upload real JNI smoke logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: spark-connect-real-jni-smoke + path: crates/spark-connect-server/.build/jvm-native/jni-smoke-* + include-hidden-files: true + if-no-files-found: warn + retention-days: 7 + + combine-native-jars: + name: Combine platform native JARs + needs: native-server + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + + - name: Download validated platform JARs + uses: actions/download-artifact@v4 + with: + pattern: spark-connect-native-platform-* + path: platform-jars + merge-multiple: true + + - name: Build all-platform native JAR + run: | + native_jars=(platform-jars/*.jar) + test "${#native_jars[@]}" -eq 4 + python3 -B crates/spark-connect-server/jvm/package-native.py \ + --combine "${native_jars[@]}" --output-dir native-jars + + - name: Load matching native library from all-platform JAR + run: | + native_jar=(native-jars/spark-connect-native-*-all.jar) + crates/spark-connect-server/jvm/test-native-loader.sh --native-jar "${native_jar[0]}" + + - name: Build macOS universal native JAR + run: | + macos_jars=(platform-jars/*-macos-*.jar) + test "${#macos_jars[@]}" -eq 2 + python3 -B crates/spark-connect-server/jvm/package-native.py \ + --combine-macos "${macos_jars[@]}" --output-dir macos-native-jars + + - name: Upload all-platform native JAR + uses: actions/upload-artifact@v4 + with: + name: spark-connect-native-all + path: native-jars/* + if-no-files-found: error + overwrite: true + retention-days: 7 + + - name: Upload macOS universal native JAR + uses: actions/upload-artifact@v4 + with: + name: spark-connect-native-macos-universal + path: macos-native-jars/* + if-no-files-found: error + overwrite: true + retention-days: 7 + + test-macos-universal: + name: Load macOS universal JAR (${{ matrix.platform }}) + needs: combine-native-jars + strategy: + fail-fast: false + matrix: + include: + - { runner: macos-15-intel, platform: macos-x86_64 } + - { runner: macos-15, platform: macos-aarch64 } + runs-on: ${{ matrix.runner }} + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Download macOS universal native JAR + uses: actions/download-artifact@v4 + with: + name: spark-connect-native-macos-universal + path: native-jars + + - name: Load matching native library from macOS universal JAR + run: | + native_jar=(native-jars/spark-connect-native-*-macos-universal.jar) + crates/spark-connect-server/jvm/test-native-loader.sh --native-jar "${native_jar[0]}" diff --git a/.gitignore b/.gitignore index 31d38d32..a8fb0305 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,9 @@ __pycache__/ /spark-warehouse /artifacts +# Generated JNI/Java builds and local server acceptance artifacts. +/crates/spark-connect-server/.build/ + # downloaded Spark distribution / pip package (never commit — CI downloads it) /pyspark-*.tar.gz /pyspark-[0-9]*/ diff --git a/AGENTS.md b/AGENTS.md index 2c2a1be0..162b7db7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -278,7 +278,12 @@ Vendor the ones we don't have a Rust implementation for. ## 7. Pre-push discipline (mandatory) -Run all of these green before pushing: +Run all of these green before pushing. Workspace Clippy includes the experimental +JNI host and requires macOS/Linux, a C compiler and `JAVA_HOME` pointing to a JDK +with JNI headers (JDK 21 is used in CI). Client-only checks without a JDK can add +`--exclude spark-connect-jni` to the workspace Clippy command. +The frontend-only `cargo test --locked -p spark-connect-server` uses injected +engines and needs no JDK; it does not establish JNI or Spark compatibility. ``` cargo test -p apache-spark-connect --release --lib cargo test -p apache-spark-connect --release --no-run # compiles ALL integration tests @@ -288,6 +293,12 @@ cargo clippy --workspace --all-targets -- -D warnings # BLOCKING CI # CI's lint job pins Rust 1.100.0, also lints datafusion,polars and the excluded crates, and runs # rustdoc of the published crates but proto with RUSTDOCFLAGS="-D warnings" (see docs/contributing.md) ``` +Native JAR license assets bind whole-file hashes for the `inputs` recorded in +`crates/spark-connect-server/jvm/native-licenses/dependencies.json`. Run +`python3 scripts/gen_native_binary_licenses.py --check` before pushing. If it reports +stale assets, run `python3 scripts/gen_native_binary_licenses.py` and commit the +regenerated native-license assets, even when the JNI dependency closure is unchanged. + Then re-run the step-2 diff (0 user-facing gaps) and the parity gate `scripts/run_official_tests.py` (runs the official connect suite through our Rust transport; `FLAKY_FILES` = retry, not skip). Only then diff --git a/Cargo.lock b/Cargo.lock index 41a9b1b3..75583244 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4048,10 +4048,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spark-connect-jni" +version = "0.1.0" +dependencies = [ + "apache-spark-connect-proto", + "arrow", + "cc", + "futures", + "libc", + "prost", + "spark-connect-server", + "tokio", + "tonic", +] + [[package]] name = "spark-connect-server" version = "0.1.0" dependencies = [ + "apache-spark-connect", "apache-spark-connect-proto", "arrow", "futures", diff --git a/Cargo.toml b/Cargo.toml index 189b8bab..70f41219 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ members = [ "crates/pyspark-rs", "examples", "crates/spark-connect-server", + "crates/spark-connect-server/jni", ] exclude = [ "examples/wasm-udfs", @@ -45,8 +46,12 @@ exclude = [ # (so `cargo test --workspace` stays green without it). Build them explicitly with # `cargo build --manifest-path examples/wasm-udfs/Cargo.toml` (etc) in CI or when # the target is installed locally. -# The experimental protocol frontend is opt-in; build or test it with -# `-p spark-connect-server`. It is included in workspace lint checks. +# The experimental protocol frontend and JNI host are opt-in. Build them with +# `cargo build -p spark-connect-server` or `cargo build -p spark-connect-jni`; +# workspace builds/tests/lint include the JNI host and need macOS/Linux, a C +# compiler and JAVA_HOME pointing to a JDK with JNI headers. JDK 21 is the +# validated adapter and CI baseline. On Windows or without a JDK, use +# `--workspace --exclude spark-connect-jni`. default-members = [ "crates/spark-connect-proto", "crates/spark-connect-core", @@ -119,3 +124,7 @@ pyo3 = { version = "0.29", features = ["extension-module", "abi3-py39"] } bytes = "1" uuid = { version = "1", features = ["v4"] } tracing = "0.1" + +# JNI bridge and its C build shim. +libc = "0.2" +cc = "1" diff --git a/README.md b/README.md index fcc755db..72f4c1d1 100644 --- a/README.md +++ b/README.md @@ -124,11 +124,15 @@ the full setup, SQL registration, supported types, and more. The unpublished [`spark-connect-server`](crates/spark-connect-server/README.md) library provides a Rust Spark Connect protocol frontend. It manages protocol sessions, operations and Arrow IPC responses, delegating planning and execution to an -application-provided `Engine`. The library includes no execution backend or standalone -server executable; its protocol tests use injected engine fixtures. - -The frontend is intended for local, trusted experiments. Read its -[security model](SECURITY.md#experimental-server-frontend) before embedding it. +application-provided `Engine`; its protocol tests use injected engine fixtures. +The optional, unpublished `spark-connect-jni` crate embeds that frontend inside a +Spark JVM and supplies an adapter for Spark planning and execution. Both crates +are explicit opt-in builds; no standalone server executable is included. See the +[server guide](crates/spark-connect-server/README.md) for the JNI prototype's +supported scope, build requirements and validation limits. + +The frontend and JNI prototype are intended for local, trusted experiments. Read their +[security model](SECURITY.md#experimental-server-frontend) before embedding them. ## Contributing diff --git a/SECURITY.md b/SECURITY.md index d80df8b4..f852caac 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -40,7 +40,49 @@ listener's network scope. Any process that can reach it can submit requests to that engine. Protocol limits and session isolation are resource and correctness guards, not a security boundary between mutually untrusted users. -This frontend does not include an execution backend or a standalone server. -Its injected-engine tests establish frontend protocol behavior, not safe -exposure of an execution engine to untrusted clients. Apache Spark's operator -security guidance remains relevant to any application that connects an engine. +The frontend library accepts an application-provided engine; the optional JNI +adapter below supplies an embedded Spark engine. The frontend's injected-engine +tests establish frontend protocol behavior, not safe exposure of an execution engine +to untrusted clients. Apache Spark's operator security guidance remains relevant +to any application that connects an engine. + +## Experimental JVM server + +The optional `spark-connect-server` and `spark-connect-jni` crates are a work in +progress for trusted local clients and query inputs. The launcher defaults to +`127.0.0.1:15002`. This frontend does not implement authentication, authorization, +or TLS. Client-supplied user and session identifiers are metadata, not verified +identities. Keep its listener on loopback; changing the address does not add +network protection or make it suitable for a multi-tenant deployment. + +Queries execute with the permissions and resource access of the hosting Spark +process. Accepted relation message forms are compatibility limits, not access +controls: SQL and named-table resolution can reach files and reflective built-in +functions according to the hosting Spark configuration. Clients can change +modifiable session SQL settings, such as `spark.sql.runSQLOnFiles`, through +Config Set. The native library, Java adapter, Spark assembly and dependencies +must be trusted. Arrow IPC batches are an internal interface between that Java +producer and Rust in the same process. Session separation, batch bounds and task +admission do not provide process isolation or complete memory and CPU quotas. + +Each engine has fixed capacities of 64 initialized JVM sessions and 64 active +tasks, with a two-second deadline for session/task permit admission. There is no +idle session expiry. Clients that exit without releasing sessions can exhaust +session capacity; unread result streams can retain task capacity. Release known +sessions with `ReleaseSession` and let owned work and deferred cleanup drain to +recover capacity, or restart the server. Abandoned sessions whose identifiers +are lost remain until a restart. These admission bounds do not prevent one +admitted query from consuming the hosting process's available resources. + +Failed JNI cleanup retains its handles and admission permits and +prevents a successful native shutdown. Retrying server close can recover from a +temporary JNI attachment failure. A Java cleanup exception may follow partial +Spark or Arrow teardown; it remains recorded and is rethrown without repeating +those side effects. Worker detach runs only on its owning OS thread; failure +after the bounded same-thread retry retains the attachment and cannot be retried +by a later server close on another thread. Such retained failures require +restarting the hosting JVM. + +See the [server guide](crates/spark-connect-server/README.md) for its supported +operations, platform requirements and lifecycle limits. Apache Spark's security +configuration does not automatically apply to this experimental Rust listener. diff --git a/crates/spark-connect-server/Cargo.toml b/crates/spark-connect-server/Cargo.toml index 96adcd4a..13988cec 100644 --- a/crates/spark-connect-server/Cargo.toml +++ b/crates/spark-connect-server/Cargo.toml @@ -39,3 +39,4 @@ uuid.workspace = true [dev-dependencies] h2.workspace = true +spark-connect = { package = "apache-spark-connect", path = "../spark-connect" } diff --git a/crates/spark-connect-server/README.md b/crates/spark-connect-server/README.md index ce2a327b..574c90a6 100644 --- a/crates/spark-connect-server/README.md +++ b/crates/spark-connect-server/README.md @@ -17,22 +17,79 @@ ~ under the License. --> -# Experimental Spark Connect server frontend - -This unpublished library serves a subset of the Spark Connect protocol using the -workspace's `apache-spark-connect-proto` messages and its optional `server` -bindings. The caller supplies an -`Engine` implementation and a bound Tokio TCP listener to `serve_with_engine`. -Use `serve_with_engine_and_options` with `ServerOptions` to configure Arrow -chunking. -No execution engine or standalone server executable is included. - -Rust owns protocol sessions, operation IDs, response framing and result-stream -lifetimes. Each protocol session receives an `EngineSession` that owns its -planning, catalog and configuration state. Its `prepare` method returns a -`PreparedQuery` or command response payloads; the frontend adds protocol IDs and -`ResultComplete`. The engine decides which plans, commands and metadata requests -it can execute and reports unsupported inputs explicitly. +# Rust Spark Connect frontend with an embedded JVM engine (WIP) + +This prototype embeds a Rust Spark Connect listener inside a Spark JVM through +JNI. Rust owns request decoding, protocol sessions, operation IDs, response +framing and stream lifetimes. The Java adapter calls Spark planning and execution +APIs through an in-process engine interface. + +The frontend uses the workspace's `apache-spark-connect-proto` messages. It sends +serialized `Relation` inputs through JNI; Java decodes them with Spark's protobuf +classes and uses `SparkConnectPlanner` to construct Catalyst plans. Spark executes +those plans in the JVM that hosts the Rust listener. This boundary does not pass +Spark Connect service requests, responses or operation registries to Java. + +This is a work in progress for local trusted workloads, with a limited protocol +surface. It does not provide authentication, authorization or TLS. Keep its +listener on loopback and read the [security policy](../../SECURITY.md) before use. + +## Build requirements and compatibility + +- Rust 1.88 or newer, Cargo and `protoc` 3.15 or newer (proto3 optional support). +- JDK 21, a C compiler with C11/pthread support and JNI headers. Set `JAVA_HOME` + to the JDK directory. +- Python 3.9 or newer for artifact metadata and packaging. Java build scripts use + `python3`; set `NATIVE_BUILD_PYTHON` to select another interpreter. +- macOS or Linux. Windows is not supported by this prototype. +- A compatible Spark assembly containing the shaded Connect protobuf/gRPC jars. + Set `SPARK_HOME` to its directory. The scripts accept jars in either + `$SPARK_HOME/jars` or `$SPARK_HOME/assembly/target/scala-2.13/jars`. + +The frontend and JNI adapter passed local checks on macOS ARM64, including the +selected-runtime acceptance described under [Validation](#validation). The +historical performance measurements below have not been reproduced with the +current result-framing frontend. + +The workspace currently tracks the Spark 4.3.0 protocol; its pin is recorded in +[PROTO_VERSION.txt](../spark-connect-proto/proto/PROTO_VERSION.txt). The JNI +implementation was developed and validated with Spark `4.2.0-SNAPSHOT`, revision +`250ccca3f2b5f043e20b415adf17bda5d202c60f`, Scala 2.13.17 and JDK 21.0.10. It depends +on internal Spark APIs, including `SessionHolder`, `SparkConnectPlanner`, +`QueryExecution` and Arrow conversion helpers. That validation does not establish +Spark 4.3.0 assembly compatibility or support for every 4.3.0 protocol feature. +Other assemblies may require Java source changes and fresh acceptance checks. + +The range-query JNI smoke test uses that unmodified Spark snapshot. The broader +acceptance run used a separate local repair to +`ArrowConverters.InternalRowIteratorFromIPCStream` for Arrow input resource +cleanup at EOF and on failure. That repair is not bundled with this prototype. +The smoke test does not cover Arrow IPC input, and the broader result does not +establish full acceptance on the unmodified snapshot. + +The `spark-connect-server` and nested `spark-connect-jni` crates are unpublished, +optional workspace members excluded from `default-members`. Build them explicitly +with `-p`; the server crate provides the frontend library and the JNI crate builds +the JVM-loaded native library. There is no standalone server executable. + +All commands below run from the repository root. Cargo places native artifacts in +`target/`; the JVM scripts place generated Java artifacts and acceptance reports +under `crates/spark-connect-server/.build/`. + +## Frontend library + +The unpublished `spark-connect-server` library uses the protocol crate's optional +`server` bindings and accepts an application-provided `Engine` and bound Tokio +TCP listener. `serve_with_engine` uses default transport options; Rust embedders +can call `serve_with_engine_and_options` with `ServerOptions` to configure Arrow +chunking. The embedded JVM launcher currently uses those defaults. The nested +`spark-connect-jni` crate supplies the Spark engine adapter described in this +guide; the frontend library can also be used without it. + +Each protocol session receives an `EngineSession` that owns planning, catalog +and configuration state. Its `prepare` method returns a `PreparedQuery` or +command response payloads; the frontend adds protocol IDs and `ResultComplete`. +The engine decides which plans, commands and metadata requests it can execute. ## Protocol subset @@ -49,7 +106,9 @@ client-observed server session IDs. Client user IDs distinguish session keys but are not authenticated identities. Protobuf nesting follows the selected Prost features; dependency feature unification can disable its recursion limit, and the frontend adds no independent -nesting guard. +nesting guard. Deep plans can fail before engine planning; when its recursion +limit is enabled, the Rust transport reports a protobuf recursion error as +`INTERNAL`. ## Arrow results @@ -77,12 +136,12 @@ frontend divides serialized IPC bytes into chunks and sets `chunk_index` and client concatenates a batch's chunks before decoding or decompressing it. A preferred chunk payload size between 1 KiB and the configured maximum is accepted; otherwise the maximum is used. `ServerOptions.max_arrow_chunk_size` defaults to -120,795,955 bytes (90% of 128 MiB), matching the JVM server's default chunk payload -limit. It limits IPC payload bytes, excluding the protobuf envelope. Setting it -to zero disables chunking even when the client opts in. An opted-in client must -accept the configured maximum plus its protobuf envelope, or request a smaller -chunk size. A 128 MiB receive limit accommodates the default maximum; raw tonic's -4 MiB default does not accommodate chunks above 4 MiB. +120,795,955 bytes (about 115.2 MiB; 90% of 128 MiB), matching the JVM server's +default chunk payload limit. It limits IPC payload bytes, excluding the protobuf +envelope. Setting it to zero disables chunking even when the client opts in. An +opted-in client must accept the configured maximum plus its protobuf envelope, +or request a smaller chunk size. A 128 MiB receive limit accommodates the default +maximum; raw tonic's 4 MiB default does not accommodate chunks above 4 MiB. Without client opt-in, or when chunking is disabled, each IPC batch is sent whole. Engines remain responsible for bounding batch production, including fitting the @@ -122,13 +181,415 @@ must not panic either. This is an experiment for trusted local clients and trusted engine outputs. Keep the supplied listener on loopback: this frontend supplies no authentication, -authorization or TLS. Requests can be up to 64 MiB. Active sessions and operations -have no admission limit or idle expiry; only released-session tombstones are -bounded, to the most recent 4096 distinct IDs. An evicted ID can create a new -session. These limits do not provide isolation or complete memory and CPU quotas. -Execution permissions and resource access are the engine's responsibility. +authorization or TLS. Requests can be up to 64 MiB. The frontend registry has no +active-session or operation admission limit or idle expiry; the JNI engine adds +the capacities described below. Released-session tombstones are bounded to the +most recent 4096 distinct IDs. An evicted ID can +create a new session. These limits do not provide isolation or complete memory +and CPU quotas. Execution permissions and resource access are the engine's +responsibility. + +## Run the embedded server + +With `JAVA_HOME` and `SPARK_HOME` set, build the native library: + +```bash +cargo build --release -p spark-connect-jni +``` + +On macOS: + +```bash +export NATIVE_LIBRARY="$PWD/target/release/libspark_connect_jni.dylib" +crates/spark-connect-server/jvm/run-engine.sh "$NATIVE_LIBRARY" 127.0.0.1:15002 +``` + +On Linux, set `NATIVE_LIBRARY` to +`$PWD/target/release/libspark_connect_jni.so` instead. The launcher requires an +absolute library path, compiles the Java adapter and starts `EmbeddedSparkEngine` +with `spark-submit`. `SPARK_MASTER` defaults to `local[2]`. + +In another terminal, use the official Rust client example: + +```bash +SPARK_REMOTE=sc://127.0.0.1:15002 cargo run -p spark-connect-server --example rust_client +``` + +It collects a range aggregate and a SQL result. A compatible official PySpark +Connect client can also run: + +```bash +SPARK_REMOTE=sc://127.0.0.1:15002 python crates/spark-connect-server/examples/jvm_client.py +``` + +Stop the launcher with Ctrl-C. Native shutdown cancels and drains owned work, +releases engine sessions and closes the Rust listener before Spark is stopped. +The native library is process-lived; unloading it while the JVM continues running +is unsupported. A failed drain is reported, and resources can remain retained +until teardown succeeds or the process exits. + +If `NativeEngineServer.close()` reports a JNI attachment failure, the server keeps +its cleanup handles and admission reservations; a later explicit `close()` can +retry after attachment recovers. Worker detachment must run on the owning OS +thread. A detach failure that persists through its bounded same-thread retry +retains the attachment and makes shutdown fail; a later close on another thread +cannot release it. Restart the hosting JVM to recover. A Java cleanup exception +can occur after Spark or Arrow has released only part of a resource. Such failures +retain their owners and the original error until JVM exit. Repeated `close()` calls +report that error without replaying potentially non-idempotent cleanup or claiming +a successful drain. Restart the hosting JVM to recover from a persistent cleanup +failure. + +## Bundle native libraries in JARs + +The Java adapter is shared across operating systems and CPU architectures for a +compatible Spark/Scala/JDK combination. Native code is packaged separately for +`linux-x86_64`, `linux-aarch64`, `macos-x86_64` and `macos-aarch64`. Each platform +JAR contains the library and version, ABI and SHA-256 metadata under +`META-INF/native//`. The artifact version comes from `jni/Cargo.toml`; +it currently is `0.1.0`, independently of the client workspace's Spark version. + +After building the native library and Java adapter, package them with Python 3: + +```bash +crates/spark-connect-server/jvm/build.sh +python3 crates/spark-connect-server/jvm/package-native.py \ + --library "$NATIVE_LIBRARY" \ + --adapter-jar crates/spark-connect-server/.build/jvm/spark-connect-native-bridge.jar \ + --output-dir crates/spark-connect-server/.build/jars +``` + +This produces a shared `spark-connect-native-adapter-0.1.0.jar`, the current +platform's `spark-connect-native-0.1.0-.jar`, and checksum sidecars. +The packager checks the actual ELF or Mach-O header against the selected +platform. Selecting another platform does not cross-compile the library. +Native JAR entries use ZIP DEFLATE compression at level 9. Archive metadata is +normalized for reproducible packaging with the same toolchain; compressed bytes +can differ between zlib versions. + +Every platform, all-platform and macOS universal JAR includes the license and +notice texts for the JNI library's normal, non-proc-macro dependencies across +all four supported Cargo targets. They are stored in `META-INF/LICENSE`, +`META-INF/NOTICE` and `META-INF/native-dependencies.json`. The inventory records +whole-file hashes for the `inputs` in `jvm/native-licenses/dependencies.json`. +Run `python3 scripts/gen_native_binary_licenses.py --check` before packaging or +pushing. If it reports stale assets, run `python3 scripts/gen_native_binary_licenses.py` +and commit the regenerated `jvm/native-licenses/` assets, even if the JNI dependency +closure is unchanged. Packaging checks input and content hashes without running Cargo. + +For example, launch the packaged native library on macOS ARM64: + +```bash +crates/spark-connect-server/jvm/run-engine.sh --native-jar \ + "$PWD/crates/spark-connect-server/.build/jars/spark-connect-native-0.1.0-macos-aarch64.jar" \ + 127.0.0.1:15002 +``` + +For a deployment using already built JARs, pass the platform JAR with +`spark-submit --jars`, use the shared adapter JAR as the application JAR, and +invoke `EmbeddedSparkEngine --bundled [listen-address]`. An existing host can +use `new NativeEngineServer(listenAddress, engine)` with both JARs on its +classpath. The shared adapter JAR must also be available to Spark executors, +where its Java code converts result rows to Arrow. Using it as the application +JAR with `spark-submit` distributes it automatically. A host that creates its +own SparkContext must distribute the adapter through `spark.jars` at context +creation or `SparkContext.addJar` before querying. The host owns this classpath +configuration; the adapter does not change a shared context's submitted JARs. +Executors do not load the native library. The explicit absolute-library-path +constructor remains available. + +The loader detects the host platform, checks matching artifact and ABI versions +and the library checksum, then extracts the library into a private temporary +directory before calling `System.load`. Exactly one matching adapter metadata, +platform metadata and library resource must be visible to the adapter's defining +classloader. Mixing duplicate or incompatible native JARs fails explicitly. +Use one defining Java adapter classloader per JVM. The loader's singleton guard +is scoped to that classloader; loading another adapter copy can load a separate +native library and independent runtime state. +The extracted library stays available for the process lifetime and is removed +at normal JVM exit. Abrupt process termination can leave the temporary directory. +If the default temporary filesystem is mounted `noexec`, set `-Djava.io.tmpdir` +(through `spark.driver.extraJavaOptions` for Spark) to a private directory on a +filesystem that permits loading native libraries. + +The native CI matrix builds and tests each of the four targets on a runner with +the matching architecture, uploads the platform JARs, and combines them into +`spark-connect-native-0.1.0-all.jar` and a macOS universal JAR. Applications still +need the shared Java adapter JAR. The `--combine` command requires all four +platform artifacts: + +```bash +python3 crates/spark-connect-server/jvm/package-native.py \ + --combine /path/to/platform-jars/spark-connect-native-0.1.0-linux-x86_64.jar \ + /path/to/platform-jars/spark-connect-native-0.1.0-linux-aarch64.jar \ + /path/to/platform-jars/spark-connect-native-0.1.0-macos-x86_64.jar \ + /path/to/platform-jars/spark-connect-native-0.1.0-macos-aarch64.jar \ + --output-dir crates/spark-connect-server/.build/jars +``` + +To bundle both macOS architectures in one JAR: + +```bash +python3 crates/spark-connect-server/jvm/package-native.py \ + --combine-macos /path/to/platform-jars/spark-connect-native-0.1.0-macos-x86_64.jar \ + /path/to/platform-jars/spark-connect-native-0.1.0-macos-aarch64.jar \ + --output-dir crates/spark-connect-server/.build/jars +``` + +This requires exactly the two macOS platform artifacts and produces +`spark-connect-native-0.1.0-macos-universal.jar`. It contains both thin native +libraries; the loader selects the JVM's architecture, including under Rosetta, +and extracts only that library. Pass this JAR with `--native-jar` or +`spark-submit --jars` using the same launch commands above. CI checks loading the +same universal JAR on both macOS architectures. -## Build and test +A Universal2 Mach-O library also contains both architectures' machine code, so +combining them into a fat binary does not itself remove those bytes. ZIP +compression reduces the universal JAR's stored and transfer size; the selected +library is decompressed when extracted. + +Linux artifacts are built on Ubuntu 22.04 and require glibc 2.35 or newer; musl +is outside this matrix. macOS artifacts target macOS 11 or newer. These are +development CI artifacts, with no Maven Central publication in this WIP. Native +packaging does not expand the Spark assembly compatibility described above. + +## Supported scope + +The frontend implements execution, configuration, session and operation release, +and the schema, `isLocal` and `isStreaming` analyses. The engine accepts these +relation messages, including their nested inputs: + +- `RANGE`, `LOCAL_RELATION`, `SQL`, `PROJECT`, `FILTER`, `LIMIT`, `OFFSET`, `TAIL`. +- `JOIN`, `SORT`, `AGGREGATE`, `DEDUPLICATE`, `SET_OP`, `SUBQUERY_ALIAS`, `TO_DF`. +- `WITH_COLUMNS_RENAMED`, `WITH_COLUMNS`, `DROP`, `REPARTITION`, + `REPARTITION_BY_EXPRESSION`, `HINT`, `READ`. + +The protobuf `READ` message accepts only its named-table form. This restricts the +message shape, not data access: `SQL` and named-table resolution follow the +hosting Spark session's semantics, including SQL-on-files and built-in reflective +functions when enabled there. Clients can change modifiable session SQL settings, +such as `spark.sql.runSQLOnFiles`, through Config Set. Queries execute with the +hosting process's resource access. This list does not establish support for every +expression or result type inside the accepted messages. Writes, streaming +execution and user-defined functions are outside this prototype's scope. + +Spark-owned internal annotations such as `__autoGeneratedAlias` are intentionally +omitted from returned schemas. User field metadata, including column comments, +non-default string collations, user-defined types and CHAR/VARCHAR return +`UNIMPLEMENTED`; Arrow schema conversion cannot preserve their information. +Duplicate field names within nested structs are also rejected, including structs +inside arrays or map keys and values, because Spark's Arrow conversion renames +them. Duplicate top-level column names and case-distinct nested names are preserved. +CHAR/VARCHAR annotations on string-typed table columns +(`__CHAR_VARCHAR_TYPE_STRING`) are also rejected. Ordinary unaliased computed +columns and aggregates remain supported within the other prototype limits. + +SQL commands such as `SHOW`, `DESCRIBE`, `SET`, `USE` and `CACHE`, Catalog API +relations and `CACHED_LOCAL_RELATION` are also rejected. Streaming relations are +rejected during planning, including `isStreaming` analysis; that analysis does +not return `true` for unsupported streaming inputs. + +Common unsupported client calls include `df.show()`, `df.printSchema()`, +`df.explain()`, `spark.version`, `df.describe()`, `df.summary()`, `df.sample()`, +`df.na.*`, `df.unpivot()`, `spark.read.parquet()` and `df.cache()`. + +Configuration is session scoped. Preparation captures analysis and Arrow +metadata, while physical planning starts on the first fetch. These stages do +not share a transactional configuration snapshot; a concurrent configuration +change can affect later physical planning. Config reads apply the hosting +SparkContext's `spark.redaction.regex` to keys and values; a session setting with +the same name does not replace that policy. This follows newer Spark Config +redaction behavior, which is absent from the pinned Spark 4.2 smoke-test assembly. + +Silent Config Set applies keys sequentially and is not atomic across keys. +Earlier successful writes remain applied if a later key encounters cancellation, +an admission timeout or another operation-level failure. All keys share the +original RPC's admission deadline; a long list does not reset that budget. + +Each partition fetch creates its own Spark SQL execution ID. Multi-partition +queries therefore appear as multiple executions in the Spark UI and SQL metrics. +Initializing a zero-partition plan also creates one execution. The setting +`spark.sql.classic.shuffleDependency.fileCleanup.enabled=true` is unsupported; +prepare and analysis reject it with `UNIMPLEMENTED`, including when the hosting +engine inherits that setting. Per-partition execution completion could otherwise +delete shuffle files still needed by later partition fetches. + +`SPARK_TESTING` or `spark.testing` can make Spark default this setting to `true`. +In test environments, explicitly set +`spark.sql.classic.shuffleDependency.fileCleanup.enabled=false` in the hosting +Spark configuration or the native session before preparing or analyzing queries. + +The frontend's [protocol subset](#protocol-subset) and +[session lifetime limits](#lifetime-and-trust-requirements) also apply. Result +types are limited by the Arrow adapter. This is not full Spark SQL or Spark +Connect parity. + +## Resource and protocol limits + +Each embedded engine has fixed capacities of 64 initialized JVM sessions and +64 active engine tasks. Session and task permit admission share a two-second +deadline per operation; exhausted capacity returns `RESOURCE_EXHAUSTED`. +These limits are not configurable, and there is no idle session expiry. +A client that exits without releasing its session can leave its session permit +reserved. An open, unread result stream can retain a task permit until it is +completed or cancelled and its cleanup finishes. + +Call `spark.stop()` on normal client shutdown. Capacity can recover by releasing +known sessions with `ReleaseSession` and allowing their owned work and deferred +cleanup to drain, or by restarting the server. Releasing an operation or closing +its result stream can also release its task capacity; it does not release the +session. Abandoned sessions whose identifiers are lost remain until a restart. +`ReleaseExecute` with `ReleaseAll` waits up to two seconds for frontend operation +teardown, then returns `DEADLINE_EXCEEDED` if that teardown is still pending. +JNI cancellation and deferred cleanup can continue even after a successful +`ReleaseAll`. Their resource owners and capacity remain retained until cleanup +actually finishes; a timeout does not release that capacity early. + +A cleanup failure while serving keeps its admission permits reserved and is not +retried automatically. Failed cancellation delivery retains its task handle and +task admission even if the query later closes successfully. That capacity stays +unavailable until an explicit `NativeEngineServer.close()` retries the failure +while stopping the server, or the hosting JVM restarts. A transient JNI +attachment failure can recover during that close. A persistent worker +detach failure or a partial Java cleanup failure remains retained until the +hosting JVM restarts. + +Deferred session closes run serially on one process-wide cleanup lane shared by +all embedded engines using the native library. A slow or blocked session close +therefore delays later session cleanup and session-capacity recovery in other +engines too. Session permits remain reserved until the corresponding Java close +completes successfully. Task cancellation has a separate process-wide lane; +a slow cancellation delays other cancellations on that lane. Query cleanup has +its own lane, so session teardown does not directly block task-capacity recovery +from task cancellation or query cleanup. These lanes do not isolate engines from +other shared Spark resources or from a query's own slow cleanup. + +Native server close waits up to ten seconds for shutdown. If work still owns +JNI resources, close reports an incomplete shutdown and retains its handle for +a later retry. A serving error is also reported on the first close; once drain +is confirmed, another close can finalize that retained handle. + +The frontend limits each decoded gRPC request to 64 MiB. JNI byte payloads are +also capped at 64 MiB. Spark's Arrow converter limits rows per batch using +`spark.sql.execution.arrow.maxRecordsPerBatch` and currently uses a 2 MiB byte +estimate before serialization, independently of frontend chunk sizing. +Nonpositive row limits use a 10,000-row fallback. The JNI size +check happens after conversion; these limits do not bound the converter's peak +allocation or guarantee that a large row will fit. + +Arrow framing follows the negotiated [byte-chunking behavior](#arrow-results); +the frontend does not row-split or cap Arrow batches. Without client opt-in, a +whole JNI batch can exceed a raw client's gRPC receive limit and fail at the +client. The independent JNI 64 MiB payload bound applies before frontend +chunking, so a batch rejected by JNI never reaches it. Schema and eager-command +response envelopes retain a separate 3 MiB limit. Execution panics return +`INTERNAL` rather than ending a partial result stream successfully. + +Java uses its standard protobuf recursion limit of 100 nested message levels, +in addition to the frontend's [Prost nesting behavior](#protocol-subset). A +DataFrame operation can add several message levels, so this is not a limit of +100 chained operations. + +Spark executors use `ArrowConverters.toBatchWithSchemaIterator`, the same helper +as the JVM Connect server. Each batch contains a complete Arrow IPC stream and +its row count. The driver fetches one partition's batches at a time, then JNI +copies each Java `byte[]` directly into a Rust-owned `Vec` with one +`GetByteArrayRegion` copy, without an intermediate C payload buffer. Moving that +storage into protobuf `Bytes` and taking byte slices do not copy its payload; +Prost copies it when encoding each outgoing gRPC message. The frontend reads the +IPC schema and preserves the existing IPC bytes, including compressed buffers, +whether it sends a complete batch or negotiated byte chunks. It does not decode, +decompress or re-encode producer IPC to split responses. + +With the default `spark.sql.execution.arrow.compression.codec=none`, partition +transfer uses uncompressed Arrow IPC. Nondefault executor/JNI codec +interoperability remains unvalidated. Network volume and the driver heap needed +for partition output can exceed Spark's compressed UnsafeRow transfer. A +partition can exceed `spark.driver.maxResultSize` even when its compressed rows +would fit; Spark's result-size job aborts are reported as `INTERNAL`. Batch +bounds do not bound the encoded output of one materialized Spark partition or +total query memory. + +Arrow conversion uses Spark's allocator and cleanup behavior. Its off-heap +allocations require executor memory overhead, and Spark can retry conversion +failures according to `spark.task.maxFailures`. Executor Arrow allocation +failures are reported as `INTERNAL`, matching the JVM Connect server; the native +IPC size check reports `RESOURCE_EXHAUSTED`. + +## Historical performance observations + +These historical measurements used an earlier prototype revision with a +different result-framing frontend. They have not been reproduced with the +current byte-chunking path and do not establish its performance. + +A November 17, 2026 comparison used native revision +`23b2eb6538b355987341b385cdfccc3222669e4b`, including the direct Java-array-to-Rust +copy, and the unmodified Spark snapshot identified above. The JVM Connect server +and embedded Rust server ran sequentially on the same Linux ARM64 host. This +compares the two servers; it does not isolate the benefit of the copy change. + +Each latency column is the geometric mean of four fresh-process fork medians. +The last column is the geometric mean of the four matched Rust/JVM ratios; +positive values mean higher Rust latency. + +| Workload | JVM (ms) | Rust (ms) | Rust latency change | +| --- | ---: | ---: | ---: | +| Config Get | 0.837 | 0.429 | -48.8% | +| Range schema analysis | 2.121 | 2.327 | +9.7% | +| Tiny SQL and collect | 33.202 | 47.829 | +44.1% | +| Range/filter aggregate and collect | 49.855 | 78.712 | +57.9% | +| Join aggregate and collect | 72.780 | 92.920 | +27.7% | +| Raw Arrow complete receipt | 246.370 | 382.529 | +55.3% | +| Raw Arrow receipt and table decode | 260.639 | 384.016 | +47.3% | +| Public `DataFrame.toArrow()` | 428.790 | 547.088 | +27.6% | +| Concurrent Config per-call latency | 0.641 | 0.615 | -4.1% | + +The public `toArrow()` matched-pair increases ranged from 24.0% to 32.6%. +Concurrent Config was slower in Rust in three of four pairs despite its lower +geometric mean of fork medians; whole-round throughput was 23.3% lower. These +results establish neither a general native speedup nor improved concurrent +capacity. + +The four matched pairs ran in AB, BA, BA, AB order, with A = JVM and B = Rust. +Each server and official Python client started in a fresh process. Metadata used +200 warmups and 500 measured calls per fork; queries and Arrow actions used +10 warmups and 30 measured calls; concurrent Config used one warmup and three +measured rounds of 1,000 calls across eight channels. All 13,152 action records +passed their recorded checks, including 9,464 measured records and 3,688 warmups. +Every client/session cleanup stage, server shutdown and listener closure passed. +Arrow inputs had 8 million int64 rows, range/filter inputs 1 million rows, and +join inputs 100,000 rows. + +The host had eight visible ARM Neoverse-V2 CPUs and 61.6 GiB visible memory. +Servers used CPUs 2-5 and clients launched with affinity to CPUs 6-7. Both servers +used `local[2]`, a 1 GiB heap, two input/shuffle partitions, UTC, 10,000 Arrow rows +per batch and IPC codec `none`. The runtime was JDK 21.0.13, Python 3.12.3, +official PySpark client 4.2.0, PyArrow 26.0.0 and gRPC 1.84.0. JNI checking was +disabled during timing. Other host services remained present; affinity did not +reserve exclusive CPUs. + +Raw receipt drains all protobuf responses through completion, with IPC decoding +outside timing. Public `toArrow()` includes the official client's response +handling, IPC decode, table materialization and schema cast. Validation and +disposal occur outside timing. These are separate sampled actions; subtracting +their times does not measure a pipeline stage. Both servers returned exactly +65,248,000 IPC bytes in 800 record batches per raw Arrow action. Response counts +were 806-810 for JVM and 802 for Rust. Public `collect()` includes Python Row +materialization; tiny SQL also includes `.sql()` construction. + +This is one host with local executors and loopback transport. Within-process +calls are correlated; no significance, distributed-network, wide-row, compressed +IPC, GC or retained-memory conclusion follows. The raw checks cover aggregate +count, schema, nulls, sum, minimum and maximum, rather than exact order or +uniqueness. Automatic asynchronous operation release remains best effort; +explicit session/client/server cleanup and listener closure passed separately. + +Arrow producer output-sizing optimization and producer/off-heap API redesign +remain follow-up work. + +## Validation + +### Frontend-only checks Use the workspace Rust toolchain and a `protoc` compiler with proto3 optional support. From the repository root: @@ -143,3 +604,86 @@ cover wire framing, schema and IPC handling, backend error propagation, backpressure, operation ownership, cancellation and session lifetime. Fixture relations select predefined outputs; they do not implement Spark operators or establish query semantics or Spark server compatibility. + +### JNI and live Spark checks + +The deterministic Rust frontend/engine and C bridge tests do not need a +running Spark distribution. JNI compilation still requires `JAVA_HOME`: + +```bash +cargo test -p spark-connect-server -p spark-connect-jni +cargo clippy -p spark-connect-server -p spark-connect-jni --all-targets --no-deps -- -D warnings +crates/spark-connect-server/jni/test-c-bridge.sh +crates/spark-connect-server/jni/test-c-bridge.sh --sanitize +python3 -B crates/spark-connect-server/jvm/test_acceptance_guards.py +python3 -B crates/spark-connect-server/jvm/test_package_native.py +crates/spark-connect-server/jvm/test-native-loader.sh +``` + +The C bridge runner requires macOS or Linux. Sanitizer mode also requires a +compiler with address and undefined-behavior sanitizer support. + +With a matching Spark assembly and `NATIVE_LIBRARY` set, run live checks: + +```bash +crates/spark-connect-server/jvm/test-engine.sh \ + --library "$NATIVE_LIBRARY" --python /path/to/pyspark-python +``` + +To include the official Rust client example in the live checks, build it and +pass the optional `--rust-example` argument: + +```bash +cargo build --release -p spark-connect-server --example rust_client +crates/spark-connect-server/jvm/test-engine.sh \ + --library "$NATIVE_LIBRARY" --python /path/to/pyspark-python \ + --rust-example "$PWD/target/release/examples/rust_client" +``` + +This compiles Java tests, checks direct planning, IPC batch transfer, native +lifecycle and cleanup-failure retention with `-Xcheck:jni`, then runs official +PySpark and raw gRPC acceptance against `EmbeddedSparkEngine`. It checks JVM +listener ownership, shutdown and +listener closure. The Python interpreter must have a compatible official PySpark +Connect client, gRPC and PyArrow installed. Listener checks prefer `lsof` when it +is installed and otherwise use Linux `/proc`; macOS requires `lsof`. +`NATIVE_TEST_PYTHON` (default `python3`) selects the interpreter for the standard +library guard tests and orchestration. `--python` selects the official PySpark +client and worker interpreter; it defaults to the orchestration interpreter. +These interoperability tests complement the Rust tests. The default report is +`crates/spark-connect-server/.build/jvm-native/engine-acceptance.json`; pass +`--output /absolute/report.json` to choose another path. + +Pass `--native-jar /absolute/platform-or-all.jar` as well as `--library` to run +the client acceptance through classpath extraction. The lifecycle checks still +exercise explicit-path loading. This mode checks the extracted library bytes +against `--library`, and verifies that the extracted file and private directory +are removed after JVM exit. + +The native CI workflow runs the deterministic Rust tests, strict native Clippy, +acceptance-runner guards, packaging tests and C ownership checks on its platform +matrix. It builds release libraries, packages platform/all-platform JARs and +tests actual native loading with a JDK-only loader suite. A separate Linux job +restores a Spark assembly cache keyed on the pinned revision above, platform and +JDK, or builds that revision from source on a cache miss. It compiles the +Spark-dependent adapter and runs `jvm/test-jni-smoke.sh` against the real native +library. The smoke test checks a range query's schema and Arrow values, +negotiated 1 KiB byte chunks with reassembly, a whole row larger than 3 MiB without +chunking opt-in, `ResultComplete`, session release, native close, listener rebind +and shared SparkContext survival under `-Xcheck:jni`. It does not exercise Arrow +IPC input or establish compatibility with other Spark revisions. The broader acceptance +suite still requires a selected, compatible assembly and local validation. Its +recorded full run used the separate Arrow input lifecycle repair described under +build requirements; the CI smoke does not apply that repair. + +Local validation on macOS ARM64 covered the frontend and JNI tests, client +release tests and integration-test compilation, acceptance-runner guards, +license generators and packaging. Rustfmt, workspace Clippy, native tests on +Rust 1.100 and MSRV 1.88, C bridge checks with and without sanitizers, native +loader checks and the real JNI smoke also passed. Full client acceptance and +the official Rust client example used the separately repaired Spark runtime +described under [Build requirements and compatibility](#build-requirements-and-compatibility). +The smoke passed on the unmodified pinned snapshot. These local results do not +establish Linux or x86-64 compatibility; hosted CI and nondefault executor/JNI +IPC compression remain separate validation requirements. Performance was not +remeasured. diff --git a/crates/spark-connect-server/examples/jvm_client.py b/crates/spark-connect-server/examples/jvm_client.py new file mode 100644 index 00000000..d2b3422f --- /dev/null +++ b/crates/spark-connect-server/examples/jvm_client.py @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Run against the Rust listener embedded in the Spark JVM by run-engine.sh.""" + +import os + +from pyspark.sql import SparkSession + +spark = SparkSession.builder.remote( + os.environ.get("SPARK_REMOTE", "sc://127.0.0.1:15002") +).getOrCreate() +try: + left = spark.range(5).withColumnRenamed("id", "key") + right = spark.range(3).withColumnRenamed("id", "key") + print("JVM DataFrame join:", left.join(right, "key").orderBy("key").collect()) + print( + "JVM complex types:", + spark.sql( + "SELECT CAST('12.34' AS DECIMAL(10,2)) AS amount, array(1,2,3) AS items" + ).collect(), + ) +finally: + spark.stop() diff --git a/crates/spark-connect-server/examples/rust_client.rs b/crates/spark-connect-server/examples/rust_client.rs new file mode 100644 index 00000000..25213ae9 --- /dev/null +++ b/crates/spark-connect-server/examples/rust_client.rs @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Start the embedded JVM server, then run +//! `cargo run -p spark-connect-server --example rust_client`. + +use spark_connect::functions as f; +use spark_connect::{col, SparkSession}; + +fn main() -> Result<(), Box> { + let remote = std::env::var("SPARK_REMOTE").unwrap_or_else(|_| "sc://127.0.0.1:15002".into()); + let spark = SparkSession::builder().remote(&remote).get_or_create()?; + + let totals = spark + .range(1_000)? + .agg(vec![f::sum(col("id")).alias("total").expression().clone()]); + let totals = totals.collect()?; + assert_eq!(totals.len(), 1); + assert_eq!( + totals[0] + .get_by_name("total") + .and_then(|value| value.as_i64()), + Some(499_500) + ); + let answer = spark.sql("SELECT 1 + 1 AS answer")?.collect()?; + assert_eq!(answer.len(), 1); + assert_eq!( + answer[0] + .get_by_name("answer") + .and_then(|value| value.as_i64()), + Some(2) + ); + println!("Range aggregate: {totals:?}"); + println!("SQL: {answer:?}"); + + spark.stop()?; + Ok(()) +} diff --git a/crates/spark-connect-server/jni/Cargo.toml b/crates/spark-connect-server/jni/Cargo.toml new file mode 100644 index 00000000..b8c384e8 --- /dev/null +++ b/crates/spark-connect-server/jni/Cargo.toml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "spark-connect-jni" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false +description = "JNI host for the Spark Connect Rust server and the direct Spark JVM engine" + +[lib] +crate-type = ["cdylib"] + +[lints] +workspace = true + +[dependencies] +spark-connect-server = { path = ".." } +futures.workspace = true +arrow.workspace = true +prost.workspace = true +spark-connect-proto = { package = "apache-spark-connect-proto", path = "../../spark-connect-proto" } +libc.workspace = true +tokio = { workspace = true, features = ["net"] } +tonic.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } + +[build-dependencies] +cc.workspace = true diff --git a/crates/spark-connect-server/jni/build.rs b/crates/spark-connect-server/jni/build.rs new file mode 100644 index 00000000..2e586d38 --- /dev/null +++ b/crates/spark-connect-server/jni/build.rs @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +fn main() { + let platform = match std::env::var("CARGO_CFG_TARGET_OS").unwrap().as_str() { + "macos" => "darwin", + "linux" => "linux", + other => panic!( + "unsupported JNI target: {other}; spark-connect-jni requires macOS or Linux with pthreads" + ), + }; + let java_home = std::env::var_os("JAVA_HOME") + .expect("Set JAVA_HOME to a JDK with JNI headers, or exclude spark-connect-jni from workspace commands"); + let include = std::path::PathBuf::from(java_home).join("include"); + cc::Build::new() + .file("native/bridge.c") + .include(&include) + .include(include.join(platform)) + .std("c11") + .warnings(true) + .compile("spark_connect_jni_bridge"); + println!("cargo:rerun-if-env-changed=JAVA_HOME"); + println!("cargo:rerun-if-changed=native/bridge.c"); + println!("cargo:rerun-if-changed=native/bridge.h"); +} diff --git a/crates/spark-connect-server/jni/native/bridge.c b/crates/spark-connect-server/jni/native/bridge.c new file mode 100644 index 00000000..1cca55c3 --- /dev/null +++ b/crates/spark-connect-server/jni/native/bridge.c @@ -0,0 +1,1006 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "bridge.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct { + JavaVM *vm; + jobject loader; + jclass thread_class; + jmethodID current_thread; + jmethodID get_loader; + jmethodID set_loader; + atomic_uint references; +} JavaContext; + +typedef struct { + JavaContext *context; + JNIEnv *env; + jobject thread; + jobject previous_loader; + int attached; + int framed; + int loader_changed; +} JavaScope; + +typedef struct { + JavaContext *context; + pthread_t owner; + int attached; +} EngineThread; + +extern int64_t spark_connect_engine_start(const uint8_t *, size_t, void *, BridgeBuffer *); +extern int spark_connect_address(int64_t, BridgeBuffer *, BridgeBuffer *); +extern int spark_connect_stop(int64_t, BridgeBuffer *); + +static void clear_buffer(BridgeBuffer *buffer) { + if (buffer != NULL) { + buffer->data = NULL; + buffer->len = 0; + } +} + +static void set_error(BridgeBuffer *error, const char *message) { + if (error == NULL || error->data != NULL) { + return; + } + size_t len = strlen(message); + uint8_t *data = malloc(len == 0 ? 1 : len); + if (data != NULL) { + memcpy(data, message, len); + error->data = data; + error->len = len; + } +} + +void spark_connect_buffer_free(BridgeBuffer *buffer) { + if (buffer != NULL) { + free(buffer->data); + clear_buffer(buffer); + } +} + +/* GetStringUTFChars uses modified UTF-8, so encode UTF-16 explicitly instead. */ +static int string_buffer(JNIEnv *env, jstring value, BridgeBuffer *out) { + jsize len = (*env)->GetStringLength(env, value); + const jchar *chars = (*env)->GetStringChars(env, value, NULL); + if (chars == NULL) { + return -1; + } + uint8_t *data = malloc((size_t) len * 3 + 1); + if (data == NULL) { + (*env)->ReleaseStringChars(env, value, chars); + return -1; + } + size_t written = 0; + for (jsize i = 0; i < len; i++) { + uint32_t code = chars[i]; + if (code >= 0xd800 && code <= 0xdbff && i + 1 < len + && chars[i + 1] >= 0xdc00 && chars[i + 1] <= 0xdfff) { + code = 0x10000 + ((code - 0xd800) << 10) + chars[++i] - 0xdc00; + } else if (code >= 0xd800 && code <= 0xdfff) { + code = 0xfffd; + } + if (code < 0x80) { + data[written++] = (uint8_t) code; + } else if (code < 0x800) { + data[written++] = (uint8_t) (0xc0 | (code >> 6)); + data[written++] = (uint8_t) (0x80 | (code & 0x3f)); + } else if (code < 0x10000) { + data[written++] = (uint8_t) (0xe0 | (code >> 12)); + data[written++] = (uint8_t) (0x80 | ((code >> 6) & 0x3f)); + data[written++] = (uint8_t) (0x80 | (code & 0x3f)); + } else { + data[written++] = (uint8_t) (0xf0 | (code >> 18)); + data[written++] = (uint8_t) (0x80 | ((code >> 12) & 0x3f)); + data[written++] = (uint8_t) (0x80 | ((code >> 6) & 0x3f)); + data[written++] = (uint8_t) (0x80 | (code & 0x3f)); + } + } + (*env)->ReleaseStringChars(env, value, chars); + out->data = data; + out->len = written; + return 0; +} + +static int exception_error(JNIEnv *env, BridgeBuffer *error) { + if (!(*env)->ExceptionCheck(env)) { + return 0; + } + jthrowable throwable = (*env)->ExceptionOccurred(env); + (*env)->ExceptionClear(env); + jclass type = throwable == NULL ? NULL : (*env)->GetObjectClass(env, throwable); + jmethodID to_string = type == NULL ? NULL + : (*env)->GetMethodID(env, type, "toString", "()Ljava/lang/String;"); + jstring description = to_string == NULL ? NULL + : (*env)->CallObjectMethod(env, throwable, to_string); + if (!(*env)->ExceptionCheck(env) && description != NULL && error != NULL + && error->data == NULL) { + (void) string_buffer(env, description, error); + } + if ((*env)->ExceptionCheck(env)) { + (*env)->ExceptionClear(env); + } + if (description != NULL) (*env)->DeleteLocalRef(env, description); + if (type != NULL) (*env)->DeleteLocalRef(env, type); + if (throwable != NULL) (*env)->DeleteLocalRef(env, throwable); + set_error(error, "Java bridge operation failed"); + return -1; +} + +static void scope_restore_loader(JavaScope *scope) { + JNIEnv *env = scope->env; + if ((*env)->ExceptionCheck(env)) (*env)->ExceptionClear(env); + if (scope->loader_changed) { + (*env)->CallVoidMethod(env, scope->thread, scope->context->set_loader, + scope->previous_loader); + if ((*env)->ExceptionCheck(env)) (*env)->ExceptionClear(env); + scope->loader_changed = 0; + } +} + +static void scope_leave(JavaScope *scope) { + if (scope->env == NULL) return; + JNIEnv *env = scope->env; + scope_restore_loader(scope); + if (scope->previous_loader != NULL) (*env)->DeleteLocalRef(env, scope->previous_loader); + if (scope->thread != NULL) (*env)->DeleteLocalRef(env, scope->thread); + if (scope->framed) (*env)->PopLocalFrame(env, NULL); + if (scope->attached) (*scope->context->vm)->DetachCurrentThread(scope->context->vm); + scope->env = NULL; +} + +static int scope_enter(JavaContext *context, JavaScope *scope, BridgeBuffer *error) { + memset(scope, 0, sizeof(*scope)); + scope->context = context; + jint state = (*context->vm)->GetEnv(context->vm, (void **) &scope->env, JNI_VERSION_1_8); + if (state == JNI_EDETACHED) { + if ((*context->vm)->AttachCurrentThreadAsDaemon( + context->vm, (void **) &scope->env, NULL) != JNI_OK) { + scope->env = NULL; + set_error(error, "cannot attach Rust worker to JVM"); + return -1; + } + scope->attached = 1; + } else if (state != JNI_OK) { + scope->env = NULL; + set_error(error, "cannot access JVM from Rust worker"); + return -1; + } + JNIEnv *env = scope->env; + if ((*env)->PushLocalFrame(env, 32) < 0) { + (void) exception_error(env, error); + set_error(error, "cannot allocate Java local reference frame"); + goto failed; + } + scope->framed = 1; + scope->thread = (*env)->CallStaticObjectMethod(env, context->thread_class, + context->current_thread); + if (exception_error(env, error) != 0 || scope->thread == NULL) goto failed; + scope->previous_loader = (*env)->CallObjectMethod(env, scope->thread, context->get_loader); + if (exception_error(env, error) != 0) goto failed; + if (!(*env)->IsSameObject(env, scope->previous_loader, context->loader)) { + (*env)->CallVoidMethod(env, scope->thread, context->set_loader, context->loader); + if (exception_error(env, error) != 0) goto failed; + scope->loader_changed = 1; + } + return 0; +failed: + set_error(error, "cannot prepare JVM worker context"); + scope_leave(scope); + return -1; +} + +static void context_release(JavaContext *context) { + if (atomic_fetch_sub_explicit(&context->references, 1, memory_order_acq_rel) != 1) return; + // The factory cannot be freed while worker guards exist. Consequently detach + // never owns the last context reference, and must not reattach just to destroy it. + // Preserve an unexpected orphan rather than deleting globals without a valid env. + atomic_store_explicit(&context->references, 1, memory_order_release); + fputs("JNI cleanup is incomplete; retained cleanup failed: orphaned Java worker context\n", + stderr); +} + +static jbyteArray byte_array(JNIEnv *env, const uint8_t *data, size_t len, + BridgeBuffer *error) { + if (len > INT_MAX || (len != 0 && data == NULL)) { + set_error(error, "Java bridge byte array is too large or invalid"); + return NULL; + } + jbyteArray array = (*env)->NewByteArray(env, (jsize) len); + if (exception_error(env, error) != 0 || array == NULL) return NULL; + if (len != 0) (*env)->SetByteArrayRegion(env, array, 0, (jsize) len, (const jbyte *) data); + if (exception_error(env, error) != 0) { + (*env)->DeleteLocalRef(env, array); + return NULL; + } + return array; +} + +static jstring utf8_string(JNIEnv *env, const uint8_t *data, size_t len, + BridgeBuffer *error) { + jbyteArray bytes = byte_array(env, data, len, error); + if (bytes == NULL) return NULL; + jclass type = (*env)->FindClass(env, "java/lang/String"); + jmethodID constructor = type == NULL ? NULL + : (*env)->GetMethodID(env, type, "", "([BLjava/lang/String;)V"); + jstring encoding = constructor == NULL ? NULL : (*env)->NewStringUTF(env, "UTF-8"); + jstring value = NULL; + if (exception_error(env, error) == 0 && encoding != NULL) { + value = (*env)->NewObject(env, type, constructor, bytes, encoding); + if (exception_error(env, error) != 0) value = NULL; + } + if (encoding != NULL) (*env)->DeleteLocalRef(env, encoding); + if (type != NULL) (*env)->DeleteLocalRef(env, type); + (*env)->DeleteLocalRef(env, bytes); + return value; +} + +static int array_buffer(JNIEnv *env, jbyteArray array, BridgeBuffer *out, + BridgeBuffer *error) { + jsize len = (*env)->GetArrayLength(env, array); + if (exception_error(env, error) != 0) return -1; + uint8_t *data = malloc(len == 0 ? 1 : (size_t) len); + if (data == NULL) { + set_error(error, "cannot allocate Java bridge buffer"); + return -1; + } + if (len != 0) (*env)->GetByteArrayRegion(env, array, 0, len, (jbyte *) data); + if (exception_error(env, error) != 0) { + free(data); + return -1; + } + out->data = data; + out->len = (size_t) len; + return 0; +} + +static void throw_error(JNIEnv *env, BridgeBuffer *error) { + if ((*env)->ExceptionCheck(env)) { + spark_connect_buffer_free(error); + return; + } + if (error->data == NULL) set_error(error, "native Spark Connect operation failed"); + jstring message = utf8_string(env, error->data, error->len, NULL); + jclass type = (*env)->FindClass(env, "java/lang/IllegalStateException"); + if (!(*env)->ExceptionCheck(env) && message != NULL && type != NULL) { + jmethodID constructor = (*env)->GetMethodID(env, type, "", "(Ljava/lang/String;)V"); + if (!(*env)->ExceptionCheck(env) && constructor != NULL) { + jthrowable throwable = (*env)->NewObject(env, type, constructor, message); + if (!(*env)->ExceptionCheck(env) && throwable != NULL) { + (*env)->Throw(env, throwable); + (*env)->DeleteLocalRef(env, throwable); + } + } + } + // Message conversion can clear its allocation exception while returning NULL. + // ThrowNew needs no Java message object and leaves any fallback OOM pending. + if (!(*env)->ExceptionCheck(env) && type != NULL) { + (void) (*env)->ThrowNew(env, type, "native Spark Connect operation failed"); + } + if (message != NULL) (*env)->DeleteLocalRef(env, message); + if (type != NULL) (*env)->DeleteLocalRef(env, type); + spark_connect_buffer_free(error); +} + +void *spark_connect_engine_native_address(void *raw_env, void *clazz, int64_t id) { + (void) clazz; + JNIEnv *env = raw_env; + BridgeBuffer address = {0}; + BridgeBuffer error = {0}; + if (spark_connect_address(id, &address, &error) != 0) { + spark_connect_buffer_free(&address); + throw_error(env, &error); + return NULL; + } + jbyteArray result = byte_array(env, address.data, address.len, &error); + spark_connect_buffer_free(&address); + if (result == NULL) throw_error(env, &error); + else spark_connect_buffer_free(&error); + return result; +} + +void spark_connect_engine_native_stop(void *raw_env, void *clazz, int64_t id) { + (void) clazz; + BridgeBuffer error = {0}; + if (spark_connect_stop(id, &error) != 0) throw_error(raw_env, &error); + else spark_connect_buffer_free(&error); +} + +/* Typed JVM query engine implementation. */ +typedef enum { + ENGINE_FACTORY, + ENGINE_TASK, + ENGINE_SESSION, + ENGINE_QUERY, + ENGINE_EXCEPTION, + ENGINE_BATCH, + ENGINE_CLASS_COUNT +} EngineKind; + +typedef struct { + JavaContext *context; + atomic_uint references; + jclass classes[ENGINE_CLASS_COUNT]; + jmethodID close[ENGINE_EXCEPTION]; + jmethodID new_task; + jmethodID open_session; + jmethodID prepare; + jmethodID analyze; + jmethodID config; + jmethodID schema; + jmethodID next_ipc; + jmethodID batch_data; + jmethodID batch_row_count; + jmethodID cancel; + jmethodID exception_code; +} EngineApi; + +typedef struct { + EngineApi *api; + jobject object; + EngineKind kind; +} EngineHandle; + +static void engine_error_bytes(BridgeBuffer *error, int code, + const uint8_t *message, size_t len) { + if (error == NULL || error->data != NULL) return; + char prefix[40]; + int prefix_len = snprintf(prefix, sizeof(prefix), "JVM_ENGINE_ERROR:%d:", code); + if (prefix_len < 0 || (size_t) prefix_len >= sizeof(prefix) + || len > SIZE_MAX - (size_t) prefix_len) return; + uint8_t *data = malloc((size_t) prefix_len + len); + if (data == NULL) return; + memcpy(data, prefix, (size_t) prefix_len); + if (len != 0) memcpy(data + prefix_len, message, len); + error->data = data; + error->len = (size_t) prefix_len + len; +} + +static void engine_error(BridgeBuffer *error, int code, const char *message) { + engine_error_bytes(error, code, (const uint8_t *) message, strlen(message)); +} + +/* Shared low-level helpers report plain text. Frame it at this ABI's boundary. */ +static void engine_normalize_error(BridgeBuffer *error, const char *fallback) { + static const char prefix[] = "JVM_ENGINE_ERROR:"; + if (error == NULL) return; + if (error->len >= sizeof(prefix) - 1 && error->data != NULL + && memcmp(error->data, prefix, sizeof(prefix) - 1) == 0) return; + BridgeBuffer previous = *error; + clear_buffer(error); + if (previous.data != NULL) { + engine_error_bytes(error, 13, previous.data, previous.len); + spark_connect_buffer_free(&previous); + } else { + engine_error(error, 13, fallback); + } +} + +static int engine_exception(JNIEnv *env, EngineApi *api, BridgeBuffer *error) { + if (!(*env)->ExceptionCheck(env)) return 0; + jthrowable throwable = (*env)->ExceptionOccurred(env); + (*env)->ExceptionClear(env); + int code = 13; + int typed = throwable != NULL && api != NULL + && api->classes[ENGINE_EXCEPTION] != NULL && api->exception_code != NULL + && (*env)->IsInstanceOf(env, throwable, api->classes[ENGINE_EXCEPTION]); + if (typed) { + jint reported = (*env)->CallIntMethod(env, throwable, api->exception_code); + if ((*env)->ExceptionCheck(env)) (*env)->ExceptionClear(env); + else if (reported >= 1 && reported <= 16) code = reported; + } + jclass type = throwable == NULL ? NULL : (*env)->GetObjectClass(env, throwable); + if ((*env)->ExceptionCheck(env)) (*env)->ExceptionClear(env); + jmethodID describe = type == NULL ? NULL : (*env)->GetMethodID(env, type, + typed ? "getMessage" : "toString", "()Ljava/lang/String;"); + if ((*env)->ExceptionCheck(env)) (*env)->ExceptionClear(env); + jstring description = describe == NULL ? NULL + : (*env)->CallObjectMethod(env, throwable, describe); + if ((*env)->ExceptionCheck(env)) { + (*env)->ExceptionClear(env); + description = NULL; + } + BridgeBuffer message = {0}; + if (description != NULL) (void) string_buffer(env, description, &message); + if ((*env)->ExceptionCheck(env)) (*env)->ExceptionClear(env); + if (message.data != NULL) engine_error_bytes(error, code, message.data, message.len); + else engine_error(error, code, "Java engine operation failed"); + spark_connect_buffer_free(&message); + if (description != NULL) (*env)->DeleteLocalRef(env, description); + if (type != NULL) (*env)->DeleteLocalRef(env, type); + if (throwable != NULL) (*env)->DeleteLocalRef(env, throwable); + return -1; +} + +static void engine_delete_classes(JNIEnv *env, EngineApi *api) { + for (int i = 0; i < ENGINE_CLASS_COUNT; i++) { + if (api->classes[i] != NULL) (*env)->DeleteGlobalRef(env, api->classes[i]); + } +} + +static void engine_api_release(JNIEnv *env, EngineApi *api) { + if (atomic_fetch_sub_explicit(&api->references, 1, memory_order_acq_rel) != 1) return; + // Startup already supplies a valid Java environment. Do not risk another + // attachment after disposing the last handle. + engine_delete_classes(env, api); + JavaContext *context = api->context; + if (atomic_fetch_sub_explicit(&context->references, 1, memory_order_acq_rel) == 1) { + if (context->loader != NULL) (*env)->DeleteGlobalRef(env, context->loader); + (*env)->DeleteGlobalRef(env, context->thread_class); + free(context); + } + free(api); +} + +static jclass engine_class(JNIEnv *env, EngineApi *api, const char *name, + BridgeBuffer *error) { + jclass local = (*env)->FindClass(env, name); + if (engine_exception(env, api, error) != 0 || local == NULL) return NULL; + jclass global = (*env)->NewGlobalRef(env, local); + (*env)->DeleteLocalRef(env, local); + if (engine_exception(env, api, error) != 0) return NULL; + return global; +} + +static int engine_method(JNIEnv *env, EngineApi *api, EngineKind kind, + jmethodID *out, const char *name, const char *signature, BridgeBuffer *error) { + *out = (*env)->GetMethodID(env, api->classes[kind], name, signature); + if (engine_exception(env, api, error) != 0 || *out == NULL) return -1; + return 0; +} + +static EngineHandle *engine_handle(JNIEnv *env, EngineApi *api, jobject object, + EngineKind kind, BridgeBuffer *error) { + EngineHandle *handle = calloc(1, sizeof(*handle)); + if (handle != NULL) { + handle->object = (*env)->NewGlobalRef(env, object); + (void) engine_exception(env, api, error); + } + if (handle == NULL || handle->object == NULL) { + engine_error(error, 13, "cannot retain Java engine handle"); + (*env)->CallVoidMethod(env, object, api->close[kind]); + (void) engine_exception(env, api, NULL); + free(handle); + return NULL; + } + handle->api = api; + handle->kind = kind; + atomic_fetch_add_explicit(&api->references, 1, memory_order_relaxed); + return handle; +} + +static EngineHandle *engine_factory(JNIEnv *env, jobject factory, BridgeBuffer *error) { + EngineApi *api = calloc(1, sizeof(*api)); + JavaContext *context = calloc(1, sizeof(*context)); + if (api == NULL || context == NULL) { + free(api); + free(context); + engine_error(error, 13, "cannot allocate Java engine context"); + return NULL; + } + api->context = context; + atomic_init(&api->references, 1); + atomic_init(&context->references, 1); + if ((*env)->GetJavaVM(env, &context->vm) != JNI_OK) goto failed; + context->thread_class = engine_class(env, api, "java/lang/Thread", error); + if (context->thread_class == NULL) goto failed; + context->current_thread = (*env)->GetStaticMethodID(env, context->thread_class, + "currentThread", "()Ljava/lang/Thread;"); + if (engine_exception(env, api, error) != 0 || context->current_thread == NULL) goto failed; + context->get_loader = (*env)->GetMethodID(env, context->thread_class, + "getContextClassLoader", "()Ljava/lang/ClassLoader;"); + if (engine_exception(env, api, error) != 0 || context->get_loader == NULL) goto failed; + context->set_loader = (*env)->GetMethodID(env, context->thread_class, + "setContextClassLoader", "(Ljava/lang/ClassLoader;)V"); + if (engine_exception(env, api, error) != 0 || context->set_loader == NULL) goto failed; + jobject current = (*env)->CallStaticObjectMethod(env, context->thread_class, + context->current_thread); + if (engine_exception(env, api, error) != 0 || current == NULL) goto failed; + jobject loader = (*env)->CallObjectMethod(env, current, context->get_loader); + (*env)->DeleteLocalRef(env, current); + if (engine_exception(env, api, error) != 0) goto failed; + if (loader != NULL) { + context->loader = (*env)->NewGlobalRef(env, loader); + (*env)->DeleteLocalRef(env, loader); + if (engine_exception(env, api, error) != 0 || context->loader == NULL) goto failed; + } + jclass factory_class = (*env)->GetObjectClass(env, factory); + if (engine_exception(env, api, error) != 0 || factory_class == NULL) goto failed; + api->classes[ENGINE_FACTORY] = (*env)->NewGlobalRef(env, factory_class); + (*env)->DeleteLocalRef(env, factory_class); + if (engine_exception(env, api, error) != 0 || api->classes[ENGINE_FACTORY] == NULL) goto failed; + api->classes[ENGINE_TASK] = engine_class(env, api, + "org/apache/spark/sql/connect/nativebridge/engine/SparkJvmEngine$Task", error); + if (api->classes[ENGINE_TASK] == NULL) goto failed; + api->classes[ENGINE_SESSION] = engine_class(env, api, + "org/apache/spark/sql/connect/nativebridge/engine/SparkJvmEngine$Session", error); + if (api->classes[ENGINE_SESSION] == NULL) goto failed; + api->classes[ENGINE_QUERY] = engine_class(env, api, + "org/apache/spark/sql/connect/nativebridge/engine/SparkJvmEngine$Query", error); + if (api->classes[ENGINE_QUERY] == NULL) goto failed; + api->classes[ENGINE_BATCH] = engine_class(env, api, + "org/apache/spark/sql/connect/nativebridge/engine/NativeIpcBatch", error); + if (api->classes[ENGINE_BATCH] == NULL) goto failed; + api->classes[ENGINE_EXCEPTION] = engine_class(env, api, + "org/apache/spark/sql/connect/nativebridge/engine/SparkJvmEngine$EngineException", error); + if (api->classes[ENGINE_EXCEPTION] == NULL) goto failed; + for (int i = ENGINE_FACTORY; i < ENGINE_EXCEPTION; i++) { + if (engine_method(env, api, (EngineKind) i, &api->close[i], "close", "()V", error) != 0) + goto failed; + } + if (engine_method(env, api, ENGINE_EXCEPTION, &api->exception_code, "code", "()I", error) != 0 + || engine_method(env, api, ENGINE_FACTORY, &api->new_task, "newTask", + "(Ljava/lang/String;)Lorg/apache/spark/sql/connect/nativebridge/engine/SparkJvmEngine$Task;", + error) != 0 + || engine_method(env, api, ENGINE_FACTORY, &api->open_session, "openSession", + "(Ljava/lang/String;Ljava/lang/String;Lorg/apache/spark/sql/connect/nativebridge/engine/SparkJvmEngine$Task;)Lorg/apache/spark/sql/connect/nativebridge/engine/SparkJvmEngine$Session;", + error) != 0 + || engine_method(env, api, ENGINE_SESSION, &api->prepare, "prepare", + "([BLorg/apache/spark/sql/connect/nativebridge/engine/SparkJvmEngine$Task;)Lorg/apache/spark/sql/connect/nativebridge/engine/SparkJvmEngine$Query;", + error) != 0 + || engine_method(env, api, ENGINE_SESSION, &api->analyze, "analyze", + "(I[BLorg/apache/spark/sql/connect/nativebridge/engine/SparkJvmEngine$Task;)[B", + error) != 0 + || engine_method(env, api, ENGINE_SESSION, &api->config, "config", + "(I[BLorg/apache/spark/sql/connect/nativebridge/engine/SparkJvmEngine$Task;)[B", + error) != 0 + || engine_method(env, api, ENGINE_TASK, &api->cancel, "cancel", "()V", error) != 0 + || engine_method(env, api, ENGINE_QUERY, &api->schema, "schemaIpc", "()[B", error) != 0 + || engine_method(env, api, ENGINE_QUERY, &api->next_ipc, "nextIpcBatch", + "()Lorg/apache/spark/sql/connect/nativebridge/engine/NativeIpcBatch;", error) != 0 + || engine_method(env, api, ENGINE_BATCH, &api->batch_data, "data", "()[B", error) != 0 + || engine_method(env, api, ENGINE_BATCH, &api->batch_row_count, "rowCount", "()J", error) != 0) + goto failed; + EngineHandle *handle = engine_handle(env, api, factory, ENGINE_FACTORY, error); + engine_api_release(env, api); + return handle; +failed: + (void) engine_exception(env, api, error); + engine_normalize_error(error, "cannot initialize Java engine ABI"); + engine_delete_classes(env, api); + if (context->loader != NULL) (*env)->DeleteGlobalRef(env, context->loader); + if (context->thread_class != NULL) (*env)->DeleteGlobalRef(env, context->thread_class); + free(context); + free(api); + return NULL; +} + +static int engine_check(EngineHandle *handle, EngineKind kind, BridgeBuffer *error) { + if (handle != NULL && handle->kind == kind) return 0; + engine_error(error, 13, "invalid Java engine handle"); + return -1; +} + +static int engine_task_check(EngineHandle *handle, EngineHandle *task, BridgeBuffer *error) { + if (engine_check(task, ENGINE_TASK, error) != 0) return -1; + if (handle->api == task->api) return 0; + engine_error(error, 13, "Java engine task belongs to another engine"); + return -1; +} + +static int engine_scope(EngineHandle *handle, JavaScope *scope, BridgeBuffer *error) { + if (scope_enter(handle->api->context, scope, error) == 0) return 0; + engine_normalize_error(error, "cannot enter Java engine context"); + return -1; +} + +void *spark_connect_engine_thread_attach(void *raw_engine, BridgeBuffer *error) { + clear_buffer(error); + EngineHandle *engine = raw_engine; + if (engine_check(engine, ENGINE_FACTORY, error) != 0) return NULL; + EngineThread *thread = calloc(1, sizeof(*thread)); + if (thread == NULL) { + engine_error(error, 8, "cannot allocate Java worker attachment"); + return NULL; + } + JavaContext *context = engine->api->context; + JNIEnv *env = NULL; + jint state = (*context->vm)->GetEnv(context->vm, (void **) &env, JNI_VERSION_1_8); + if (state == JNI_EDETACHED) { + static atomic_uint_fast64_t next_worker = 1; + uint_fast64_t id = atomic_load_explicit(&next_worker, memory_order_relaxed); + do { + if (id == UINT_FAST64_MAX) { + engine_error(error, 8, "Java engine worker identifiers are exhausted"); + free(thread); + return NULL; + } + } while (!atomic_compare_exchange_weak_explicit(&next_worker, &id, id + 1, + memory_order_relaxed, memory_order_relaxed)); + char name[64]; + (void) snprintf(name, sizeof(name), "spark-connect-engine-worker-%" PRIuFAST64, id); + /* JNI consumes the name during attachment; the VM keeps its own copy. */ + JavaVMAttachArgs args = { + .version = JNI_VERSION_1_8, + .name = name, + .group = NULL, + }; + if ((*context->vm)->AttachCurrentThreadAsDaemon( + context->vm, (void **) &env, &args) != JNI_OK) { + engine_error(error, 13, "cannot attach Java engine worker"); + free(thread); + return NULL; + } + thread->attached = 1; + } else if (state != JNI_OK) { + engine_error(error, 13, "cannot access Java engine worker context"); + free(thread); + return NULL; + } + if (env == NULL) { + if (thread->attached) (*context->vm)->DetachCurrentThread(context->vm); + engine_error(error, 13, "Java engine worker has no JNI environment"); + free(thread); + return NULL; + } + thread->context = context; + thread->owner = pthread_self(); + atomic_fetch_add_explicit(&context->references, 1, memory_order_relaxed); + return thread; +} + +int spark_connect_engine_thread_detach(void *raw_attachment, BridgeBuffer *error) { + clear_buffer(error); + EngineThread *thread = raw_attachment; + if (thread == NULL) return 0; + if (!pthread_equal(thread->owner, pthread_self())) { + engine_error(error, 13, "Java worker attachment belongs to another thread"); + return -1; + } + JavaContext *context = thread->context; + if (thread->attached && (*context->vm)->DetachCurrentThread(context->vm) != JNI_OK) { + engine_error(error, 13, "cannot detach Java engine worker"); + return -1; + } + context_release(context); + free(thread); + return 0; +} + +void *spark_connect_engine_task_new(void *raw_engine, BridgeBuffer *error) { + clear_buffer(error); + EngineHandle *engine = raw_engine; + if (engine_check(engine, ENGINE_FACTORY, error) != 0) return NULL; + static atomic_uint_fast64_t next_task = 1; + uint_fast64_t id = atomic_load_explicit(&next_task, memory_order_relaxed); + do { + if (id == UINT_FAST64_MAX) { + engine_error(error, 8, "Java engine task identifiers are exhausted"); + return NULL; + } + } while (!atomic_compare_exchange_weak_explicit(&next_task, &id, id + 1, + memory_order_relaxed, memory_order_relaxed)); + char name[64]; + int len = snprintf(name, sizeof(name), "spark-engine-%" PRIuFAST64, id); + JavaScope scope; + if (engine_scope(engine, &scope, error) != 0) return NULL; + JNIEnv *env = scope.env; + jstring description = utf8_string(env, (const uint8_t *) name, (size_t) len, error); + jobject object = description == NULL ? NULL : (*env)->CallObjectMethod(env, + engine->object, engine->api->new_task, description); + EngineHandle *task = NULL; + if (engine_exception(env, engine->api, error) == 0 && object != NULL) + task = engine_handle(env, engine->api, object, ENGINE_TASK, error); + if (object != NULL) (*env)->DeleteLocalRef(env, object); + if (description != NULL) (*env)->DeleteLocalRef(env, description); + scope_leave(&scope); + if (task == NULL) engine_normalize_error(error, "Java engine did not create a task"); + return task; +} + +void *spark_connect_engine_open_session(void *raw_engine, const uint8_t *user, size_t user_len, + const uint8_t *id, size_t id_len, void *raw_task, BridgeBuffer *error) { + clear_buffer(error); + EngineHandle *engine = raw_engine; + EngineHandle *task = raw_task; + if (engine_check(engine, ENGINE_FACTORY, error) != 0 + || engine_task_check(engine, task, error) != 0) return NULL; + JavaScope scope; + if (engine_scope(engine, &scope, error) != 0) return NULL; + JNIEnv *env = scope.env; + jstring user_string = utf8_string(env, user, user_len, error); + jstring id_string = user_string == NULL ? NULL : utf8_string(env, id, id_len, error); + jobject object = id_string == NULL ? NULL : (*env)->CallObjectMethod(env, + engine->object, engine->api->open_session, user_string, id_string, task->object); + EngineHandle *session = NULL; + if (engine_exception(env, engine->api, error) == 0 && object != NULL) + session = engine_handle(env, engine->api, object, ENGINE_SESSION, error); + if (object != NULL) (*env)->DeleteLocalRef(env, object); + if (id_string != NULL) (*env)->DeleteLocalRef(env, id_string); + if (user_string != NULL) (*env)->DeleteLocalRef(env, user_string); + scope_leave(&scope); + if (session == NULL) engine_normalize_error(error, "Java engine did not open a session"); + return session; +} + +void *spark_connect_engine_prepare(void *raw_session, const uint8_t *relation, size_t len, + void *raw_task, BridgeBuffer *error) { + clear_buffer(error); + EngineHandle *session = raw_session; + EngineHandle *task = raw_task; + if (engine_check(session, ENGINE_SESSION, error) != 0 + || engine_task_check(session, task, error) != 0) return NULL; + JavaScope scope; + if (engine_scope(session, &scope, error) != 0) return NULL; + JNIEnv *env = scope.env; + jbyteArray input = byte_array(env, relation, len, error); + jobject object = input == NULL ? NULL : (*env)->CallObjectMethod(env, + session->object, session->api->prepare, input, task->object); + EngineHandle *query = NULL; + if (engine_exception(env, session->api, error) == 0 && object != NULL) + query = engine_handle(env, session->api, object, ENGINE_QUERY, error); + if (object != NULL) (*env)->DeleteLocalRef(env, object); + if (input != NULL) (*env)->DeleteLocalRef(env, input); + scope_leave(&scope); + if (query == NULL) engine_normalize_error(error, "Java engine did not prepare a query"); + return query; +} + +int spark_connect_engine_schema(void *raw_query, BridgeBuffer *out, BridgeBuffer *error) { + EngineHandle *query = raw_query; + clear_buffer(out); + clear_buffer(error); + if (engine_check(query, ENGINE_QUERY, error) != 0) return -1; + if (out == NULL) { + engine_error(error, 13, "missing Java engine result buffer"); + return -1; + } + JavaScope scope; + if (engine_scope(query, &scope, error) != 0) return -1; + JNIEnv *env = scope.env; + jbyteArray bytes = (*env)->CallObjectMethod(env, query->object, query->api->schema); + int result = engine_exception(env, query->api, error); + if (result == 0 && bytes != NULL) { + result = array_buffer(env, bytes, out, error); + } else if (result == 0) { + engine_error(error, 13, "Java engine returned no schema"); + result = -1; + } + if (bytes != NULL) (*env)->DeleteLocalRef(env, bytes); + scope_leave(&scope); + if (result < 0) engine_normalize_error(error, "cannot read Java engine query"); + return result; +} + +int spark_connect_engine_next_ipc(void *raw_query, IpcAllocate allocate, void *context, + int64_t *row_count, BridgeBuffer *error) { + clear_buffer(error); + if (row_count != NULL) *row_count = 0; + EngineHandle *query = raw_query; + if (engine_check(query, ENGINE_QUERY, error) != 0) return -1; + if (allocate == NULL || row_count == NULL) { + engine_error(error, 13, "missing Java engine IPC destination"); + return -1; + } + JavaScope scope; + if (engine_scope(query, &scope, error) != 0) return -1; + JNIEnv *env = scope.env; + jobject batch = (*env)->CallObjectMethod(env, query->object, query->api->next_ipc); + int result = engine_exception(env, query->api, error); + jbyteArray bytes = NULL; + if (result == 0 && batch != NULL) { + jsize len = 0; + jlong count = (*env)->CallLongMethod(env, batch, query->api->batch_row_count); + result = engine_exception(env, query->api, error); + if (result == 0 && count < 0) { + engine_error(error, 13, "negative Java engine IPC row count"); + result = -1; + } + if (result == 0) { + bytes = (*env)->CallObjectMethod(env, batch, query->api->batch_data); + result = engine_exception(env, query->api, error); + if (result == 0 && bytes == NULL) { + engine_error(error, 13, "Java engine returned no IPC data"); + result = -1; + } + } + if (result == 0) { + len = (*env)->GetArrayLength(env, bytes); + result = engine_exception(env, query->api, error); + if (result == 0 && len > 64 * 1024 * 1024) { + engine_error(error, 8, "Spark Arrow batch exceeds JNI limit"); + result = -1; + } + } + if (result == 0) { + uint8_t *data = allocate(context, (size_t) len); + if (data == NULL) { + engine_error(error, 8, "cannot allocate Rust IPC buffer"); + result = -1; + } else { + if (len != 0) (*env)->GetByteArrayRegion(env, bytes, 0, len, (jbyte *) data); + result = engine_exception(env, query->api, error); + } + } + if (result == 0) { + *row_count = (int64_t) count; + result = 1; + } + } + if (bytes != NULL) (*env)->DeleteLocalRef(env, bytes); + if (batch != NULL) (*env)->DeleteLocalRef(env, batch); + scope_leave(&scope); + if (result < 0) engine_normalize_error(error, "cannot read Java engine IPC batch"); + return result; +} + +int spark_connect_engine_analyze(void *raw_session, int32_t opcode, const uint8_t *relation, size_t len, + void *raw_task, BridgeBuffer *out, BridgeBuffer *error) { + clear_buffer(out); + clear_buffer(error); + EngineHandle *session = raw_session; + EngineHandle *task = raw_task; + if (engine_check(session, ENGINE_SESSION, error) != 0 + || engine_task_check(session, task, error) != 0) return -1; + if (out == NULL) { + engine_error(error, 13, "missing Java engine analysis buffer"); + return -1; + } + JavaScope scope; + if (engine_scope(session, &scope, error) != 0) return -1; + JNIEnv *env = scope.env; + jbyteArray input = byte_array(env, relation, len, error); + jbyteArray bytes = input == NULL ? NULL : (*env)->CallObjectMethod(env, + session->object, session->api->analyze, (jint) opcode, input, task->object); + int result = engine_exception(env, session->api, error); + if (result == 0 && bytes != NULL) result = array_buffer(env, bytes, out, error); + else if (result == 0) { + engine_normalize_error(error, "Java engine returned no analysis result"); + result = -1; + } + if (bytes != NULL) (*env)->DeleteLocalRef(env, bytes); + if (input != NULL) (*env)->DeleteLocalRef(env, input); + scope_leave(&scope); + if (result < 0) engine_normalize_error(error, "cannot analyze Java engine relation"); + return result; +} + +int spark_connect_engine_config(void *raw_session, int32_t opcode, const uint8_t *fields, size_t len, + void *raw_task, BridgeBuffer *out, BridgeBuffer *error) { + clear_buffer(out); + clear_buffer(error); + EngineHandle *session = raw_session; + EngineHandle *task = raw_task; + if (engine_check(session, ENGINE_SESSION, error) != 0 + || engine_task_check(session, task, error) != 0) return -1; + if (out == NULL) { + engine_error(error, 13, "missing Java engine config buffer"); + return -1; + } + JavaScope scope; + if (engine_scope(session, &scope, error) != 0) return -1; + JNIEnv *env = scope.env; + jbyteArray input = byte_array(env, fields, len, error); + jbyteArray bytes = input == NULL ? NULL : (*env)->CallObjectMethod(env, + session->object, session->api->config, (jint) opcode, input, task->object); + int result = engine_exception(env, session->api, error); + if (result == 0 && bytes != NULL) result = array_buffer(env, bytes, out, error); + else if (result == 0) { + engine_normalize_error(error, "Java engine returned no config result"); + result = -1; + } + if (bytes != NULL) (*env)->DeleteLocalRef(env, bytes); + if (input != NULL) (*env)->DeleteLocalRef(env, input); + scope_leave(&scope); + if (result < 0) engine_normalize_error(error, "cannot configure Java engine session"); + return result; +} + +int spark_connect_engine_cancel(void *raw_task, BridgeBuffer *error) { + clear_buffer(error); + EngineHandle *task = raw_task; + if (engine_check(task, ENGINE_TASK, error) != 0) return -1; + JavaScope scope; + if (engine_scope(task, &scope, error) != 0) return -1; + (*scope.env)->CallVoidMethod(scope.env, task->object, task->api->cancel); + int result = engine_exception(scope.env, task->api, error); + scope_leave(&scope); + if (result < 0) engine_normalize_error(error, "cannot cancel Java engine task"); + return result; +} + +int spark_connect_engine_free_checked(void *raw_handle, BridgeBuffer *error) { + clear_buffer(error); + EngineHandle *handle = raw_handle; + if (handle == NULL) return 0; + EngineApi *api = handle->api; + if (handle->kind == ENGINE_FACTORY + && atomic_load_explicit(&api->context->references, memory_order_acquire) != 1) { + engine_error(error, 13, "Java engine worker attachments remain during factory cleanup"); + return -1; + } + JavaScope scope; + if (engine_scope(handle, &scope, error) != 0) return -1; + JNIEnv *env = scope.env; + (*env)->CallVoidMethod(env, handle->object, api->close[handle->kind]); + if (engine_exception(env, api, error) != 0) { + scope_leave(&scope); + return -1; + } + (*env)->DeleteGlobalRef(env, handle->object); + // Delete final API/context globals in this valid scope, never by entering a + // second fallible JNI scope after ownership has already been consumed. + int final_api = atomic_fetch_sub_explicit(&api->references, 1, memory_order_acq_rel) == 1; + JavaContext *context = api->context; + int final_context = 0; + if (final_api) { + engine_delete_classes(env, api); + final_context = atomic_fetch_sub_explicit(&context->references, 1, + memory_order_acq_rel) == 1; + if (final_context) { + scope_restore_loader(&scope); + if (context->loader != NULL) (*env)->DeleteGlobalRef(env, context->loader); + (*env)->DeleteGlobalRef(env, context->thread_class); + } + } + scope_leave(&scope); + free(handle); + if (final_context) free(context); + if (final_api) free(api); + return 0; +} + +void spark_connect_engine_free(void *raw_handle) { + BridgeBuffer error = {0}; + if (spark_connect_engine_free_checked(raw_handle, &error) != 0) { + // Startup failures can have no registered Rust owner to retry. Preserve the + // C/Java resources for process lifetime and make the incomplete cleanup visible. + fputs("JNI cleanup is incomplete; retained cleanup failed: ", stderr); + (void) fwrite(error.data, 1, error.len, stderr); + fputc('\n', stderr); + } + spark_connect_buffer_free(&error); +} + +int64_t spark_connect_engine_native_start(void *raw_env, void *clazz, void *raw_address, void *raw_factory) { + (void) clazz; + JNIEnv *env = raw_env; + BridgeBuffer error = {0}; + BridgeBuffer address = {0}; + if (raw_address == NULL || raw_factory == NULL) { + engine_error(&error, 3, "native address and engine factory must be non-null"); + throw_error(env, &error); + return 0; + } + if (array_buffer(env, raw_address, &address, &error) != 0) { + engine_normalize_error(&error, "cannot read native engine address"); + throw_error(env, &error); + return 0; + } + EngineHandle *factory = engine_factory(env, raw_factory, &error); + if (factory == NULL) { + spark_connect_buffer_free(&address); + throw_error(env, &error); + return 0; + } + /* spark_connect_engine_start owns factory on both success and failure. */ + int64_t id = spark_connect_engine_start(address.data, address.len, factory, &error); + spark_connect_buffer_free(&address); + if (id == 0) throw_error(env, &error); + else spark_connect_buffer_free(&error); + return id; +} diff --git a/crates/spark-connect-server/jni/native/bridge.h b/crates/spark-connect-server/jni/native/bridge.h new file mode 100644 index 00000000..514021d6 --- /dev/null +++ b/crates/spark-connect-server/jni/native/bridge.h @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef SPARK_CONNECT_BRIDGE_H +#define SPARK_CONNECT_BRIDGE_H + +#include +#include + +typedef struct { + uint8_t *data; + size_t len; +} BridgeBuffer; + +void spark_connect_buffer_free(BridgeBuffer *buffer); + +/* Typed query engine ABI; these handles never carry service RPC envelopes. */ +/* Retain an attachment for a dedicated native worker. Release on the same OS + * thread, in reverse order for nested attachments. This does not change its + * class loader; individual engine calls still restore their scoped context. + * Only a guard that attached the thread detaches it. A failed detach retains + * the guard so its owner can retry. Callers that cannot retry must report the + * failure; the guard and retained context remain allocated for safety. The + * runtime must release guards before the hosting JVM is destroyed. */ +void *spark_connect_engine_thread_attach(void *engine, BridgeBuffer *error); +int spark_connect_engine_thread_detach(void *attachment, BridgeBuffer *error); +void *spark_connect_engine_task_new(void *engine, BridgeBuffer *error); +void *spark_connect_engine_open_session(void *engine, const uint8_t *user, size_t user_len, + const uint8_t *id, size_t id_len, void *task, BridgeBuffer *error); +void *spark_connect_engine_prepare(void *session, const uint8_t *relation, size_t len, + void *task, BridgeBuffer *error); +/* Opcodes 0/1/2: schema/isLocal/isStreaming. Output: bool byte, then schema IPC. + * Schema is always 0; streaming is currently 0 (streaming relations are rejected). */ +int spark_connect_engine_analyze(void *session, int32_t opcode, const uint8_t *relation, size_t len, + void *task, BridgeBuffer *out, BridgeBuffer *error); +int spark_connect_engine_schema(void *query, BridgeBuffer *out, BridgeBuffer *error); + +typedef uint8_t *(*IpcAllocate)(void *context, size_t len); +/* One complete Arrow IPC stream and its row count. Returns 1 for a batch, 0 for EOF, + * or -1 for error. After validating the batch, calls allocate synchronously once, + * including for an empty batch. It must return a nonnull pointer writable for len + * bytes, or NULL if allocation is refused or fails. C never retains or frees the + * callback, context or destination. The caller owns the destination on success and + * failure and must only consider its bytes initialized when this returns 1. The + * row_count is zero on EOF/error. No callback is made for EOF or invalid batches. */ +int spark_connect_engine_next_ipc(void *query, IpcAllocate allocate, void *context, + int64_t *row_count, BridgeBuffer *error); +int spark_connect_engine_config(void *session, int32_t opcode, const uint8_t *fields, size_t len, + void *task, BridgeBuffer *out, BridgeBuffer *error); +/* Cancellation retains the handle. Failure may mean JNI delivery failed or Java + * partially cancelled; preserve the adapter's non-replay policy for the latter. */ +int spark_connect_engine_cancel(void *task, BridgeBuffer *error); +void spark_connect_engine_free(void *handle); +/* Success consumes the handle. Failure retains its Java global reference and API ownership. + * Retrying is safe only when the Java owner's close contract permits it; the supplied adapter + * preserves non-retryable Java close errors rather than repeating partial side effects. */ +int spark_connect_engine_free_checked(void *handle, BridgeBuffer *error); + +int64_t spark_connect_engine_native_start(void *env, void *clazz, void *address, void *factory); +void *spark_connect_engine_native_address(void *env, void *clazz, int64_t id); +void spark_connect_engine_native_stop(void *env, void *clazz, int64_t id); + +#endif diff --git a/crates/spark-connect-server/jni/src/engine.rs b/crates/spark-connect-server/jni/src/engine.rs new file mode 100644 index 00000000..e731cd53 --- /dev/null +++ b/crates/spark-connect-server/jni/src/engine.rs @@ -0,0 +1,2934 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Typed JNI implementation of the shared engine contract. +//! No Spark Connect RPC handler or request/response envelope crosses this boundary. +use super::{take_buffer, BridgeBuffer, MAX_BYTES}; +use arrow::{datatypes::SchemaRef, ipc::reader::StreamReader}; +use futures::stream; +use prost::Message; +use spark_connect_proto as proto; +use spark_connect_server::engine::{ + BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, + ResultBatch, SessionContext, +}; +use std::{ + collections::HashMap, + ffi::c_void, + io::{Cursor, Write}, + marker::PhantomData, + rc::Rc, + sync::{ + atomic::{AtomicBool, AtomicPtr, AtomicU64, AtomicUsize, Ordering}, + mpsc, Arc, Condvar, Mutex, Weak, + }, + time::Duration, +}; +use tokio::sync::{ + Mutex as AsyncMutex, OwnedSemaphorePermit, RwLock, RwLockReadGuard, RwLockWriteGuard, Semaphore, +}; +use tokio::time::Instant; +use tonic::Status; + +const SESSION_ADMISSION_TIMEOUT: Duration = Duration::from_secs(2); +const CLEANUP_ATTACHMENT_IDLE: Duration = Duration::from_millis(10); + +extern "C" { + fn spark_connect_engine_thread_attach( + engine: *mut c_void, + error: *mut BridgeBuffer, + ) -> *mut c_void; + fn spark_connect_engine_thread_detach(attachment: *mut c_void, error: *mut BridgeBuffer) + -> i32; + fn spark_connect_engine_task_new(engine: *mut c_void, error: *mut BridgeBuffer) -> *mut c_void; + fn spark_connect_engine_open_session( + engine: *mut c_void, + user: *const u8, + user_len: usize, + id: *const u8, + id_len: usize, + task: *mut c_void, + error: *mut BridgeBuffer, + ) -> *mut c_void; + fn spark_connect_engine_prepare( + session: *mut c_void, + relation: *const u8, + len: usize, + task: *mut c_void, + error: *mut BridgeBuffer, + ) -> *mut c_void; + fn spark_connect_engine_schema( + query: *mut c_void, + out: *mut BridgeBuffer, + error: *mut BridgeBuffer, + ) -> i32; + fn spark_connect_engine_next_ipc( + query: *mut c_void, + allocate: unsafe extern "C" fn(*mut c_void, usize) -> *mut u8, + context: *mut c_void, + row_count: *mut i64, + error: *mut BridgeBuffer, + ) -> i32; + fn spark_connect_engine_analyze( + session: *mut c_void, + opcode: i32, + relation: *const u8, + len: usize, + task: *mut c_void, + out: *mut BridgeBuffer, + error: *mut BridgeBuffer, + ) -> i32; + fn spark_connect_engine_config( + session: *mut c_void, + opcode: i32, + fields: *const u8, + len: usize, + task: *mut c_void, + out: *mut BridgeBuffer, + error: *mut BridgeBuffer, + ) -> i32; + fn spark_connect_engine_cancel(task: *mut c_void, error: *mut BridgeBuffer) -> i32; + fn spark_connect_engine_free(handle: *mut c_void); + fn spark_connect_engine_free_checked(handle: *mut c_void, error: *mut BridgeBuffer) -> i32; +} + +fn engine_error(error: BridgeBuffer) -> Status { + let bytes = take_buffer(error); + decode_error(&String::from_utf8_lossy(&bytes)) +} +fn decode_error(message: &str) -> Status { + if let Some((code, text)) = message + .strip_prefix("JVM_ENGINE_ERROR:") + .and_then(|text| text.split_once(':')) + { + if let Ok(code @ 1..=16) = code.parse::() { + return Status::new(tonic::Code::from_i32(code), text.to_owned()); + } + } + Status::internal(format!("JVM engine: {message}")) +} + +#[derive(Default)] +struct Counts { + workers: usize, + handles: usize, + tasks: HashMap>, + failed_cleanup: Vec, +} +pub(super) struct JvmEngine { + factory: AtomicPtr, + closed: AtomicBool, + counts: Mutex, + drained: Condvar, + permits: Arc, + session_permits: Arc, + worker_attachments: AtomicUsize, +} +// C owns JavaVM/global references. JNI worker attachments remain thread-local. +unsafe impl Send for JvmEngine {} +unsafe impl Sync for JvmEngine {} +impl Drop for JvmEngine { + fn drop(&mut self) { + // Fallback for startup failures that never reached normal host shutdown. + unsafe { + spark_connect_engine_free(self.factory.swap(std::ptr::null_mut(), Ordering::SeqCst)) + }; + } +} +impl JvmEngine { + pub(super) fn new(factory: *mut c_void) -> Arc { + Arc::new(Self { + factory: AtomicPtr::new(factory), + closed: AtomicBool::new(false), + counts: Mutex::new(Counts::default()), + drained: Condvar::new(), + permits: Arc::new(Semaphore::new(64)), + session_permits: Arc::new(Semaphore::new(64)), + worker_attachments: AtomicUsize::new(0), + }) + } + pub(super) fn attach_worker(self: &Arc) -> Result { + if self.closed.load(Ordering::SeqCst) { + return Err(Status::unavailable("Spark engine is stopping")); + } + let mut error = BridgeBuffer::default(); + let pointer = unsafe { + spark_connect_engine_thread_attach(self.factory.load(Ordering::SeqCst), &mut error) + }; + if pointer.is_null() { + return Err(engine_error(error)); + } + let _ = take_buffer(error); + self.worker_attachments.fetch_add(1, Ordering::SeqCst); + Ok(WorkerAttachment { + pointer, + core: self.clone(), + _thread: PhantomData, + }) + } + /// Check after the runtime stops; wait_drained cannot include this count + /// because the runtime owns the workers that must detach during its stop. + pub(super) fn verify_workers_detached(&self) -> Result<(), String> { + let remaining = self.worker_attachments.load(Ordering::SeqCst); + if remaining == 0 { + Ok(()) + } else { + Err(format!( + "{remaining} JNI worker attachments remain after runtime stop" + )) + } + } + #[cfg(test)] + pub(super) fn set_worker_attachment_count_for_test(&self, count: usize) { + self.worker_attachments.store(count, Ordering::SeqCst); + } + #[cfg(test)] + pub(super) fn fail_handle_cleanup_for_test(self: &Arc) { + self.counts.lock().unwrap().handles += 1; + cleanup_with( + Cleanup::Free { + pointer: 0, + core: self.clone(), + permit: None, + lane: CleanupLane::Task, + retained_task: None, + }, + |_| Err(Status::internal("injected Java cleanup failure")), + ); + } + pub(super) fn shutdown(&self) { + let tasks = { + let counts = self + .counts + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + self.closed.store(true, Ordering::SeqCst); + counts + .tasks + .values() + .filter_map(Weak::upgrade) + .collect::>() + }; + self.permits.close(); + self.session_permits.close(); + for task in tasks { + task.cancel(); + } + } + pub(super) fn wait_drained(&self) { + let mut counts = self + .counts + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + while counts.workers != 0 || counts.handles != 0 { + counts = self + .drained + .wait(counts) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + } + pub(super) fn cleanup_failure(&self) -> Option { + let counts = self + .counts + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + counts.failed_cleanup.first().map(|failure| { + format!( + "JNI cleanup is incomplete; retained cleanup failed: {} ({} pending); retry close after the failure is resolved", + failure.error, + counts.failed_cleanup.len() + ) + }) + } + fn take_failed_cleanup(&self) -> Vec { + std::mem::take( + &mut self + .counts + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .failed_cleanup, + ) + } + /// Only an explicit owner close retries failed cleanup. Preserve each item's + /// original lane, Java ownership and admission until success. + pub(super) fn retry_failed_cleanup(&self) { + for failure in self.take_failed_cleanup() { + queue(failure.item); + } + } + fn take_factory_after_drain(&self) -> Result<*mut c_void, String> { + if !self.closed.load(Ordering::SeqCst) { + return Err("Spark engine factory release requires shutdown".into()); + } + self.verify_workers_detached()?; + let counts = self + .counts + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if counts.workers != 0 || counts.handles != 0 { + return Err("Spark engine factory release requires drained work".into()); + } + // Shutdown prevents new producers, and every outstanding JNI operation + // is represented above. Never hold the mutex across JNI. + drop(counts); + Ok(self.factory.swap(std::ptr::null_mut(), Ordering::SeqCst)) + } + /// The host holds its stopping mutex, serializing this synchronous release + /// with retries, and retains the engine until the native free returns. + pub(super) fn release_factory(&self) -> Result<(), String> { + self.release_factory_with(release_native) + } + fn release_factory_with( + &self, + release: impl FnOnce(usize) -> Result<(), Status>, + ) -> Result<(), String> { + let factory = self.take_factory_after_drain()?; + // stop_server retains its own Arc throughout this call, so Drop cannot + // race to free the factory. Cleanup workers may still hold harmless core + // Arcs after publishing zero counters; their later drops see null. + match release(factory as usize) { + Ok(()) => Ok(()), + Err(error) => { + // The checked ABI did not consume the handle. The host stopping + // mutex excludes another close; retain the same factory for retry. + self.factory.store(factory, Ordering::SeqCst); + Err(format!( + "JNI cleanup is incomplete; retained cleanup failed: {error}" + )) + } + } + } + async fn admit_session( + self: &Arc, + deadline: Instant, + ) -> Result { + tokio::time::timeout_at(deadline, self.session_permits.clone().acquire_owned()) + .await + .map_err(|_| Status::resource_exhausted("Spark engine session capacity exhausted"))? + .map_err(|_| Status::unavailable("Spark engine is stopping")) + } + async fn task(self: &Arc, deadline: Instant) -> Result { + let permit = tokio::time::timeout_at(deadline, self.permits.clone().acquire_owned()) + .await + .map_err(|_| Status::resource_exhausted("Spark engine task capacity exhausted"))? + .map_err(|_| Status::unavailable("Spark engine is stopping"))?; + static NEXT: AtomicU64 = AtomicU64::new(1); + let id = NEXT.fetch_add(1, Ordering::Relaxed); + let task = Arc::new(Task { + core: self.clone(), + id, + handle: Mutex::new(None), + cancelled: AtomicBool::new(false), + permit: Mutex::new(Some(permit)), + }); + { + let mut counts = self + .counts + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.closed.load(Ordering::SeqCst) { + return Err(Status::unavailable("Spark engine is stopping")); + } + counts.tasks.insert(id, Arc::downgrade(&task)); + } + Ok(TaskGuard { + task, + complete: false, + }) + } + async fn blocking( + self: &Arc, + action: impl FnOnce() -> Result + Send + 'static, + ) -> Result { + // Reserve activity before spawning. Dropping a future cancels its TaskGuard; + // the worker and all global references remain retained until JNI returns. + let activity = { + let mut counts = self + .counts + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.closed.load(Ordering::SeqCst) { + return Err(Status::unavailable("Spark engine is stopping")); + } + counts.workers += 1; + Activity(self.clone()) + }; + tokio::task::spawn_blocking(move || { + let worker_activity = activity; + super::attach_engine_worker(&worker_activity.0); + action() + }) + .await + .map_err(|e| Status::internal(format!("Spark engine worker failed: {e}")))? + } + fn handle( + self: &Arc, + pointer: *mut c_void, + error: BridgeBuffer, + kind: HandleKind, + retained_task: Option>, + ) -> Result, Status> { + if pointer.is_null() { + return Err(engine_error(error)); + } + // A successful C call must not return an owned error buffer. + let _ = take_buffer(error); + self.counts + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .handles += 1; + Ok(Arc::new(Handle { + pointer, + core: self.clone(), + permit: Mutex::new(None), + kind, + retained_task, + })) + } +} + +/// Retains the JVM until the same runtime worker detaches on thread exit. +/// Rc's marker prevents moving or sharing the guard across threads. Drop makes +/// one same-thread retry; a persistent failure retains the C attachment/context +/// for safety, and the attachment count makes shutdown report that failure. +pub(super) struct WorkerAttachment { + pointer: *mut c_void, + core: Arc, + _thread: PhantomData>, +} +impl WorkerAttachment { + pub(super) fn belongs_to(&self, core: &Arc) -> bool { + Arc::ptr_eq(&self.core, core) + } +} +impl Drop for WorkerAttachment { + fn drop(&mut self) { + let result = detach_with_retry(|| { + let mut error = BridgeBuffer::default(); + if unsafe { spark_connect_engine_thread_detach(self.pointer, &mut error) } != 0 { + Err(engine_error(error)) + } else { + let _ = take_buffer(error); + Ok(()) + } + }); + match result { + Ok(()) => { + self.core.worker_attachments.fetch_sub(1, Ordering::SeqCst); + } + Err(error) => log_diagnostic(format_args!("JVM worker detach failed: {error}")), + } + } +} + +fn detach_with_retry(mut detach: impl FnMut() -> Result<(), Status>) -> Result<(), Status> { + // A failed C detach retains its guard. Retry while its owning OS thread + // still exists; a later server close cannot perform a cross-thread detach. + detach().or_else(|_| detach()) +} + +pub(super) fn log_diagnostic(message: std::fmt::Arguments<'_>) { + log_diagnostic_to(std::io::stderr(), message); +} + +fn log_diagnostic_to(mut writer: impl Write, message: std::fmt::Arguments<'_>) { + // Diagnostics must never unwind a cleanup owner on a broken output pipe. + let _ = writeln!(writer, "{message}"); +} +struct Activity(Arc); +impl Drop for Activity { + fn drop(&mut self) { + let mut c = self + .0 + .counts + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + c.workers -= 1; + self.0.drained.notify_all(); + } +} +struct Handle { + pointer: *mut c_void, + core: Arc, + // Keep task/session admission until its final Java reference is released. + permit: Mutex>, + kind: HandleKind, + // Query teardown must precede reclamation of its admission-holding task. + retained_task: Option>, +} +#[derive(Clone, Copy)] +enum HandleKind { + Task, + Session, + Query, +} +#[derive(Clone, Copy)] +enum CleanupLane { + Task, + Query, + Session, +} +impl HandleKind { + fn cleanup_lane(self) -> CleanupLane { + match self { + Self::Task => CleanupLane::Task, + Self::Query => CleanupLane::Query, + Self::Session => CleanupLane::Session, + } + } +} +unsafe impl Send for Handle {} +unsafe impl Sync for Handle {} +impl Handle { + fn pointer(&self) -> *mut c_void { + self.pointer + } +} +impl Drop for Handle { + fn drop(&mut self) { + queue(Cleanup::Free { + pointer: self.pointer as usize, + core: self.core.clone(), + lane: self.kind.cleanup_lane(), + retained_task: self.retained_task.take(), + permit: self + .permit + .get_mut() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(), + }); + } +} +enum Cleanup { + Cancel(Arc), + Free { + pointer: usize, + core: Arc, + permit: Option, + lane: CleanupLane, + retained_task: Option>, + }, + #[cfg(test)] + TestAction { + lane: CleanupLane, + action: Box, + }, +} +struct FailedCleanup { + item: Cleanup, + error: Status, +} +impl Cleanup { + fn core(&self) -> Option<&Arc> { + match self { + Self::Cancel(handle) => Some(&handle.core), + Self::Free { core, .. } => Some(core), + #[cfg(test)] + Self::TestAction { .. } => None, + } + } + fn lane(&self) -> CleanupLane { + match self { + Self::Cancel(_) => CleanupLane::Task, + Self::Free { lane, .. } => *lane, + #[cfg(test)] + Self::TestAction { lane, .. } => *lane, + } + } +} + +/// Cleanup lanes outlive individual servers. Retain an attachment only during +/// a burst, and count that whole lifetime as activity so shutdown waits for the +/// same-thread detach before it releases the engine factory. +struct CleanupAttachment { + worker: Option, + activity: Activity, +} +impl CleanupAttachment { + fn attach_with( + core: &Arc, + attach: impl FnOnce(&Arc) -> Result, + ) -> Result { + let activity = { + let mut counts = core + .counts + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if core.closed.load(Ordering::SeqCst) { + return Err(Status::unavailable("Spark engine is stopping")); + } + counts.workers += 1; + Activity(core.clone()) + }; + // Each queued item still owns a handle or export, retaining the factory + // across attachment acquisition and any concurrent shutdown. + let worker = attach(core)?; + Ok(Self { + worker: Some(worker), + activity, + }) + } + fn attach_or_fallback( + core: &Arc, + attach: impl FnOnce(&Arc) -> Result, + reported: &mut Option>, + report: impl FnOnce(&Status), + ) -> Option { + match Self::attach_with(core, attach) { + Ok(attachment) => { + *reported = None; + Some(attachment) + } + Err(error) if error.code() == tonic::Code::Unavailable => None, + Err(error) => { + // Suppress consecutive reports for the same engine until an + // attachment succeeds. Alternating failing engines report once + // per switch. The weak identity does not retain the engine. + let identity = Arc::downgrade(core); + if reported + .as_ref() + .is_none_or(|previous| !Weak::ptr_eq(previous, &identity)) + { + *reported = Some(identity); + report(&error); + } + None + } + } + } + fn belongs_to(&self, core: &Arc) -> bool { + Arc::ptr_eq(&self.activity.0, core) + } +} +impl Drop for CleanupAttachment { + fn drop(&mut self) { + // Activity must stay reserved until detach completes. A detach failure + // remains visible through verify_workers_detached's attachment count. + drop(self.worker.take()); + } +} + +fn cleanup_loop(rx: mpsc::Receiver) { + cleanup_loop_with(rx, CLEANUP_ATTACHMENT_IDLE, JvmEngine::attach_worker); +} + +// The generic worker constructor lets tests exercise successful attachment +// lifetimes without a JVM; the production loop still uses the native guard. +fn cleanup_loop_with( + rx: mpsc::Receiver, + idle: Duration, + mut attach: impl FnMut(&Arc) -> Result, +) { + let mut attachment: Option> = None; + let mut reported = None; + loop { + let item = if attachment.is_some() { + match rx.recv_timeout(idle) { + Ok(item) => item, + Err(mpsc::RecvTimeoutError::Timeout) => { + drop(attachment.take()); + continue; + } + Err(mpsc::RecvTimeoutError::Disconnected) => break, + } + } else { + match rx.recv() { + Ok(item) => item, + Err(_) => break, + } + }; + match item.core() { + Some(core) => { + if attachment + .as_ref() + .is_none_or(|current| !current.belongs_to(core)) + { + drop(attachment.take()); + // Per-call JNI scopes remain the fallback on attachment + // failure or when shutdown has already closed admission. + attachment = CleanupAttachment::attach_or_fallback( + core, + &mut attach, + &mut reported, + |error| { + log_diagnostic(format_args!( + "JVM cleanup lane attachment failed; using per-call attachment: {error}" + )); + }, + ); + } + } + None => drop(attachment.take()), + } + cleanup(item); + } +} + +fn cleanup_sender( + lane: CleanupLane, +) -> (&'static Mutex>>, &'static str) { + // Session close may block on Spark teardown. Isolate it from Query frees + // that hold task admission, and Task/cancel reclamation itself. + static TASK: Mutex>> = Mutex::new(None); + static QUERY: Mutex>> = Mutex::new(None); + static SESSION: Mutex>> = Mutex::new(None); + match lane { + CleanupLane::Task => (&TASK, "spark-connect-engine-task-cleanup"), + CleanupLane::Query => (&QUERY, "spark-connect-engine-query-cleanup"), + CleanupLane::Session => (&SESSION, "spark-connect-engine-session-cleanup"), + } +} +fn release_native(pointer: usize) -> Result<(), Status> { + let mut error = BridgeBuffer::default(); + let result = unsafe { spark_connect_engine_free_checked(pointer as *mut c_void, &mut error) }; + if result != 0 { + Err(engine_error(error)) + } else { + let _ = take_buffer(error); + Ok(()) + } +} +fn cleanup(item: Cleanup) { + cleanup_with(item, release_native); +} +fn cleanup_with(item: Cleanup, release: impl FnOnce(usize) -> Result<(), Status>) { + cleanup_with_actions(item, release, cancel_native); +} +fn cancel_native(pointer: *mut c_void) -> Result<(), Status> { + let mut error = BridgeBuffer::default(); + if unsafe { spark_connect_engine_cancel(pointer, &mut error) } != 0 { + Err(engine_error(error)) + } else { + let _ = take_buffer(error); + Ok(()) + } +} +fn retain_failed_cleanup(item: Cleanup, error: Status) { + let message = error.to_string(); + // Retain native ownership and admission before attempting diagnostics. + // Explicit close retries a transient failure on the item's original lane. + let core = item.core().expect("native cleanup has an engine").clone(); + core.counts + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .failed_cleanup + .push(FailedCleanup { item, error }); + core.drained.notify_all(); + log_diagnostic(format_args!( + "JNI cleanup is incomplete; retained cleanup failed: {message}" + )); +} +fn cleanup_with_actions( + item: Cleanup, + release: impl FnOnce(usize) -> Result<(), Status>, + cancel: impl FnOnce(*mut c_void) -> Result<(), Status>, +) { + let result = match &item { + Cleanup::Free { pointer, .. } => release(*pointer), + Cleanup::Cancel(handle) => cancel(handle.pointer()), + #[cfg(test)] + Cleanup::TestAction { .. } => Ok(()), + }; + if let Err(error) = result { + retain_failed_cleanup(item, error); + return; + } + match item { + Cleanup::Cancel(_) => {} + Cleanup::Free { + pointer: _, + core, + permit, + lane: _, + retained_task, + } => { + // Publish resource/admission release before reporting this handle + // drained. Query dependencies may schedule their own Task free. + drop(retained_task); + drop(permit); + let mut counts = core + .counts + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + counts.handles -= 1; + core.drained.notify_all(); + } + #[cfg(test)] + Cleanup::TestAction { action, .. } => action(), + } +} +fn queue(item: Cleanup) { + let (sender, name) = cleanup_sender(item.lane()); + if let Err((item, error)) = enqueue_cleanup(item, sender, |rx| { + std::thread::Builder::new() + .name(name.into()) + .spawn(move || cleanup_loop(rx)) + .map(|_| ()) + }) { + // Never run potentially blocking JNI on an async worker when a cleanup + // thread cannot start. Keep ownership for an explicit close retry. + retain_failed_cleanup(item, error); + } +} + +fn enqueue_cleanup( + item: Cleanup, + sender: &Mutex>>, + spawn: impl FnOnce(mpsc::Receiver) -> std::io::Result<()>, +) -> Result<(), (Cleanup, Status)> { + let mut sender = sender + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let item = if let Some(tx) = sender.as_ref() { + match tx.send(item) { + Ok(()) => return Ok(()), + Err(error) => { + *sender = None; + error.0 + } + } + } else { + item + }; + let (tx, rx) = mpsc::channel(); + if let Err(error) = spawn(rx) { + return Err(( + item, + Status::unavailable(format!("cannot start JNI cleanup lane: {error}")), + )); + } + match tx.send(item) { + Ok(()) => { + *sender = Some(tx); + Ok(()) + } + Err(error) => Err((error.0, Status::unavailable("JNI cleanup lane stopped"))), + } +} +struct Task { + core: Arc, + id: u64, + handle: Mutex>>, + cancelled: AtomicBool, + permit: Mutex>, +} +impl Task { + fn initialize(&self) -> Result, Status> { + let existing = self + .handle + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + let handle = match existing { + Some(handle) => handle, + None => { + if self.cancelled.load(Ordering::SeqCst) { + return Err(Status::cancelled("Spark work cancelled")); + } + // Never hold the publication mutex across JNI: future drop and + // native stop must be able to mark cancellation immediately. + let mut error = BridgeBuffer::default(); + let pointer = unsafe { + spark_connect_engine_task_new( + self.core.factory.load(Ordering::SeqCst), + &mut error, + ) + }; + let created = self.core.handle(pointer, error, HandleKind::Task, None)?; + self.handle + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get_or_insert(created) + .clone() + } + }; + if let Some(permit) = self + .permit + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + *handle + .permit + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(permit); + } + if self.cancelled.load(Ordering::SeqCst) { + cleanup(Cleanup::Cancel(handle)); + return Err(Status::cancelled("Spark work cancelled")); + } + Ok(handle) + } + fn cancel(&self) { + if !self.cancelled.swap(true, Ordering::SeqCst) { + if let Some(handle) = self + .handle + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + { + queue(Cleanup::Cancel(handle)); + } + } + } +} +impl Drop for Task { + fn drop(&mut self) { + self.core + .counts + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .tasks + .remove(&self.id); + } +} +struct TaskGuard { + task: Arc, + complete: bool, +} +impl TaskGuard { + fn finish(&mut self) { + self.complete = true; + } +} +impl Drop for TaskGuard { + fn drop(&mut self) { + if !self.complete { + self.task.cancel(); + } + } +} + +pub(super) struct EngineFactory(pub Arc); +impl Engine for EngineFactory { + fn create_session(&self, context: SessionContext) -> Result, Status> { + Ok(Arc::new(JvmSession { + core: self.0.clone(), + context, + handle: AsyncMutex::new(None), + catalog: RwLock::new(()), + })) + } +} +struct JvmSession { + core: Arc, + context: SessionContext, + handle: AsyncMutex>>, + catalog: RwLock<()>, +} +#[derive(Clone, Copy)] +enum CatalogAccess { + Read, + Write, +} +enum CatalogGuard<'a> { + Read { _guard: RwLockReadGuard<'a, ()> }, + Write { _guard: RwLockWriteGuard<'a, ()> }, +} +impl CatalogAccess { + // JNI Config ABI: 1..=5 are read-only, 6/7/8 mutate. Unknown opcodes stay + // exclusive conservatively. The Java adapter mirrors this contract. + fn config(opcode: i32) -> Self { + match opcode { + 1..=5 => Self::Read, + _ => Self::Write, + } + } + async fn lock_until<'a>( + self, + catalog: &'a RwLock<()>, + deadline: Instant, + ) -> Result, Status> { + tokio::time::timeout_at(deadline, self.lock(catalog)) + .await + .map_err(|_| Status::resource_exhausted("Spark engine session catalog busy")) + } + async fn lock(self, catalog: &RwLock<()>) -> CatalogGuard<'_> { + match self { + Self::Read => CatalogGuard::Read { + _guard: catalog.read().await, + }, + Self::Write => CatalogGuard::Write { + _guard: catalog.write().await, + }, + } + } +} +#[derive(Clone, Copy)] +#[repr(i32)] +enum AnalysisMode { + Schema = 0, + IsLocal = 1, + IsStreaming = 2, +} +struct Metadata { + schema: SchemaRef, + flag: bool, +} +impl JvmSession { + async fn handle(&self, deadline: Instant) -> Result, Status> { + // A concurrent Java initializer may run longer than the admission budget. + // Bound this caller's wait without cancelling the initializer that owns + // its task/session permits and still has a JNI call in flight. + let mut cached = tokio::time::timeout_at(deadline, self.handle.lock()) + .await + .map_err(|_| Status::resource_exhausted("Spark engine session initialization busy"))?; + if let Some(handle) = cached.as_ref() { + return Ok(handle.clone()); + } + // Bound heavyweight Java sessions through actual deferred close, + // without consuming the independent in-flight task budget. The fixed + // operation-entry admission deadline also spans initializer retries. + let session_permit = self.core.admit_session(deadline).await?; + let mut guard = self.core.task(deadline).await?; + let task = guard.task.clone(); + let core = self.core.clone(); + let user = self + .context + .user_context + .as_ref() + .map(|u| u.user_id.clone()) + .unwrap_or_default(); + let id = self.context.session_id.clone(); + let handle = self + .core + .blocking(move || { + let task = task.initialize()?; + let mut error = BridgeBuffer::default(); + let pointer = unsafe { + spark_connect_engine_open_session( + core.factory.load(Ordering::SeqCst), + user.as_ptr(), + user.len(), + id.as_ptr(), + id.len(), + task.pointer(), + &mut error, + ) + }; + let handle = core.handle(pointer, error, HandleKind::Session, None)?; + *handle + .permit + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(session_permit); + Ok(handle) + }) + .await?; + guard.finish(); + *cached = Some(handle.clone()); + Ok(handle) + } + async fn metadata( + &self, + relation: proto::Relation, + mode: AnalysisMode, + deadline: Instant, + ) -> Result { + let session = self.handle(deadline).await?; + let mut guard = self.core.task(deadline).await?; + let task = guard.task.clone(); + let bytes = relation.encode_to_vec(); + let metadata = self + .core + .blocking(move || { + let task = task.initialize()?; + let bytes = output(|out, error| unsafe { + spark_connect_engine_analyze( + session.pointer(), + mode as i32, + bytes.as_ptr(), + bytes.len(), + task.pointer(), + out, + error, + ) + })?; + decode_metadata(bytes, mode) + }) + .await?; + guard.finish(); + Ok(metadata) + } + async fn config_pairs( + &self, + opcode: i32, + pairs: Vec, + deadline: Instant, + ) -> Result, Status> { + let bytes = encode_pairs(&pairs)?; + let session = self.handle(deadline).await?; + let mut guard = self.core.task(deadline).await?; + let task = guard.task.clone(); + let bytes = self + .core + .blocking(move || { + let task = task.initialize()?; + output(|out, error| unsafe { + spark_connect_engine_config( + session.pointer(), + opcode, + bytes.as_ptr(), + bytes.len(), + task.pointer(), + out, + error, + ) + }) + }) + .await?; + guard.finish(); + decode_pairs(&bytes) + } + async fn relation( + &self, + relation: proto::Relation, + deadline: Instant, + ) -> Result { + let session = self.handle(deadline).await?; + let guard = self.core.task(deadline).await?; + let task = guard.task.clone(); + let core = self.core.clone(); + let bytes = relation.encode_to_vec(); + let (handle, schema) = self + .core + .blocking(move || { + let task = task.initialize()?; + let mut error = BridgeBuffer::default(); + let pointer = unsafe { + spark_connect_engine_prepare( + session.pointer(), + bytes.as_ptr(), + bytes.len(), + task.pointer(), + &mut error, + ) + }; + // Preserve the task-admission bound through actual query close, + // even though task frees use the independent cleanup lane. + let query = core.handle(pointer, error, HandleKind::Query, Some(task))?; + let bytes = output(|out, error| unsafe { + spark_connect_engine_schema(query.pointer(), out, error) + })?; + let reader = ipc(bytes)?; + let schema = reader.schema(); + Ok((query, schema)) + }) + .await?; + Ok(JvmQuery { + core: self.core.clone(), + handle, + guard, + schema, + }) + } +} +fn decode_metadata(bytes: Vec, mode: AnalysisMode) -> Result { + decode_metadata_with_limit(bytes, mode, MAX_BYTES) +} +fn decode_metadata_with_limit( + bytes: Vec, + mode: AnalysisMode, + ipc_limit: usize, +) -> Result { + let flag = match bytes.first() { + Some(0) => false, + Some(1) if !matches!(mode, AnalysisMode::Schema) => true, + _ => return Err(Status::internal("invalid Spark metadata flag")), + }; + // The transport prefix is not part of the Arrow IPC size limit. Keep the + // allocation intact and start the reader after it, avoiding a payload copy. + if bytes.len() - 1 > ipc_limit { + return Err(Status::resource_exhausted( + "Spark Arrow batch exceeds JNI limit", + )); + } + let mut cursor = Cursor::new(bytes); + cursor.set_position(1); + let reader = StreamReader::try_new(cursor, None) + .map_err(|e| Status::internal(format!("invalid Spark Arrow stream: {e}")))?; + Ok(Metadata { + schema: reader.schema(), + flag, + }) +} +fn ipc(bytes: Vec) -> Result>>, Status> { + if bytes.len() > MAX_BYTES { + return Err(Status::resource_exhausted( + "Spark Arrow batch exceeds JNI limit", + )); + } + StreamReader::try_new(Cursor::new(bytes), None) + .map_err(|e| Status::internal(format!("invalid Spark Arrow stream: {e}"))) +} +fn output( + action: impl FnOnce(*mut BridgeBuffer, *mut BridgeBuffer) -> i32, +) -> Result, Status> { + let mut out = BridgeBuffer::default(); + let mut error = BridgeBuffer::default(); + if action(&mut out, &mut error) < 0 { + let _ = take_buffer(out); + Err(engine_error(error)) + } else { + let _ = take_buffer(error); + Ok(take_buffer(out)) + } +} +struct JvmQuery { + core: Arc, + handle: Arc, + guard: TaskGuard, + schema: SchemaRef, +} + +#[derive(Default)] +struct IpcBuffer { + data: Vec, + requested: Option, +} +impl IpcBuffer { + /// # Safety + /// If storage was requested, its entire region must be initialized. The bridge + /// guarantees this only when `spark_connect_engine_next_ipc` returns 1. + unsafe fn into_vec(mut self) -> Result, Status> { + let len = self + .requested + .ok_or_else(|| Status::internal("JVM IPC buffer was not allocated"))?; + // The callback reserved capacity; a successful JNI transfer initialized it. + unsafe { self.data.set_len(len) }; + Ok(self.data) + } +} + +/// This synchronous callback only reserves storage; no bytes become visible to +/// Rust until the C bridge reports a successful copy. All operations are fallible +/// or non-panicking, so no Rust panic can unwind through the C bridge. +/// +/// # Safety +/// `context` must point to an exclusively borrowed, live `IpcBuffer` for this call. +unsafe extern "C" fn allocate_ipc(context: *mut c_void, len: usize) -> *mut u8 { + let out = unsafe { &mut *context.cast::() }; + if len > MAX_BYTES || out.requested.is_some() || out.data.try_reserve_exact(len).is_err() { + return std::ptr::null_mut(); + } + out.requested = Some(len); + out.data.as_mut_ptr() +} + +#[tonic::async_trait] +impl PreparedQuery for JvmQuery { + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + async fn execute(self: Box) -> Result { + Ok(Box::pin(stream::try_unfold( + *self, + |mut query| async move { + let handle = query.handle.clone(); + let task = query.guard.task.clone(); + let result = query + .core + .blocking(move || { + task.initialize()?; + let mut out = IpcBuffer::default(); + let mut error = BridgeBuffer::default(); + let mut row_count = 0; + let result = unsafe { + spark_connect_engine_next_ipc( + handle.pointer(), + allocate_ipc, + (&mut out as *mut IpcBuffer).cast(), + &mut row_count, + &mut error, + ) + }; + if result < 0 { + return Err(engine_error(error)); + } + let _ = take_buffer(error); + if result == 0 { + return Ok(None); + } + // Success guarantees that JNI filled the reserved region. + let data = unsafe { out.into_vec()? }; + Ok(Some(ResultBatch::Ipc { data, row_count })) + }) + .await?; + match result { + Some(batch) => Ok(Some((batch, query))), + None => { + query.guard.finish(); + Ok(None) + } + } + }, + ))) + } +} +fn root(plan: Option) -> Result { + match plan.and_then(|p| p.op_type) { + Some(proto::plan::OpType::Root(r)) => Ok(r), + _ => Err(Status::invalid_argument( + "analysis requires a root relation", + )), + } +} +#[tonic::async_trait] +impl EngineSession for JvmSession { + async fn prepare(&self, operation: proto::plan::OpType) -> Result { + let deadline = Instant::now() + SESSION_ADMISSION_TIMEOUT; + // Preparation accepts read-only plans and excludes configuration writes + // during logical planning. Physical planning stays lazy until the first + // fetch, so configuration changes after prepare can affect execution. + let _catalog = CatalogAccess::Read + .lock_until(&self.catalog, deadline) + .await?; + match operation { + proto::plan::OpType::Root(relation) => Ok(PreparedOperation::Query(Box::new( + self.relation(relation, deadline).await?, + ))), + proto::plan::OpType::Command(command) => match command.command_type { + Some(proto::command::CommandType::SqlCommand(sql)) => { + // Inspect legacy argument fields to reject unsupported encodings. + #[allow(deprecated)] + if !sql.args.is_empty() + || !sql.pos_args.is_empty() + || !sql.named_arguments.is_empty() + || !sql.pos_arguments.is_empty() + { + return Err(Status::unimplemented("Spark engine SQL command arguments")); + } + // Accept the legacy SQL string when the relation input is absent. + #[allow(deprecated)] + let relation = match sql.input { + Some(r) => r, + None if !sql.sql.is_empty() => proto::Relation { + rel_type: Some(proto::relation::RelType::Sql(proto::Sql { + query: sql.sql, + ..Default::default() + })), + ..Default::default() + }, + _ => return Err(Status::invalid_argument("missing SQL input")), + }; + // Validate as a read-only query before returning the relation. + let _metadata = self + .metadata(relation.clone(), AnalysisMode::Schema, deadline) + .await?; + Ok(PreparedOperation::Command(vec![ + proto::execute_plan_response::ResponseType::SqlCommandResult( + proto::execute_plan_response::SqlCommandResult { + relation: Some(relation), + }, + ), + ])) + } + _ => Err(Status::unimplemented( + "Spark engine command is not implemented", + )), + }, + _ => Err(Status::unimplemented("compressed Spark engine plans")), + } + } + async fn analyze( + &self, + input: proto::analyze_plan_request::Analyze, + ) -> Result { + use proto::{analyze_plan_request::Analyze, analyze_plan_response as out}; + let deadline = Instant::now() + SESSION_ADMISSION_TIMEOUT; + let _catalog = CatalogAccess::Read + .lock_until(&self.catalog, deadline) + .await?; + Ok(match input { + Analyze::Schema(input) => { + let metadata = self + .metadata(root(input.plan)?, AnalysisMode::Schema, deadline) + .await?; + out::Result::Schema(out::Schema { + schema: Some(spark_connect_server::engine::spark_schema( + metadata.schema.as_ref(), + )?), + }) + } + Analyze::IsLocal(input) => { + let metadata = self + .metadata(root(input.plan)?, AnalysisMode::IsLocal, deadline) + .await?; + out::Result::IsLocal(out::IsLocal { + is_local: metadata.flag, + }) + } + Analyze::IsStreaming(input) => { + let metadata = self + .metadata(root(input.plan)?, AnalysisMode::IsStreaming, deadline) + .await?; + out::Result::IsStreaming(out::IsStreaming { + is_streaming: metadata.flag, + }) + } + _ => { + return Err(Status::unimplemented( + "Spark engine analysis is not implemented", + )) + } + }) + } + async fn config( + &self, + input: proto::config_request::operation::OpType, + ) -> Result { + use proto::config_request::operation::OpType; + let deadline = Instant::now() + SESSION_ADMISSION_TIMEOUT; + let silent = matches!(&input, OpType::Set(value) if value.silent == Some(true)); + let keys = |items: Vec| { + items + .into_iter() + .map(|key| proto::KeyValue { key, value: None }) + .collect::>() + }; + let (opcode, pairs) = match input { + OpType::Get(v) => (1, keys(v.keys)), + OpType::GetOption(v) => (2, keys(v.keys)), + OpType::GetWithDefault(v) => (3, v.pairs), + OpType::GetAll(v) => (4, v.prefix.map(|p| keys(vec![p])).unwrap_or_default()), + OpType::IsModifiable(v) => (5, keys(v.keys)), + OpType::Set(v) => (6, v.pairs), + OpType::Unset(v) => (7, keys(v.keys)), + }; + let _catalog = CatalogAccess::config(opcode) + .lock_until(&self.catalog, deadline) + .await?; + if silent { + // Spark's silent Set applies valid keys even when another key fails. + // Each native call has its own task: completing one cannot mask + // cancellation of the next. The catalog writer spans the full set. + return silent_config_set(pairs, |pair| self.config_pairs(8, vec![pair], deadline)) + .await; + } + Ok(ConfigResult { + pairs: self.config_pairs(opcode, pairs, deadline).await?, + warnings: Vec::new(), + }) + } +} + +async fn silent_config_set( + pairs: Vec, + mut set: F, +) -> Result +where + F: FnMut(proto::KeyValue) -> Fut, + Fut: std::future::Future, Status>>, +{ + let mut warnings = Vec::new(); + for pair in pairs { + // Opcode 8 returns sanitized setter failures as typed pairs. JNI and + // operation errors retain their status/provenance and abort the RPC. + for failure in set(pair).await? { + let reason = failure + .value + .ok_or_else(|| Status::internal("missing Spark config warning reason"))?; + warnings.push(format!("Failed to set {} ({reason})", failure.key)); + } + } + Ok(ConfigResult { + pairs: Vec::new(), + warnings, + }) +} + +fn encode_pairs(pairs: &[proto::KeyValue]) -> Result, Status> { + let mut bytes = Vec::new(); + bytes.extend_from_slice( + &i32::try_from(pairs.len()) + .map_err(|_| Status::resource_exhausted("too many config fields"))? + .to_be_bytes(), + ); + for pair in pairs { + let key_len = i32::try_from(pair.key.len()) + .map_err(|_| Status::resource_exhausted("config key too large"))?; + bytes.extend_from_slice(&key_len.to_be_bytes()); + bytes.extend_from_slice(pair.key.as_bytes()); + if let Some(value) = &pair.value { + let len = i32::try_from(value.len()) + .map_err(|_| Status::resource_exhausted("config value too large"))?; + bytes.extend_from_slice(&len.to_be_bytes()); + bytes.extend_from_slice(value.as_bytes()); + } else { + bytes.extend_from_slice(&(-1_i32).to_be_bytes()); + } + } + Ok(bytes) +} + +// SparkJvmEngine uses DataInputStream/DataOutputStream: each field starts with +// a big-endian 32-bit length, and only nullable values may use the -1 sentinel. +struct ConfigCursor<'a>(&'a [u8]); +impl<'a> ConfigCursor<'a> { + fn integer(&mut self) -> Result { + let bytes = self + .0 + .get(..4) + .ok_or_else(|| Status::internal("truncated Spark config integer"))?; + let value = u32::from_be_bytes(bytes.try_into().unwrap()); + self.0 = &self.0[4..]; + Ok(value) + } + + fn bytes(&mut self) -> Result<&'a [u8], Status> { + let length = self.integer()? as usize; + let bytes = self + .0 + .get(..length) + .ok_or_else(|| Status::internal("invalid Spark config key length"))?; + self.0 = &self.0[length..]; + Ok(bytes) + } +} + +fn decode_pairs(bytes: &[u8]) -> Result, Status> { + let mut cursor = ConfigCursor(bytes); + let count = cursor.integer()? as usize; + if count > cursor.0.len() / 8 { + return Err(Status::internal("invalid Spark config pair count")); + } + let mut result = Vec::with_capacity(count); + for _ in 0..count { + let key = String::from_utf8(cursor.bytes()?.to_vec()) + .map_err(|_| Status::internal("invalid Spark config UTF8"))?; + let len = cursor.integer()? as i32; + let value = if len == -1 { + None + } else if len < 0 { + return Err(Status::internal("invalid Spark config value length")); + } else { + let len = len as usize; + let field = cursor + .0 + .get(..len) + .ok_or_else(|| Status::internal("truncated Spark config value"))?; + cursor.0 = &cursor.0[len..]; + Some( + String::from_utf8(field.to_vec()) + .map_err(|_| Status::internal("invalid Spark config UTF8"))?, + ) + }; + result.push(proto::KeyValue { key, value }); + } + if !cursor.0.is_empty() { + return Err(Status::internal("trailing Spark config data")); + } + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + static CLEANUP_TEST_LOCK: Mutex<()> = Mutex::new(()); + + #[test] + fn ipc_destination_commits_the_same_allocation_after_copy() { + let mut out = IpcBuffer::default(); + let mut source = [0, 1, 127, 128, 255]; + let context = (&mut out as *mut IpcBuffer).cast(); + let destination = unsafe { allocate_ipc(context, source.len()) }; + assert!(!destination.is_null()); + assert!(out.data.is_empty()); + // Simulate the successful JNI region write before committing the length. + unsafe { std::ptr::copy_nonoverlapping(source.as_ptr(), destination, source.len()) }; + source.fill(0); + let data = unsafe { out.into_vec().unwrap() }; + assert_eq!(data.as_ptr(), destination); + assert_eq!(data, [0, 1, 127, 128, 255]); + } + + #[test] + fn ipc_destination_rejects_oversize_and_repeated_allocation() { + let mut out = IpcBuffer::default(); + let context = (&mut out as *mut IpcBuffer).cast(); + assert!(unsafe { allocate_ipc(context, MAX_BYTES + 1) }.is_null()); + assert!(out.requested.is_none()); + let destination = unsafe { allocate_ipc(context, 8) }; + assert!(!destination.is_null()); + assert!(unsafe { allocate_ipc(context, 8) }.is_null()); + assert_eq!(out.data.as_mut_ptr(), destination); + // Dropping an uncommitted buffer does not expose its reserved bytes. + assert!(out.data.is_empty()); + drop(out); + } + + #[test] + fn ipc_destination_distinguishes_empty_batch_from_missing_allocation() { + assert!(unsafe { IpcBuffer::default().into_vec() }.is_err()); + let mut out = IpcBuffer::default(); + let context = (&mut out as *mut IpcBuffer).cast(); + assert!(!unsafe { allocate_ipc(context, 0) }.is_null()); + assert!(unsafe { out.into_vec().unwrap() }.is_empty()); + } + + fn metadata_payload(flag: u8) -> (Vec, arrow::datatypes::Schema) { + use arrow::{ + datatypes::{DataType, Field, Schema}, + ipc::writer::StreamWriter, + }; + let schema = Schema::new(vec![Field::new("时区 😀", DataType::Utf8, true)]); + let mut bytes = vec![flag]; + // Deliberately only a schema header: metadata preserves the existing + // StreamReader::try_new/schema validation boundary, without reading data. + drop(StreamWriter::try_new(&mut bytes, &schema).unwrap()); + (bytes, schema) + } + #[test] + fn metadata_decoder_preserves_schema_and_boolean_results() { + for mode in [ + AnalysisMode::Schema, + AnalysisMode::IsLocal, + AnalysisMode::IsStreaming, + ] { + let (bytes, schema) = metadata_payload(0); + let metadata = decode_metadata(bytes, mode).unwrap(); + assert_eq!(metadata.schema.as_ref(), &schema); + assert!(!metadata.flag); + } + for mode in [AnalysisMode::IsLocal, AnalysisMode::IsStreaming] { + let (bytes, schema) = metadata_payload(1); + let metadata = decode_metadata(bytes, mode).unwrap(); + assert_eq!(metadata.schema.as_ref(), &schema); + assert!(metadata.flag); + } + } + #[test] + fn metadata_decoder_rejects_missing_invalid_and_schema_true_flags() { + for bytes in [vec![], vec![2], vec![255]] { + for mode in [ + AnalysisMode::Schema, + AnalysisMode::IsLocal, + AnalysisMode::IsStreaming, + ] { + let status = decode_metadata(bytes.clone(), mode).err().unwrap(); + assert_eq!(status.code(), tonic::Code::Internal); + assert_eq!(status.message(), "invalid Spark metadata flag"); + } + } + let (bytes, _) = metadata_payload(1); + assert_eq!( + decode_metadata(bytes, AnalysisMode::Schema) + .err() + .unwrap() + .message(), + "invalid Spark metadata flag" + ); + } + #[test] + fn metadata_decoder_validates_ipc_for_every_mode() { + for mode in [ + AnalysisMode::Schema, + AnalysisMode::IsLocal, + AnalysisMode::IsStreaming, + ] { + for bytes in [vec![0], vec![0, 1, 2, 3], vec![0, 255, 255, 255, 255]] { + let status = decode_metadata(bytes, mode).err().unwrap(); + assert_eq!(status.code(), tonic::Code::Internal); + assert!(status.message().starts_with("invalid Spark Arrow stream:")); + } + } + } + #[test] + fn metadata_decoder_size_limit_excludes_transport_prefix() { + let (bytes, schema) = metadata_payload(0); + let ipc_limit = bytes.len() - 1; + assert_eq!( + decode_metadata_with_limit(bytes.clone(), AnalysisMode::Schema, ipc_limit) + .unwrap() + .schema + .as_ref(), + &schema + ); + assert_eq!( + decode_metadata_with_limit(bytes, AnalysisMode::Schema, ipc_limit - 1) + .err() + .unwrap() + .code(), + tonic::Code::ResourceExhausted + ); + } + + #[derive(Debug)] + struct CleanupAttachmentEvent { + action: &'static str, + core: usize, + thread: std::thread::ThreadId, + workers: usize, + handles: usize, + } + struct TestCleanupWorker { + core: Arc, + events: mpsc::Sender, + release: Option>, + _thread: PhantomData>, + } + impl TestCleanupWorker { + fn new( + core: &Arc, + events: &mpsc::Sender, + release: Option>, + ) -> Self { + let worker = Self { + core: core.clone(), + events: events.clone(), + release, + _thread: PhantomData, + }; + worker.record("attach"); + worker + } + fn record(&self, action: &'static str) { + let counts = self.core.counts.lock().unwrap(); + let _ = self.events.send(CleanupAttachmentEvent { + action, + core: Arc::as_ptr(&self.core) as usize, + thread: std::thread::current().id(), + workers: counts.workers, + handles: counts.handles, + }); + } + } + impl Drop for TestCleanupWorker { + fn drop(&mut self) { + self.record("detach"); + if let Some(release) = &self.release { + // The test owns a CleanupBlock that also releases on unwind. + let _ = release.recv(); + } + } + } + fn empty_cleanup(core: &Arc) -> Cleanup { + core.counts.lock().unwrap().handles += 1; + Cleanup::Free { + pointer: 0, + core: core.clone(), + permit: None, + lane: CleanupLane::Task, + retained_task: None, + } + } + fn attachment_event(rx: &mpsc::Receiver) -> CleanupAttachmentEvent { + rx.recv_timeout(Duration::from_secs(5)).unwrap() + } + struct CleanupTestLane { + thread: std::thread::JoinHandle<()>, + done: mpsc::Receiver<()>, + } + impl CleanupTestLane { + fn spawn(action: impl FnOnce() + Send + 'static) -> Self { + let (done_tx, done) = mpsc::channel(); + let thread = std::thread::spawn(move || { + action(); + let _ = done_tx.send(()); + }); + Self { thread, done } + } + fn join(self) { + self.done.recv_timeout(Duration::from_secs(5)).unwrap(); + self.thread.join().unwrap(); + } + } + fn cleanup_drained(core: Arc) { + let (done_tx, done_rx) = mpsc::channel(); + let waiter = std::thread::spawn(move || { + core.wait_drained(); + let _ = done_tx.send(()); + }); + done_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + waiter.join().unwrap(); + } + #[test] + fn cleanup_attachment_detaches_on_idle_without_closing_the_lane() { + let core = JvmEngine::new(std::ptr::null_mut()); + let (tx, rx) = mpsc::channel(); + let (events_tx, events_rx) = mpsc::channel(); + assert!(tx.send(empty_cleanup(&core)).is_ok()); + let lane = CleanupTestLane::spawn(move || { + // Zero makes the empty-queue timeout deterministic, independent of + // scheduler speed; the sender stays open until detach is observed. + cleanup_loop_with(rx, Duration::ZERO, |core| { + Ok(TestCleanupWorker::new(core, &events_tx, None)) + }); + }); + let attached = attachment_event(&events_rx); + let detached = attachment_event(&events_rx); + assert_eq!((attached.action, detached.action), ("attach", "detach")); + assert_eq!(attached.thread, detached.thread); + assert_eq!((detached.workers, detached.handles), (1, 0)); + cleanup_drained(core.clone()); + assert_eq!(core.counts.lock().unwrap().workers, 0); + drop(tx); + lane.join(); + } + #[test] + fn cleanup_attachment_reuses_a_burst_and_detaches_before_switching_engines() { + let first = JvmEngine::new(std::ptr::null_mut()); + let second = JvmEngine::new(std::ptr::null_mut()); + let (tx, rx) = mpsc::channel(); + let (events_tx, events_rx) = mpsc::channel(); + for core in [&first, &first, &second] { + assert!(tx.send(empty_cleanup(core)).is_ok()); + } + drop(tx); + let first_core = first.clone(); + let lane = CleanupTestLane::spawn(move || { + // All work and disconnect are queued before the receiver starts. + cleanup_loop_with(rx, Duration::from_secs(60), |core| { + if !Arc::ptr_eq(core, &first_core) { + assert_eq!(first_core.counts.lock().unwrap().workers, 0); + } + Ok(TestCleanupWorker::new(core, &events_tx, None)) + }); + }); + lane.join(); + let events = events_rx.try_iter().collect::>(); + assert_eq!( + events.iter().map(|event| event.action).collect::>(), + ["attach", "detach", "attach", "detach"] + ); + assert_eq!(events[0].core, Arc::as_ptr(&first) as usize); + assert_eq!(events[1].core, events[0].core); + assert_eq!(events[2].core, Arc::as_ptr(&second) as usize); + assert_eq!(events[3].core, events[2].core); + assert!(events.iter().all(|event| event.thread == events[0].thread)); + assert!(events.iter().all(|event| event.workers == 1)); + assert_eq!((events[1].handles, events[3].handles), (0, 0)); + for core in [first, second] { + assert_eq!(core.counts.lock().unwrap().workers, 0); + cleanup_drained(core); + } + } + #[test] + fn cleanup_attachment_activity_outlives_a_blocked_detach_during_shutdown() { + let core = JvmEngine::new(std::ptr::null_mut()); + let (tx, rx) = mpsc::channel(); + let (events_tx, events_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let release = CleanupBlock(release_tx); + assert!(tx.send(empty_cleanup(&core)).is_ok()); + drop(tx); + let lane = CleanupTestLane::spawn(move || { + let mut release_rx = Some(release_rx); + cleanup_loop_with(rx, Duration::from_secs(60), |core| { + Ok(TestCleanupWorker::new(core, &events_tx, release_rx.take())) + }); + }); + let attached = attachment_event(&events_rx); + let detaching = attachment_event(&events_rx); + assert_eq!((attached.action, detaching.action), ("attach", "detach")); + assert_eq!(attached.thread, detaching.thread); + assert_eq!((detaching.workers, detaching.handles), (1, 0)); + core.shutdown(); + assert!(core + .take_factory_after_drain() + .unwrap_err() + .contains("drained work")); + let waiting_core = core.clone(); + let (done_tx, done_rx) = mpsc::channel(); + let waiter = std::thread::spawn(move || { + waiting_core.wait_drained(); + done_tx.send(()).unwrap(); + }); + assert_eq!(core.counts.lock().unwrap().workers, 1); + drop(release); + done_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + waiter.join().unwrap(); + lane.join(); + assert_eq!(core.counts.lock().unwrap().workers, 0); + core.release_factory().unwrap(); + } + #[test] + fn cleanup_attachment_failure_releases_activity_and_closed_engines_still_clean_up() { + let core = JvmEngine::new(std::ptr::null_mut()); + let status = CleanupAttachment::attach_with(&core, |core| { + assert_eq!(core.counts.lock().unwrap().workers, 1); + Err::(Status::internal("injected attach failure")) + }) + .err() + .unwrap(); + assert_eq!(status.code(), tonic::Code::Internal); + assert_eq!(core.counts.lock().unwrap().workers, 0); + core.shutdown(); + let (tx, rx) = mpsc::channel(); + assert!(tx.send(empty_cleanup(&core)).is_ok()); + drop(tx); + cleanup_loop_with( + rx, + Duration::ZERO, + |_| -> Result { + panic!("closed cleanup must use the per-call fallback") + }, + ); + assert_eq!(core.counts.lock().unwrap().handles, 0); + assert_eq!(core.counts.lock().unwrap().workers, 0); + cleanup_drained(core.clone()); + core.release_factory().unwrap(); + } + + #[test] + fn cleanup_attachment_success_resets_failure_reporting() { + let core = JvmEngine::new(std::ptr::null_mut()); + let (events_tx, _events_rx) = mpsc::channel(); + let mut attempts = 0; + let mut attach = |core: &Arc| { + attempts += 1; + if attempts == 2 { + Ok(TestCleanupWorker::new(core, &events_tx, None)) + } else { + Err(Status::internal("injected attach failure")) + } + }; + let mut reported = None; + let mut reports = 0; + for (failed, expected_reports) in [(true, 1), (false, 1), (true, 2)] { + let attachment = + CleanupAttachment::attach_or_fallback(&core, &mut attach, &mut reported, |_| { + reports += 1 + }); + assert_eq!(attachment.is_none(), failed); + assert_eq!(reports, expected_reports); + assert_eq!( + reported + .as_ref() + .is_some_and(|reported| Weak::ptr_eq(reported, &Arc::downgrade(&core))), + failed + ); + drop(attachment); + assert_eq!(core.counts.lock().unwrap().workers, 0); + } + assert_eq!(attempts, 3); + cleanup_drained(core); + } + + #[test] + fn cleanup_attachment_failure_reporting_tracks_engines_without_retaining_them() { + let first = JvmEngine::new(std::ptr::null_mut()); + let second = JvmEngine::new(std::ptr::null_mut()); + let mut reported = None; + let mut reports = 0; + for (core, expected_reports) in [ + (&first, 1), + (&first, 1), + (&second, 2), + (&second, 2), + (&first, 3), + ] { + let attachment = CleanupAttachment::attach_or_fallback( + core, + |_| -> Result { + Err(Status::internal("injected attach failure")) + }, + &mut reported, + |_| reports += 1, + ); + assert!(attachment.is_none()); + assert_eq!(reports, expected_reports); + assert!(Weak::ptr_eq( + reported.as_ref().unwrap(), + &Arc::downgrade(core) + )); + assert_eq!(Arc::strong_count(core), 1); + assert_eq!(core.counts.lock().unwrap().workers, 0); + } + cleanup_drained(first.clone()); + cleanup_drained(second.clone()); + drop(first); + drop(second); + assert!(reported.unwrap().upgrade().is_none()); + } + + #[test] + fn cleanup_attachment_loop_retries_failures_and_drains_every_item() { + let first = JvmEngine::new(std::ptr::null_mut()); + let second = JvmEngine::new(std::ptr::null_mut()); + let (tx, rx) = mpsc::channel(); + for core in [&first, &first, &second, &second] { + assert!(tx.send(empty_cleanup(core)).is_ok()); + } + drop(tx); + let attempts = Arc::new(AtomicUsize::new(0)); + let worker_attempts = attempts.clone(); + let lane = CleanupTestLane::spawn(move || { + cleanup_loop_with( + rx, + Duration::ZERO, + |_| -> Result { + worker_attempts.fetch_add(1, Ordering::SeqCst); + Err(Status::internal("injected attach failure")) + }, + ); + }); + lane.join(); + assert_eq!(attempts.load(Ordering::SeqCst), 4); + for core in [first, second] { + { + let counts = core.counts.lock().unwrap(); + assert_eq!((counts.handles, counts.workers), (0, 0)); + } + cleanup_drained(core); + } + } + + #[test] + fn factory_release_requires_drain_and_transfers_ownership_once_with_core_arcs_retained() { + // This is only an opaque ownership token, never a JNI handle. Prevent + // the fallback destructor from receiving it if a test assertion fails. + let token = std::ptr::NonNull::::dangling().as_ptr().cast(); + let core = std::mem::ManuallyDrop::new(JvmEngine::new(token)); + let error = core.take_factory_after_drain().unwrap_err(); + assert!(error.contains("requires shutdown"), "{error}"); + core.shutdown(); + core.counts.lock().unwrap().workers += 1; + let error = core.take_factory_after_drain().unwrap_err(); + assert!(error.contains("requires drained work"), "{error}"); + core.counts.lock().unwrap().workers -= 1; + core.set_worker_attachment_count_for_test(1); + let error = core.take_factory_after_drain().unwrap_err(); + assert!(error.contains("1 JNI worker attachments remain"), "{error}"); + core.set_worker_attachment_count_for_test(0); + // A cleanup worker can retain an Arc after decrementing its counter. + // Factory release must already complete while that Arc is still alive. + let delayed_cleanup = Arc::clone(&core); + assert_eq!(core.take_factory_after_drain().unwrap(), token); + assert!(core.take_factory_after_drain().unwrap().is_null()); + assert!(delayed_cleanup.factory.load(Ordering::SeqCst).is_null()); + core.release_factory().unwrap(); + drop(delayed_cleanup); + // Ownership was taken above; the remaining destructor is a C no-op. + drop(std::mem::ManuallyDrop::into_inner(core)); + } + #[test] + fn failed_cleanup_retains_session_admission_until_explicit_retry() { + let core = JvmEngine::new(std::ptr::null_mut()); + let permit = core + .session_permits + .clone() + .try_acquire_many_owned(64) + .unwrap(); + core.counts.lock().unwrap().handles += 1; + let item = Cleanup::Free { + pointer: 0, + core: core.clone(), + permit: Some(permit), + lane: CleanupLane::Session, + retained_task: None, + }; + let attempts = AtomicUsize::new(0); + cleanup_with(item, |pointer| { + assert_eq!(pointer, 0); + attempts.fetch_add(1, Ordering::SeqCst); + Err(Status::internal("injected Java close failure")) + }); + assert_eq!(attempts.load(Ordering::SeqCst), 1); + assert_eq!(core.session_permits.available_permits(), 0); + assert_eq!(core.counts.lock().unwrap().handles, 1); + assert!(core + .cleanup_failure() + .unwrap() + .contains("injected Java close failure")); + let mut failed = core.take_failed_cleanup(); + assert_eq!(failed.len(), 1); + assert!(matches!(failed[0].item.lane(), CleanupLane::Session)); + cleanup_with(failed.pop().unwrap().item, |_| { + attempts.fetch_add(1, Ordering::SeqCst); + Ok(()) + }); + assert_eq!(attempts.load(Ordering::SeqCst), 2); + assert_eq!(core.session_permits.available_permits(), 64); + assert_eq!(core.counts.lock().unwrap().handles, 0); + assert!(core.cleanup_failure().is_none()); + } + #[test] + fn failed_cancel_retains_task_admission_until_delivery_is_retried() { + let _serial = CLEANUP_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let core = JvmEngine::new(std::ptr::null_mut()); + let permit = core.permits.clone().try_acquire_many_owned(64).unwrap(); + let task = empty_handle(&core, HandleKind::Task, Some(permit), None); + cleanup_with_actions( + Cleanup::Cancel(task), + |_| unreachable!(), + |_| Err(Status::unavailable("injected attachment failure")), + ); + assert_eq!(core.permits.available_permits(), 0); + assert_eq!(core.counts.lock().unwrap().handles, 1); + assert!(core + .cleanup_failure() + .unwrap() + .contains("injected attachment failure")); + let failure = core.take_failed_cleanup().pop().unwrap(); + assert!(matches!(failure.item, Cleanup::Cancel(_))); + cleanup_with_actions(failure.item, |_| unreachable!(), |_| Ok(())); + cleanup_fence(CleanupLane::Task); + assert_eq!(core.permits.available_permits(), 64); + assert_eq!(core.counts.lock().unwrap().handles, 0); + } + #[test] + fn cleanup_spawn_failure_is_not_cached_and_never_runs_the_item_inline() { + let core = JvmEngine::new(std::ptr::null_mut()); + let sender = Mutex::new(None); + let (item, error) = enqueue_cleanup(empty_cleanup(&core), &sender, |_| { + Err(std::io::Error::other("injected spawn failure")) + }) + .err() + .unwrap(); + assert!(sender.lock().unwrap().is_none()); + assert_eq!(core.counts.lock().unwrap().handles, 1); + retain_failed_cleanup(item, error); + let failure = core.take_failed_cleanup().pop().unwrap(); + let mut receiver = None; + assert!(enqueue_cleanup(failure.item, &sender, |rx| { + receiver = Some(rx); + Ok(()) + }) + .is_ok()); + let item = receiver + .unwrap() + .recv_timeout(Duration::from_secs(1)) + .unwrap(); + cleanup_with(item, |_| Ok(())); + assert_eq!(core.counts.lock().unwrap().handles, 0); + } + #[test] + fn disconnected_cleanup_sender_is_replaced() { + let core = JvmEngine::new(std::ptr::null_mut()); + let (tx, rx) = mpsc::channel(); + drop(rx); + let sender = Mutex::new(Some(tx)); + let mut receiver = None; + assert!(enqueue_cleanup(empty_cleanup(&core), &sender, |rx| { + receiver = Some(rx); + Ok(()) + }) + .is_ok()); + let item = receiver + .unwrap() + .recv_timeout(Duration::from_secs(1)) + .unwrap(); + cleanup_with(item, |_| Ok(())); + assert_eq!(core.counts.lock().unwrap().handles, 0); + } + #[test] + fn detach_retries_only_on_the_owning_thread_and_reports_persistent_failure() { + let owner = std::thread::current().id(); + let mut attempts = 0; + detach_with_retry(|| { + assert_eq!(std::thread::current().id(), owner); + attempts += 1; + if attempts == 1 { + Err(Status::internal("transient")) + } else { + Ok(()) + } + }) + .unwrap(); + assert_eq!(attempts, 2); + attempts = 0; + let error = detach_with_retry(|| { + attempts += 1; + Err(Status::internal("persistent")) + }) + .unwrap_err(); + assert_eq!(attempts, 2); + assert_eq!(error.message(), "persistent"); + } + #[test] + fn broken_diagnostic_output_does_not_unwind_cleanup() { + struct BrokenPipe; + impl Write for BrokenPipe { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::from(std::io::ErrorKind::BrokenPipe)) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + log_diagnostic_to(BrokenPipe, format_args!("failed cleanup")); + } + #[test] + fn failed_query_cleanup_retains_its_task_and_retries_on_the_query_lane() { + let _serial = CLEANUP_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let core = JvmEngine::new(std::ptr::null_mut()); + let permit = core.permits.clone().try_acquire_many_owned(64).unwrap(); + let task = empty_handle(&core, HandleKind::Task, Some(permit), None); + core.counts.lock().unwrap().handles += 1; + cleanup_with( + Cleanup::Free { + pointer: 0, + core: core.clone(), + permit: None, + lane: CleanupLane::Query, + retained_task: Some(task), + }, + |_| Err(Status::internal("injected query close failure")), + ); + cleanup_fence(CleanupLane::Task); + assert_eq!(core.permits.available_permits(), 0); + assert_eq!(core.counts.lock().unwrap().handles, 2); + let blocked = block_cleanup(CleanupLane::Query); + core.retry_failed_cleanup(); + cleanup_fence(CleanupLane::Task); + assert_eq!(core.permits.available_permits(), 0); + assert_eq!(core.counts.lock().unwrap().handles, 2); + drop(blocked); + cleanup_fence(CleanupLane::Query); + cleanup_fence(CleanupLane::Task); + assert_eq!(core.permits.available_permits(), 64); + assert_eq!(core.counts.lock().unwrap().handles, 0); + assert!(core.cleanup_failure().is_none()); + } + #[test] + fn cleanup_retry_snapshot_preserves_failures_arriving_during_retry() { + let core = JvmEngine::new(std::ptr::null_mut()); + cleanup_with(empty_cleanup(&core), |_| Err(Status::internal("first"))); + let mut retry = core.take_failed_cleanup(); + cleanup_with(empty_cleanup(&core), |_| { + Err(Status::internal("concurrent")) + }); + cleanup_with(retry.pop().unwrap().item, |_| Ok(())); + assert_eq!(core.counts.lock().unwrap().handles, 1); + assert!(core.cleanup_failure().unwrap().contains("concurrent")); + let mut remaining = core.take_failed_cleanup(); + assert_eq!(remaining.len(), 1); + cleanup_with(remaining.pop().unwrap().item, |_| Ok(())); + assert_eq!(core.counts.lock().unwrap().handles, 0); + } + #[test] + fn failed_factory_cleanup_restores_the_same_handle_for_explicit_retry() { + let token = std::ptr::NonNull::::dangling().as_ptr().cast(); + // Do not let an assertion unwind send the opaque token to real JNI. + let core = std::mem::ManuallyDrop::new(JvmEngine::new(token)); + core.shutdown(); + let error = core + .release_factory_with(|pointer| { + assert_eq!(pointer, token as usize); + Err(Status::internal("injected factory close failure")) + }) + .unwrap_err(); + assert!(error.starts_with("JNI cleanup is incomplete; retained cleanup failed:")); + assert_eq!(core.factory.load(Ordering::SeqCst), token); + core.release_factory_with(|pointer| { + assert_eq!(pointer, token as usize); + Ok(()) + }) + .unwrap(); + assert!(core.factory.load(Ordering::SeqCst).is_null()); + drop(std::mem::ManuallyDrop::into_inner(core)); + } + struct CleanupBlock(mpsc::Sender<()>); + impl Drop for CleanupBlock { + fn drop(&mut self) { + let _ = self.0.send(()); + } + } + fn block_cleanup(lane: CleanupLane) -> CleanupBlock { + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + queue(Cleanup::TestAction { + lane, + action: Box::new(move || { + if entered_tx.send(()).is_ok() { + let _ = release_rx.recv(); + } + }), + }); + let block = CleanupBlock(release_tx); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + block + } + fn cleanup_fence(lane: CleanupLane) { + let (done_tx, done_rx) = mpsc::channel(); + queue(Cleanup::TestAction { + lane, + action: Box::new(move || { + let _ = done_tx.send(()); + }), + }); + done_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + } + fn empty_handle( + core: &Arc, + kind: HandleKind, + permit: Option, + retained_task: Option>, + ) -> Arc { + core.counts.lock().unwrap().handles += 1; + Arc::new(Handle { + // C spark_connect_engine_free(NULL) is a no-op; counters/permits still take the exact + // production deferred cleanup path without a running JVM. + pointer: std::ptr::null_mut(), + core: core.clone(), + permit: Mutex::new(permit), + kind, + retained_task, + }) + } + #[test] + fn slow_session_close_does_not_block_task_admission_reclamation() { + let _serial = CLEANUP_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let blocked = block_cleanup(CleanupLane::Session); + let core = JvmEngine::new(std::ptr::null_mut()); + let session_permit = core + .session_permits + .clone() + .try_acquire_many_owned(64) + .unwrap(); + let task_permit = core.permits.clone().try_acquire_many_owned(64).unwrap(); + drop(empty_handle( + &core, + HandleKind::Session, + Some(session_permit), + None, + )); + drop(empty_handle( + &core, + HandleKind::Task, + Some(task_permit), + None, + )); + cleanup_fence(CleanupLane::Task); + assert_eq!(core.permits.available_permits(), 64); + assert_eq!(core.session_permits.available_permits(), 0); + assert_eq!(core.counts.lock().unwrap().handles, 1); + drop(blocked); + core.wait_drained(); + assert_eq!(core.session_permits.available_permits(), 64); + } + #[test] + fn query_close_retains_task_admission_until_query_cleanup_finishes() { + let _serial = CLEANUP_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let blocked = block_cleanup(CleanupLane::Query); + let core = JvmEngine::new(std::ptr::null_mut()); + let permit = core.permits.clone().try_acquire_many_owned(64).unwrap(); + let task = empty_handle(&core, HandleKind::Task, Some(permit), None); + drop(empty_handle(&core, HandleKind::Query, None, Some(task))); + cleanup_fence(CleanupLane::Task); + assert_eq!(core.permits.available_permits(), 0); + assert_eq!(core.counts.lock().unwrap().handles, 2); + drop(blocked); + core.wait_drained(); + assert_eq!(core.permits.available_permits(), 64); + } + #[test] + fn slow_session_close_does_not_block_query_task_reclamation() { + let _serial = CLEANUP_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let blocked = block_cleanup(CleanupLane::Session); + let core = JvmEngine::new(std::ptr::null_mut()); + let permit = core.permits.clone().try_acquire_many_owned(64).unwrap(); + let task = empty_handle(&core, HandleKind::Task, Some(permit), None); + drop(empty_handle(&core, HandleKind::Query, None, Some(task))); + cleanup_fence(CleanupLane::Query); + cleanup_fence(CleanupLane::Task); + assert_eq!(core.permits.available_permits(), 64); + assert_eq!(core.counts.lock().unwrap().handles, 0); + drop(blocked); + core.wait_drained(); + } + #[tokio::test] + async fn session_admission_times_out_and_recovers_after_actual_session_free() { + let core = JvmEngine::new(std::ptr::null_mut()); + let held = core + .session_permits + .clone() + .try_acquire_many_owned(64) + .unwrap(); + let status = core + .admit_session(Instant::now() + Duration::from_millis(2)) + .await + .err() + .unwrap(); + assert_eq!(status.code(), tonic::Code::ResourceExhausted, "{status}"); + assert_eq!(core.permits.available_permits(), 64); + { + let _serial = CLEANUP_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + drop(empty_handle(&core, HandleKind::Session, Some(held), None)); + cleanup_fence(CleanupLane::Session); + } + let restored = core + .admit_session(Instant::now() + Duration::from_millis(2)) + .await + .unwrap(); + assert_eq!(core.session_permits.available_permits(), 63); + drop(restored); + core.wait_drained(); + } + #[tokio::test(start_paused = true)] + async fn task_admission_bounds_shared_query_calls_and_recovers_after_capacity_returns() { + let core = JvmEngine::new(std::ptr::null_mut()); + let held = core.permits.clone().try_acquire_many_owned(64).unwrap(); + let session = JvmSession { + core: core.clone(), + context: SessionContext { + session_id: "task-capacity-session".into(), + user_context: None, + }, + handle: AsyncMutex::new(None), + catalog: RwLock::new(()), + }; + // Cache a synthetic session so every call reaches task admission without + // crossing JNI. Its null handle is safe to reclaim through the C shim. + *session.handle.lock().await = Some(empty_handle(&core, HandleKind::Session, None, None)); + let relation = || proto::Relation { + rel_type: Some(proto::relation::RelType::Sql(proto::Sql { + query: "SELECT 1".into(), + ..Default::default() + })), + ..Default::default() + }; + { + // Read-only preparation must overlap an existing catalog reader. + // An accidental exclusive prepare lock would never reach its timer. + let _reader = CatalogAccess::Read.lock(&session.catalog).await; + let calls = async { + futures::join!( + async { + session + .prepare(proto::plan::OpType::Root(relation())) + .await + .map(|_| ()) + }, + async { + session + .analyze(proto::analyze_plan_request::Analyze::Schema( + proto::analyze_plan_request::Schema { + plan: Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(relation())), + }), + }, + )) + .await + .map(|_| ()) + }, + async { session.config(config_get()).await.map(|_| ()) }, + ) + }; + tokio::pin!(calls); + let started = Instant::now(); + assert!(futures::poll!(&mut calls).is_pending()); + assert!(core.counts.lock().unwrap().tasks.is_empty()); + tokio::time::advance(SESSION_ADMISSION_TIMEOUT).await; + let results = match futures::poll!(&mut calls) { + std::task::Poll::Ready(results) => results, + std::task::Poll::Pending => panic!("task admission exceeded its deadline"), + }; + for result in [results.0, results.1, results.2] { + let status = result.unwrap_err(); + assert_eq!(status.code(), tonic::Code::ResourceExhausted, "{status}"); + assert_eq!(status.message(), "Spark engine task capacity exhausted"); + } + assert!(started.elapsed() <= SESSION_ADMISSION_TIMEOUT); + } + assert!(session.catalog.try_write().is_ok()); + assert!(core.counts.lock().unwrap().tasks.is_empty()); + assert_eq!(core.counts.lock().unwrap().workers, 0); + assert_eq!(core.session_permits.available_permits(), 64); + drop(held); + let restored = core + .task(Instant::now() + SESSION_ADMISSION_TIMEOUT) + .await + .unwrap(); + assert_eq!(core.permits.available_permits(), 63); + drop(restored); + assert_eq!(core.permits.available_permits(), 64); + drop(session); + core.wait_drained(); + } + #[tokio::test(start_paused = true)] + async fn task_admission_waits_are_cancelled_by_future_drop_and_engine_shutdown() { + let core = JvmEngine::new(std::ptr::null_mut()); + let held = core.permits.clone().try_acquire_many_owned(64).unwrap(); + let mut cancelled = Box::pin(core.task(Instant::now() + SESSION_ADMISSION_TIMEOUT)); + assert!(futures::poll!(&mut cancelled).is_pending()); + drop(cancelled); + drop(held); + assert!(core.counts.lock().unwrap().tasks.is_empty()); + // Cancelling the wait must neither leak a permit nor create a JNI task. + let _held = core.permits.clone().try_acquire_many_owned(64).unwrap(); + let pending = core.task(Instant::now() + SESSION_ADMISSION_TIMEOUT); + tokio::pin!(pending); + assert!(futures::poll!(&mut pending).is_pending()); + core.shutdown(); + let status = pending.await.err().unwrap(); + assert_eq!(status.code(), tonic::Code::Unavailable, "{status}"); + assert!(core.counts.lock().unwrap().tasks.is_empty()); + assert_eq!(core.counts.lock().unwrap().workers, 0); + core.wait_drained(); + } + fn blocked_session(id: &str) -> (Arc, OwnedSemaphorePermit, Arc) { + let core = JvmEngine::new(std::ptr::null_mut()); + let held = core + .session_permits + .clone() + .try_acquire_many_owned(64) + .unwrap(); + let factory = EngineFactory(core.clone()); + let session = factory + .create_session(SessionContext { + session_id: id.into(), + user_context: None, + }) + .unwrap(); + (core, held, session) + } + async fn assert_config_admission_deadlines( + operations: Vec, + ) { + let (core, _held, session) = blocked_session("concurrent-blocked-session"); + let calls = futures::future::join_all(operations.into_iter().map(|input| { + let session = &session; + async move { + let started = Instant::now(); + let result = session.config(input).await; + (result, started.elapsed()) + } + })); + tokio::pin!(calls); + // Poll every real Config call before advancing time. The scenarios + // exercise shared readers, queued writers and mixed access under + // heavyweight session capacity pressure. + assert!(futures::poll!(&mut calls).is_pending()); + assert_eq!(core.permits.available_permits(), 64); + tokio::time::advance(SESSION_ADMISSION_TIMEOUT).await; + for (result, elapsed) in calls.await { + assert!( + matches!(&result, Err(status) if status.code() == tonic::Code::ResourceExhausted), + "{:?}", + result.as_ref().err().map(|s| (s.code(), s.message())) + ); + assert!(elapsed <= SESSION_ADMISSION_TIMEOUT, "{elapsed:?}"); + } + assert_eq!(core.permits.available_permits(), 64); + core.wait_drained(); + } + fn config_get() -> proto::config_request::operation::OpType { + proto::config_request::operation::OpType::Get(proto::config_request::Get { + keys: vec!["key".into()], + }) + } + fn config_set() -> proto::config_request::operation::OpType { + proto::config_request::operation::OpType::Set(proto::config_request::Set { + pairs: vec![proto::KeyValue { + key: "key".into(), + value: Some("value".into()), + }], + silent: None, + }) + } + #[tokio::test(start_paused = true)] + async fn session_initializer_wait_obeys_deadline_and_preserves_the_initializer() { + let core = JvmEngine::new(std::ptr::null_mut()); + let session = JvmSession { + core: core.clone(), + context: SessionContext { + session_id: "slow-initializer".into(), + user_context: None, + }, + handle: AsyncMutex::new(None), + catalog: RwLock::new(()), + }; + // Model an initializer that already owns the slot and admission and is in Java. + let session_permit = core.session_permits.clone().try_acquire_owned().unwrap(); + let task_permit = core.permits.clone().try_acquire_owned().unwrap(); + let mut initializing = session.handle.lock().await; + { + let waiting = session.handle(Instant::now() + SESSION_ADMISSION_TIMEOUT); + tokio::pin!(waiting); + assert!(futures::poll!(&mut waiting).is_pending()); + tokio::time::advance(SESSION_ADMISSION_TIMEOUT).await; + let error = waiting.await.err().unwrap(); + assert_eq!(error.code(), tonic::Code::ResourceExhausted); + assert!(error.message().contains("initialization busy")); + } + assert!(session.handle.try_lock().is_err()); + assert_eq!(core.permits.available_permits(), 63); + assert_eq!(core.session_permits.available_permits(), 63); + // Timing out a waiter does not cancel or discard the original result. + let handle = empty_handle(&core, HandleKind::Session, Some(session_permit), None); + *initializing = Some(handle.clone()); + drop(initializing); + drop(task_permit); + let cached = session + .handle(Instant::now() + SESSION_ADMISSION_TIMEOUT) + .await + .unwrap(); + assert!(Arc::ptr_eq(&cached, &handle)); + drop(cached); + drop(handle); + drop(session); + core.wait_drained(); + assert_eq!(core.permits.available_permits(), 64); + assert_eq!(core.session_permits.available_permits(), 64); + } + #[tokio::test(start_paused = true)] + async fn catalog_waits_obey_the_original_operation_admission_deadline() { + let core = JvmEngine::new(std::ptr::null_mut()); + let concrete = JvmSession { + core: core.clone(), + context: SessionContext { + session_id: "busy-catalog".into(), + user_context: None, + }, + handle: AsyncMutex::new(None), + catalog: RwLock::new(()), + }; + let catalog = concrete.catalog.write().await; + let operations = futures::future::join_all( + [config_get(), config_set()] + .into_iter() + .map(|input| concrete.config(input)), + ); + tokio::pin!(operations); + assert!(futures::poll!(&mut operations).is_pending()); + tokio::time::advance(SESSION_ADMISSION_TIMEOUT).await; + for result in operations.await { + let error = result.err().unwrap(); + assert_eq!(error.code(), tonic::Code::ResourceExhausted); + assert!(error.message().contains("catalog busy")); + } + assert!(concrete.catalog.try_write().is_err()); + drop(catalog); + core.wait_drained(); + } + #[tokio::test(start_paused = true)] + async fn concurrent_first_config_calls_do_not_restart_admission_timeouts() { + assert_config_admission_deadlines(vec![config_get(), config_get(), config_get()]).await; + } + #[tokio::test(start_paused = true)] + async fn silent_config_requests_preserve_native_admission_errors() { + let input = proto::config_request::operation::OpType::Set(proto::config_request::Set { + pairs: vec![proto::KeyValue { + key: "spark.application.option".into(), + value: Some("value".into()), + }], + silent: Some(true), + }); + assert_config_admission_deadlines(vec![input]).await; + } + #[tokio::test(start_paused = true)] + async fn concurrent_first_config_writers_do_not_restart_admission_timeouts() { + assert_config_admission_deadlines(vec![config_set(), config_set(), config_set()]).await; + } + #[tokio::test(start_paused = true)] + async fn config_readers_behind_writers_keep_their_original_admission_deadlines() { + assert_config_admission_deadlines(vec![config_set(), config_get(), config_set()]).await; + } + #[tokio::test(start_paused = true)] + async fn prepare_and_analyze_behind_config_keep_their_original_admission_deadlines() { + let (core, _held, session) = blocked_session("mixed-blocked-session"); + let relation = || proto::Relation { + rel_type: Some(proto::relation::RelType::Sql(proto::Sql { + query: "SELECT 1".into(), + ..Default::default() + })), + ..Default::default() + }; + let calls = async { + futures::join!( + async { session.config(config_set()).await.map(|_| ()) }, + async { + session + .prepare(proto::plan::OpType::Root(relation())) + .await + .map(|_| ()) + }, + async { + session + .analyze(proto::analyze_plan_request::Analyze::Schema( + proto::analyze_plan_request::Schema { + plan: Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(relation())), + }), + }, + )) + .await + .map(|_| ()) + }, + ) + }; + tokio::pin!(calls); + let started = Instant::now(); + // Config Set enters first; prepare and analyze queue behind its writer. + assert!(futures::poll!(&mut calls).is_pending()); + assert_eq!(core.permits.available_permits(), 64); + tokio::time::advance(SESSION_ADMISSION_TIMEOUT).await; + let (config, prepare, analyze) = calls.await; + for result in [config, prepare, analyze] { + assert!( + matches!(&result, Err(status) if status.code() == tonic::Code::ResourceExhausted), + "{:?}", + result.as_ref().err().map(|s| (s.code(), s.message())) + ); + } + let elapsed = started.elapsed(); + assert!(elapsed <= SESSION_ADMISSION_TIMEOUT, "{elapsed:?}"); + assert_eq!(core.permits.available_permits(), 64); + core.wait_drained(); + } + #[tokio::test(start_paused = true)] + async fn invalid_inputs_preserve_errors_without_waiting_for_session_admission() { + let (core, _held, session) = blocked_session("invalid-input-session"); + let started = Instant::now(); + let status = session + .analyze(proto::analyze_plan_request::Analyze::Explain( + Default::default(), + )) + .await + .err() + .unwrap(); + assert_eq!(status.code(), tonic::Code::Unimplemented, "{status}"); + assert_eq!(status.message(), "Spark engine analysis is not implemented"); + let status = session + .analyze(proto::analyze_plan_request::Analyze::Schema( + proto::analyze_plan_request::Schema { plan: None }, + )) + .await + .err() + .unwrap(); + assert_eq!(status.code(), tonic::Code::InvalidArgument, "{status}"); + assert_eq!(status.message(), "analysis requires a root relation"); + let status = session + .prepare(proto::plan::OpType::Command(proto::Command { + command_type: Some(proto::command::CommandType::SqlCommand(Default::default())), + })) + .await + .err() + .unwrap(); + assert_eq!(status.code(), tonic::Code::InvalidArgument, "{status}"); + assert_eq!(status.message(), "missing SQL input"); + // Exercise legacy argument rejection before missing-input validation. + #[allow(deprecated)] + let sql_with_arguments = proto::SqlCommand { + pos_arguments: vec![Default::default()], + ..Default::default() + }; + let status = session + .prepare(proto::plan::OpType::Command(proto::Command { + command_type: Some(proto::command::CommandType::SqlCommand(sql_with_arguments)), + })) + .await + .err() + .unwrap(); + assert_eq!(status.code(), tonic::Code::Unimplemented, "{status}"); + assert_eq!(status.message(), "Spark engine SQL command arguments"); + let result = session + .config(proto::config_request::operation::OpType::Set( + proto::config_request::Set { + silent: Some(true), + ..Default::default() + }, + )) + .await + .unwrap(); + assert!(result.pairs.is_empty() && result.warnings.is_empty()); + assert_eq!(started.elapsed(), Duration::ZERO); + assert_eq!(core.session_permits.available_permits(), 0); + assert_eq!(core.permits.available_permits(), 64); + assert_eq!(core.counts.lock().unwrap().handles, 0); + core.wait_drained(); + } + #[tokio::test] + async fn queued_config_writer_precedes_later_catalog_readers() { + let catalog = RwLock::new(()); + let first_reader = CatalogAccess::Read.lock(&catalog).await; + let writer = CatalogAccess::config(6).lock(&catalog); + tokio::pin!(writer); + assert!(futures::poll!(&mut writer).is_pending()); + let late_reader = CatalogAccess::config(1).lock(&catalog); + tokio::pin!(late_reader); + assert!(futures::poll!(&mut late_reader).is_pending()); + drop(first_reader); + let writer_guard = writer.await; + assert!(matches!(&writer_guard, CatalogGuard::Write { .. })); + assert!(futures::poll!(&mut late_reader).is_pending()); + drop(writer_guard); + let late_reader_guard = late_reader.await; + assert!(matches!(&late_reader_guard, CatalogGuard::Read { .. })); + // A Config Get guard remains shared while another Analyze reader enters. + let overlapping_reader = CatalogAccess::Read.lock(&catalog); + tokio::pin!(overlapping_reader); + assert!(futures::poll!(&mut overlapping_reader).is_ready()); + } + #[tokio::test] + async fn shutdown_wakes_pending_session_admission_without_consuming_task_slots() { + let (core, _held, session) = blocked_session("blocked-session"); + let pending = session.config(config_get()); + tokio::pin!(pending); + assert!(futures::poll!(&mut pending).is_pending()); + assert_eq!(core.permits.available_permits(), 64); + core.shutdown(); + let result = pending.await; + assert!( + matches!(&result, Err(status) if status.code() == tonic::Code::Unavailable), + "{:?}", + result.as_ref().err().map(|s| (s.code(), s.message())) + ); + core.wait_drained(); + } + #[test] + fn typed_errors_preserve_codes_and_reject_ok_or_invalid_codes() { + assert_eq!( + decode_error("JVM_ENGINE_ERROR:12:unsupported: feature").code(), + tonic::Code::Unimplemented + ); + assert_eq!( + decode_error("JVM_ENGINE_ERROR:1:cancelled 😀").message(), + "cancelled 😀" + ); + for text in [ + "JVM_ENGINE_ERROR:0:bad", + "JVM_ENGINE_ERROR:99:bad", + "plain failure", + ] { + assert_eq!(decode_error(text).code(), tonic::Code::Internal); + } + } + #[tokio::test] + async fn shutdown_cancels_pending_tokens_before_their_first_jni_call() { + // No JNI call occurs: cancellation is observed before task publication. + let core = JvmEngine::new(std::ptr::null_mut()); + let guard = core + .task(Instant::now() + SESSION_ADMISSION_TIMEOUT) + .await + .unwrap(); + core.shutdown(); + assert!( + matches!(guard.task.initialize(), Err(status) if status.code() == tonic::Code::Cancelled) + ); + assert!(core + .task(Instant::now() + SESSION_ADMISSION_TIMEOUT) + .await + .is_err()); + drop(guard); + core.wait_drained(); + } + #[test] + fn config_codec_preserves_null_unicode_and_empty() { + let pairs = vec![ + proto::KeyValue { + key: "时区".into(), + value: Some("😀".into()), + }, + proto::KeyValue { + key: "".into(), + value: None, + }, + proto::KeyValue { + key: "empty".into(), + value: Some("".into()), + }, + ]; + assert_eq!(pairs, decode_pairs(&encode_pairs(&pairs).unwrap()).unwrap()); + } + #[tokio::test] + async fn silent_config_sets_continue_after_per_key_errors_without_disclosing_values() { + let pairs = ["first", "blocked", "last"] + .into_iter() + .map(|key| proto::KeyValue { + key: key.into(), + value: Some("sensitive-value".into()), + }) + .collect::>(); + let mut attempted = Vec::new(); + let result = silent_config_set(pairs.clone(), |pair| { + attempted.push(pair.clone()); + std::future::ready(if pair.key == "blocked" { + Ok(vec![proto::KeyValue { + key: pair.key, + value: Some("INVALID_CONF_VALUE".into()), + }]) + } else { + Ok(Vec::new()) + }) + }) + .await + .unwrap(); + assert_eq!(attempted, pairs); + assert!(result.pairs.is_empty()); + assert_eq!( + result.warnings, + ["Failed to set blocked (INVALID_CONF_VALUE)"] + ); + } + #[tokio::test] + async fn silent_config_sets_stop_on_cancellation_shutdown_and_adapter_failures() { + for code in [ + tonic::Code::InvalidArgument, + tonic::Code::NotFound, + tonic::Code::AlreadyExists, + tonic::Code::OutOfRange, + tonic::Code::Cancelled, + tonic::Code::DeadlineExceeded, + tonic::Code::ResourceExhausted, + tonic::Code::FailedPrecondition, + tonic::Code::Unavailable, + tonic::Code::Internal, + ] { + let mut attempted = 0; + let pairs = ["first", "never"] + .into_iter() + .map(|key| proto::KeyValue { + key: key.into(), + value: Some("value".into()), + }) + .collect(); + let error = silent_config_set(pairs, |_| { + attempted += 1; + std::future::ready(Err(Status::new(code, "operation failed"))) + }) + .await + .err() + .unwrap(); + assert_eq!(error.code(), code); + assert_eq!(error.message(), "operation failed"); + assert_eq!(attempted, 1); + } + } + #[test] + fn config_codec_matches_java_data_stream_format() { + let pairs = vec![proto::KeyValue { + key: "k".into(), + value: Some("v".into()), + }]; + let bytes = [0, 0, 0, 1, 0, 0, 0, 1, b'k', 0, 0, 0, 1, b'v']; + assert_eq!(encode_pairs(&pairs).unwrap(), bytes); + assert_eq!(decode_pairs(&bytes).unwrap(), pairs); + } + #[test] + fn config_codec_rejects_trailing_and_invalid_lengths() { + assert!(decode_pairs(&[0, 0, 0, 0, 1]).is_err()); + assert!(decode_pairs(&[0, 0, 0, 1, 0, 0, 0, 0, 255, 255, 255, 254]).is_err()); + assert!(decode_pairs(&[]).is_err()); + assert!(decode_pairs(&[255, 255, 255, 255]).is_err()); + assert!(decode_pairs(&[0, 0, 0, 1, 255, 255, 255, 255, 0, 0, 0, 0]).is_err()); + assert!(decode_pairs(&[0, 0, 0, 1, 0, 0, 0, 2, b'k', 0, 0, 0, 0]).is_err()); + } +} diff --git a/crates/spark-connect-server/jni/src/lib.rs b/crates/spark-connect-server/jni/src/lib.rs new file mode 100644 index 00000000..9b70a3e2 --- /dev/null +++ b/crates/spark-connect-server/jni/src/lib.rs @@ -0,0 +1,638 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! JVM-loaded JNI host. Java objects are retained by the C shim, never by JNIEnv pointers. + +mod engine; + +use std::{ + cell::{Cell, RefCell}, + collections::HashMap, + ffi::c_void, + net::SocketAddr, + ptr, + sync::{ + atomic::{AtomicBool, AtomicI64, Ordering}, + mpsc as std_mpsc, Arc, Mutex, OnceLock, + }, + time::Duration, +}; + +use tokio::sync::oneshot; +use tonic::Code; + +const MAX_BYTES: usize = 64 * 1024 * 1024; + +#[repr(C)] +pub struct BridgeBuffer { + data: *mut u8, + len: usize, +} +impl Default for BridgeBuffer { + fn default() -> Self { + Self { + data: ptr::null_mut(), + len: 0, + } + } +} + +extern "C" { + fn spark_connect_buffer_free(buffer: *mut BridgeBuffer); +} + +fn take_buffer(mut buffer: BridgeBuffer) -> Vec { + let result = if buffer.data.is_null() || buffer.len == 0 { + Vec::new() + } else { + // C returns malloc-owned buffers valid until spark_connect_buffer_free. + unsafe { std::slice::from_raw_parts(buffer.data, buffer.len).to_vec() } + }; + unsafe { + spark_connect_buffer_free(&mut buffer); + } + result +} + +struct ServerEntry { + address: String, + shutdown: Mutex>>, + done: Mutex>>, + thread: Mutex>>, + stopping: Mutex<()>, + running: Arc, + drained: AtomicBool, + backend: Arc, +} +fn servers() -> &'static Mutex>> { + static SERVERS: OnceLock>>> = OnceLock::new(); + SERVERS.get_or_init(Mutex::default) +} + +thread_local! { + // Attach lazily on the first engine action. The runtime's stop hook runs on + // that same blocking worker; each individual call still restores its context. + static ENGINE_ATTACHMENT: RefCell> = const { RefCell::new(None) }; + static ENGINE_ATTACH_ERROR_REPORTED: Cell = const { Cell::new(false) }; +} +fn attach_engine_worker(engine: &Arc) { + ENGINE_ATTACHMENT.with(|slot| { + if let Some(attachment) = slot.borrow().as_ref() { + debug_assert!(attachment.belongs_to(engine)); + return; + } + match engine.attach_worker() { + Ok(attachment) => *slot.borrow_mut() = Some(attachment), + Err(error) if error.code() == Code::Unavailable => { + // Shutdown can win after blocking work is reserved. Its task + // cancellation and normal per-call checks handle that race. + } + Err(error) => { + // The existing per-call JNI scope remains a safe fallback. + // Retry transient failures, but report at most once per worker. + ENGINE_ATTACH_ERROR_REPORTED.with(|reported| { + if !reported.replace(true) { + engine::log_diagnostic(format_args!( + "JVM worker attachment failed; using per-call attachment: {error}" + )); + } + }); + } + } + }); +} +fn start_host(address: &[u8], backend: Arc) -> Result { + static NEXT: AtomicI64 = AtomicI64::new(1); + #[allow(deprecated)] // Renamed only in Rust 1.95; preserve the declared MSRV. + let id = NEXT + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |id| id.checked_add(1)) + .map_err(|_| "JNI server IDs exhausted")?; + let address: SocketAddr = std::str::from_utf8(address) + .map_err(|_| "listen address must be UTF-8")? + .parse() + .map_err(|_| "listen address must be an IP:port")?; + let listener = std::net::TcpListener::bind(address).map_err(|e| e.to_string())?; + listener.set_nonblocking(true).map_err(|e| e.to_string())?; + let address = listener + .local_addr() + .map_err(|e| e.to_string())? + .to_string(); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .max_blocking_threads(256) + .on_thread_stop(|| { + let attachment = ENGINE_ATTACHMENT.with(|slot| slot.borrow_mut().take()); + drop(attachment); + }) + .enable_all() + .build() + .map_err(|e| e.to_string())?; + let (shutdown, stopped) = oneshot::channel(); + let (done, completed) = std_mpsc::channel(); + let running = Arc::new(AtomicBool::new(true)); + let worker_running = running.clone(); + let worker_backend = backend.clone(); + let thread = std::thread::Builder::new() + .name("spark-connect-jni-server".into()) + .spawn(move || { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + runtime.block_on(async { + let listener = + tokio::net::TcpListener::from_std(listener).map_err(|e| e.to_string())?; + spark_connect_server::serve_with_engine( + listener, + Arc::new(engine::EngineFactory(worker_backend.clone())), + async { + let _ = stopped.await; + }, + ) + .await + .map_err(|e| e.to_string()) + }) + })) + .unwrap_or_else(|_| Err("Rust listener panicked".into())); + // Listener health must become visible even when retained JNI work + // prevents the subsequent cleanup drain from finishing. + worker_running.store(false, Ordering::SeqCst); + worker_backend.shutdown(); + // Never destroy a JVM global reference or runtime while attached work remains. + worker_backend.wait_drained(); + runtime.shutdown_timeout(Duration::from_secs(1)); + let result = result.and_then(|()| worker_backend.verify_workers_detached()); + let _ = done.send(result); + }) + .map_err(|e| e.to_string())?; + servers() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + id, + Arc::new(ServerEntry { + address, + shutdown: Mutex::new(Some(shutdown)), + done: Mutex::new(completed), + thread: Mutex::new(Some(thread)), + stopping: Mutex::new(()), + running, + drained: AtomicBool::new(false), + backend, + }), + ); + Ok(id) +} +fn find_server(id: i64) -> Result, String> { + servers() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&id) + .cloned() + .ok_or_else(|| "invalid or closed native server handle".into()) +} +fn stop_server(id: i64) -> Result<(), String> { + let server = find_server(id)?; + let _stopping = server + .stopping + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + server.backend.retry_failed_cleanup(); + if server + .thread + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .is_none() + { + if !server.drained.load(Ordering::SeqCst) { + return Err("JNI server drain state is unknown after a listener-thread failure; references retained until JVM exit".into()); + } + server.backend.shutdown(); + server.backend.verify_workers_detached()?; + server.backend.release_factory()?; + servers() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&id); + return Ok(()); + } + server.backend.shutdown(); + if let Some(shutdown) = server + .shutdown + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + let _ = shutdown.send(()); + } + let completed = server + .done + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let result = loop { + if let Some(error) = server.backend.cleanup_failure() { + // The listener thread still owns the runtime and waits for the + // retained handles/permits. A later explicit close can retry them. + return Err(error); + } + let remaining = deadline.saturating_duration_since(std::time::Instant::now()); + if remaining.is_zero() { + return Err("JNI stop is incomplete; active JVM calls retained safely, retry close after they finish".into()); + } + match completed.recv_timeout(remaining.min(Duration::from_millis(20))) { + Ok(result) => { + // Only this completion certifies drain and runtime stop. + server.drained.store(true, Ordering::SeqCst); + break result; + } + Err(std_mpsc::RecvTimeoutError::Timeout) => {} + Err(std_mpsc::RecvTimeoutError::Disconnected) => { + break Err("JNI server completion channel closed without confirming teardown; drain state unknown".into()); + } + } + }; + drop(completed); + let mut result = result; + if let Some(thread) = server + .thread + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + if thread.join().is_err() { + result = Err("JNI server thread panicked".into()); + } + } + // NativeEngineServer retains its handle on any error. Keep even a fully stopped + // entry until the owner retries, rather than returning a stale handle. + if result.is_ok() { + // Join and confirmed drain precede the synchronous Java factory close. + // Retaining server here prevents a concurrent final core Drop. + result = server.backend.release_factory(); + } + if result.is_ok() { + servers() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&id); + } + result +} + +unsafe fn write_buffer(out: *mut BridgeBuffer, bytes: &[u8]) -> Result<(), String> { + write_buffer_with(out, bytes, |length| libc::malloc(length).cast()) +} + +unsafe fn write_buffer_with( + out: *mut BridgeBuffer, + bytes: &[u8], + allocate: impl FnOnce(usize) -> *mut u8, +) -> Result<(), String> { + if out.is_null() { + return Err("missing native bridge output buffer".into()); + } + *out = BridgeBuffer::default(); + if bytes.is_empty() { + return Ok(()); + } + let data = allocate(bytes.len()); + if data.is_null() { + return Err("cannot allocate native bridge output buffer".into()); + } + ptr::copy_nonoverlapping(bytes.as_ptr(), data, bytes.len()); + *out = BridgeBuffer { + data, + len: bytes.len(), + }; + Ok(()) +} +fn ffi_result( + error: *mut BridgeBuffer, + action: impl FnOnce() -> Result, +) -> Option { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(action)) { + Ok(Ok(result)) => Some(result), + Ok(Err(message)) => { + unsafe { + let _ = write_buffer(error, message.as_bytes()); + } + None + } + Err(_) => { + unsafe { + let _ = write_buffer(error, b"panic contained at JNI boundary"); + } + None + } + } +} + +/// # Safety +/// Output pointers must refer to writable BridgeBuffer objects owned by bridge.c. +#[no_mangle] +pub unsafe extern "C" fn spark_connect_address( + id: i64, + out: *mut BridgeBuffer, + error: *mut BridgeBuffer, +) -> i32 { + ffi_result(error, || { + let server = find_server(id)?; + write_buffer(out, server.address.as_bytes()) + }) + .map_or(-1, |_| 0) +} +/// # Safety +/// Error must be null or a writable BridgeBuffer owned by bridge.c. +#[no_mangle] +pub unsafe extern "C" fn spark_connect_stop(id: i64, error: *mut BridgeBuffer) -> i32 { + ffi_result(error, || stop_server(id)).map_or(-1, |_| 0) +} + +/// # Safety +/// C supplies a readable address and transfers an owned typed Java engine handle. +#[no_mangle] +pub unsafe extern "C" fn spark_connect_engine_start( + address: *const u8, + length: usize, + factory: *mut c_void, + error: *mut BridgeBuffer, +) -> i64 { + ffi_result(error, || { + if factory.is_null() { + return Err("null Spark engine".into()); + } + let backend = engine::JvmEngine::new(factory); + if address.is_null() && length != 0 { + return Err("null listen address".into()); + } + let address = if length == 0 { + &[] + } else { + std::slice::from_raw_parts(address, length) + }; + start_host(address, backend) + }) + .unwrap_or(0) +} + +extern "C" { + fn spark_connect_engine_native_start( + env: *mut c_void, + class: *mut c_void, + address: *mut c_void, + factory: *mut c_void, + ) -> i64; + fn spark_connect_engine_native_address( + env: *mut c_void, + class: *mut c_void, + id: i64, + ) -> *mut c_void; + fn spark_connect_engine_native_stop(env: *mut c_void, class: *mut c_void, id: i64); +} +/// # Safety +/// The JVM supplies valid JNI arguments for NativeEngineServer.nativeStart. +#[no_mangle] +pub unsafe extern "system" fn Java_org_apache_spark_sql_connect_nativebridge_engine_NativeEngineServer_nativeStart( + env: *mut c_void, + class: *mut c_void, + address: *mut c_void, + factory: *mut c_void, +) -> i64 { + spark_connect_engine_native_start(env, class, address, factory) +} +/// # Safety +/// The JVM supplies valid JNI arguments for NativeEngineServer.nativeAddress. +#[no_mangle] +pub unsafe extern "system" fn Java_org_apache_spark_sql_connect_nativebridge_engine_NativeEngineServer_nativeAddress( + env: *mut c_void, + class: *mut c_void, + id: i64, +) -> *mut c_void { + spark_connect_engine_native_address(env, class, id) +} +/// # Safety +/// The JVM supplies valid JNI arguments for NativeEngineServer.nativeStop. +#[no_mangle] +pub unsafe extern "system" fn Java_org_apache_spark_sql_connect_nativebridge_engine_NativeEngineServer_nativeStop( + env: *mut c_void, + class: *mut c_void, + id: i64, +) { + spark_connect_engine_native_stop(env, class, id) +} + +fn listener_running(id: i64) -> bool { + find_server(id).is_ok_and(|server| server.running.load(Ordering::SeqCst)) +} + +fn server_drained(id: i64) -> bool { + find_server(id).is_ok_and(|server| server.drained.load(Ordering::SeqCst)) +} + +#[no_mangle] +pub extern "system" fn Java_org_apache_spark_sql_connect_nativebridge_engine_NativeEngineServer_nativeIsRunning( + _env: *mut c_void, + _class: *mut c_void, + id: i64, +) -> u8 { + u8::from(std::panic::catch_unwind(|| listener_running(id)).unwrap_or(false)) +} + +#[no_mangle] +pub extern "system" fn Java_org_apache_spark_sql_connect_nativebridge_engine_NativeEngineServer_nativeIsDrained( + _env: *mut c_void, + _class: *mut c_void, + id: i64, +) -> u8 { + u8::from(std::panic::catch_unwind(|| server_drained(id)).unwrap_or(false)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn native_output_allocation_failure_is_an_error_instead_of_empty_success() { + let mut output = BridgeBuffer::default(); + let mut error = BridgeBuffer::default(); + let result = ffi_result(&mut error, || unsafe { + write_buffer_with(&mut output, b"127.0.0.1:15002", |_| ptr::null_mut()) + }); + assert!(result.is_none()); + assert!(output.data.is_null() && output.len == 0); + assert_eq!( + take_buffer(error), + b"cannot allocate native bridge output buffer" + ); + unsafe { + write_buffer(&mut output, b"127.0.0.1:15002").unwrap(); + } + assert_eq!(take_buffer(output), b"127.0.0.1:15002"); + let mut empty = BridgeBuffer::default(); + unsafe { + write_buffer_with(&mut empty, b"", |_| panic!("empty output allocated")).unwrap(); + } + assert!(empty.data.is_null() && empty.len == 0); + } + + #[test] + fn stopped_listener_reports_unhealthy_while_retained_cleanup_blocks_drain() { + let backend = engine::JvmEngine::new(ptr::null_mut()); + backend.fail_handle_cleanup_for_test(); + let id = start_host(b"127.0.0.1:0", backend).unwrap(); + let server = find_server(id).unwrap(); + let result = std::panic::catch_unwind(|| { + assert!(listener_running(id)); + server + .shutdown + .lock() + .unwrap() + .take() + .unwrap() + .send(()) + .unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while listener_running(id) && std::time::Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(2)); + } + // Exercise the real listener thread: its health must change before + // wait_drained can finish, even though the retained handle remains. + assert!(!listener_running(id)); + assert!(!server_drained(id)); + assert!(matches!( + server.done.lock().unwrap().try_recv(), + Err(std_mpsc::TryRecvError::Empty) + )); + assert!(server.thread.lock().unwrap().is_some()); + }); + // Retry the synthetic null-handle cleanup even if an assertion failed, + // so this test never strands its listener thread in wait_drained. + let cleanup = stop_server(id); + result.unwrap(); + cleanup.unwrap(); + assert!(find_server(id).is_err()); + } + + #[test] + fn completed_server_errors_require_a_confirmed_drain_before_retry_succeeds() { + for (id, disconnect) in [(-1, false), (-2, true)] { + let backend = engine::JvmEngine::new(ptr::null_mut()); + let (done, completed) = std_mpsc::channel(); + let thread = std::thread::spawn(move || { + if !disconnect { + done.send(Err("listener failed".into())).unwrap(); + } + }); + servers().lock().unwrap().insert( + id, + Arc::new(ServerEntry { + address: "127.0.0.1:0".into(), + shutdown: Mutex::new(None), + done: Mutex::new(completed), + thread: Mutex::new(Some(thread)), + stopping: Mutex::new(()), + running: Arc::new(AtomicBool::new(false)), + drained: AtomicBool::new(false), + backend, + }), + ); + assert!(!listener_running(id)); + assert!(!server_drained(id)); + assert!(stop_server(id).is_err()); + assert!(find_server(id).unwrap().thread.lock().unwrap().is_none()); + assert_eq!(server_drained(id), !disconnect); + if disconnect { + let retry = stop_server(id).unwrap_err(); + assert!(retry.contains("drain state is unknown"), "{retry}"); + assert!(find_server(id).is_ok()); + // This synthetic entry owns no JNI objects; remove the test + // fixture explicitly. Real unknown-drain entries stay retained. + servers().lock().unwrap().remove(&id); + } else { + stop_server(id).unwrap(); + assert!(find_server(id).is_err()); + } + } + } + + #[test] + fn completed_engine_server_retry_requires_detached_workers() { + let core = engine::JvmEngine::new(ptr::null_mut()); + core.set_worker_attachment_count_for_test(1); + let (_done, completed) = std_mpsc::channel(); + let id = -303; + servers().lock().unwrap().insert( + id, + Arc::new(ServerEntry { + address: "127.0.0.1:0".into(), + shutdown: Mutex::new(None), + done: Mutex::new(completed), + thread: Mutex::new(None), + stopping: Mutex::new(()), + running: Arc::new(AtomicBool::new(false)), + drained: AtomicBool::new(true), + backend: core.clone(), + }), + ); + let error = stop_server(id).unwrap_err(); + assert!(error.contains("1 JNI worker attachments remain"), "{error}"); + assert!(find_server(id).is_ok()); + // Simulate the previously unfinished worker successfully detaching. + core.set_worker_attachment_count_for_test(0); + stop_server(id).unwrap(); + assert!(find_server(id).is_err()); + } + + #[test] + fn failed_cleanup_preserves_the_listener_and_drain_state_until_explicit_retry() { + let core = engine::JvmEngine::new(ptr::null_mut()); + let (shutdown, stopping) = oneshot::channel(); + let (done, completed) = std_mpsc::channel(); + let listener_core = core.clone(); + let thread = std::thread::spawn(move || { + stopping.blocking_recv().unwrap(); + // Failure arrives after stop_server's retry snapshot. The listener + // must keep waiting with its runtime alive until another close. + listener_core.fail_handle_cleanup_for_test(); + listener_core.wait_drained(); + done.send(Ok(())).unwrap(); + }); + let id = -304; + servers().lock().unwrap().insert( + id, + Arc::new(ServerEntry { + address: "127.0.0.1:0".into(), + shutdown: Mutex::new(Some(shutdown)), + done: Mutex::new(completed), + thread: Mutex::new(Some(thread)), + stopping: Mutex::new(()), + running: Arc::new(AtomicBool::new(false)), + drained: AtomicBool::new(false), + backend: core, + }), + ); + let error = stop_server(id).unwrap_err(); + assert!(error.starts_with("JNI cleanup is incomplete; retained cleanup failed:")); + let retained = find_server(id).unwrap(); + assert!(!listener_running(id)); + assert!(!server_drained(id)); + assert!(!retained.drained.load(Ordering::SeqCst)); + assert!(retained.thread.lock().unwrap().is_some()); + stop_server(id).unwrap(); + assert!(retained.drained.load(Ordering::SeqCst)); + assert!(retained.thread.lock().unwrap().is_none()); + assert!(find_server(id).is_err()); + } +} diff --git a/crates/spark-connect-server/jni/test-c-bridge.sh b/crates/spark-connect-server/jni/test-c-bridge.sh new file mode 100755 index 00000000..eb5238d9 --- /dev/null +++ b/crates/spark-connect-server/jni/test-c-bridge.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +case "${1:-}" in + '') sanitize=0 ;; + --sanitize) sanitize=1 ;; + *) echo "Usage: $0 [--sanitize]" >&2; exit 2 ;; +esac +if [ "$#" -gt 1 ]; then + echo "Usage: $0 [--sanitize]" >&2 + exit 2 +fi + +: "${JAVA_HOME:?Set JAVA_HOME to a JDK containing JNI headers}" +case "$(uname -s)" in + Darwin) jni_platform=darwin ;; + Linux) jni_platform=linux ;; + *) echo 'The C bridge test requires macOS or Linux with pthreads' >&2; exit 2 ;; +esac + +task_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +output_dir="$task_root/.build/c-bridge" +mkdir -p "$output_dir" +set -- -std=c11 -Wall -Wextra -Werror -pthread +test_binary="$output_dir/bridge" +if [ "$sanitize" -eq 1 ]; then + set -- "$@" -fsanitize=address,undefined -fno-omit-frame-pointer -g + test_binary="$output_dir/bridge-sanitized" +fi +"${CC:-cc}" "$@" \ + -I"$JAVA_HOME/include" -I"$JAVA_HOME/include/$jni_platform" \ + "$task_root/jni/tests/c_bridge.c" -o "$test_binary" +"$test_binary" diff --git a/crates/spark-connect-server/jni/tests/c_bridge.c b/crates/spark-connect-server/jni/tests/c_bridge.c new file mode 100644 index 00000000..e03735d5 --- /dev/null +++ b/crates/spark-connect-server/jni/tests/c_bridge.c @@ -0,0 +1,796 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* Exercise native ownership and pre-JNI validation without starting a JVM. */ +#undef NDEBUG +#include "../native/bridge.c" +#include +#include +#include + +/* The C bridge links to these Rust entry points; this test does not invoke them. */ +int64_t spark_connect_engine_start(const uint8_t *p, size_t n, void *b, BridgeBuffer *e) { + (void) p; (void) n; (void) b; (void) e; return 0; +} +int spark_connect_address(int64_t id, BridgeBuffer *o, BridgeBuffer *e) { + (void) id; (void) o; (void) e; return -1; +} +int spark_connect_stop(int64_t id, BridgeBuffer *e) { (void) id; (void) e; return -1; } + +static void expect_analysis_failure(void *session, void *task, int provide_output, + const char *expected) { + uint8_t untouched = 0x5a; + BridgeBuffer out = {.data = &untouched, .len = 1}; + BridgeBuffer error = {0}; + assert(spark_connect_engine_analyze(session, 0, NULL, 0, task, + provide_output ? &out : NULL, &error) == -1); + if (provide_output) assert(out.data == NULL && out.len == 0); + assert(untouched == 0x5a); + assert(error.data != NULL && error.len == strlen(expected)); + assert(memcmp(error.data, expected, error.len) == 0); + spark_connect_buffer_free(&error); + assert(error.data == NULL && error.len == 0); + spark_connect_buffer_free(&error); + assert(error.data == NULL && error.len == 0); +} + +static void analysis_preconditions(void) { + /* No JavaVM exists: every rejection must occur before entering a JNI scope. */ + EngineApi api = {0}; + EngineApi other_api = {0}; + EngineHandle session = {.api = &api, .kind = ENGINE_SESSION}; + EngineHandle query = {.api = &api, .kind = ENGINE_QUERY}; + EngineHandle task = {.api = &api, .kind = ENGINE_TASK}; + EngineHandle other_task = {.api = &other_api, .kind = ENGINE_TASK}; + const char *invalid = "JVM_ENGINE_ERROR:13:invalid Java engine handle"; + expect_analysis_failure(NULL, &task, 1, invalid); + expect_analysis_failure(&query, &task, 1, invalid); + expect_analysis_failure(&session, NULL, 1, invalid); + expect_analysis_failure(&session, &session, 1, invalid); + expect_analysis_failure(&session, &other_task, 1, + "JVM_ENGINE_ERROR:13:Java engine task belongs to another engine"); + expect_analysis_failure(&session, &task, 0, + "JVM_ENGINE_ERROR:13:missing Java engine analysis buffer"); + puts("C_ANALYZE_PRECONDITIONS_OK: invalid handles and cross-engine tasks rejected before JNI; error buffers released"); +} + +/* A small JNI invocation table verifies native-worker ownership without a JVM. */ +static int mock_attached; +static int mock_attaches; +static int mock_named_attaches; +static int mock_detaches; +static int mock_attach_result; +static int mock_detach_result; +static int mock_get_env_result; +static int mock_push_result; +static int mock_frame_depth; +static int mock_frame_capacities[256]; +static int mock_frame_references[256]; +static int mock_frame_exception_references[256]; +static int mock_max_frame_depth; +static int mock_max_frame_references; +static int mock_max_exception_references; +static int mock_fail_frame_depth = -1; +static int mock_exception_pending; +static int mock_pushes; +static int mock_pops; +static jobject mock_loader; +static JNIEnv mock_env; +static char mock_worker_name[64]; +static uintmax_t mock_worker_index; +static const char mock_exception_message[] = "injected child access failure"; +static jchar mock_exception_chars[sizeof(mock_exception_message)]; +static int mock_close_calls; +static int mock_close_failure; +static int mock_globals_deleted; +static int mock_loader_deletions; +static int mock_new_global_failure; +static int mock_next_failure; +static jobject mock_next_batch; +static jbyteArray mock_ipc_data; +static jlong mock_row_count; +static int mock_copy_failure; +static int mock_copy_calls; +static jbyte *mock_copy_destination; +static int mock_cancel_calls; +static int mock_cancel_failure; +static int mock_message_allocation_calls; +static int mock_throw_new_calls; +static int mock_throw_new_failure; + +#define MOCK_THROWABLE ((jthrowable) (uintptr_t) 20) +#define MOCK_THROWABLE_CLASS ((jclass) (uintptr_t) 21) +#define MOCK_DESCRIPTION ((jstring) (uintptr_t) 22) +#define MOCK_DESCRIBE ((jmethodID) (uintptr_t) 14) +#define MOCK_NEXT ((jmethodID) (uintptr_t) 15) +#define MOCK_DATA ((jmethodID) (uintptr_t) 16) +#define MOCK_CLOSE ((jmethodID) (uintptr_t) 17) +#define MOCK_CANCEL ((jmethodID) (uintptr_t) 18) + +typedef struct { + jsize len; + uint8_t *data; +} MockBytes; + +static jobject mock_new_local(jobject object) { + if (object == NULL) return NULL; + assert(mock_frame_depth > 0); + int *references = &mock_frame_references[mock_frame_depth - 1]; + (*references)++; + assert(*references <= mock_frame_capacities[mock_frame_depth - 1]); + if (*references > mock_max_frame_references) mock_max_frame_references = *references; + if (object == MOCK_THROWABLE || object == MOCK_THROWABLE_CLASS + || object == MOCK_DESCRIPTION) { + int *exceptions = &mock_frame_exception_references[mock_frame_depth - 1]; + (*exceptions)++; + assert(*exceptions <= 3); + if (*exceptions > mock_max_exception_references) + mock_max_exception_references = *exceptions; + } + return object; +} + +static jint JNICALL mock_get_env(JavaVM *vm, void **out, jint version) { + (void) vm; + assert(version == JNI_VERSION_1_8); + if (mock_get_env_result != JNI_OK) return mock_get_env_result; + if (!mock_attached) return JNI_EDETACHED; + *out = &mock_env; + return JNI_OK; +} + +static jint JNICALL mock_attach(JavaVM *vm, void **out, void *args) { + (void) vm; + if (args != NULL) { + JavaVMAttachArgs *attachment = args; + assert(attachment->version == JNI_VERSION_1_8); + static const char prefix[] = "spark-connect-engine-worker-"; + assert(strncmp(attachment->name, prefix, sizeof(prefix) - 1) == 0); + char *end; + uintmax_t index = strtoumax(attachment->name + sizeof(prefix) - 1, &end, 10); + assert(*end == '\0' && index > mock_worker_index); + mock_worker_index = index; + assert(strlen(attachment->name) < sizeof(mock_worker_name)); + strcpy(mock_worker_name, attachment->name); + assert(attachment->group == NULL); + } + if (mock_attach_result != JNI_OK) return mock_attach_result; + mock_attached = 1; + mock_attaches++; + if (args != NULL) mock_named_attaches++; + *out = &mock_env; + return JNI_OK; +} + +static jint JNICALL mock_detach(JavaVM *vm) { + (void) vm; + if (mock_detach_result != JNI_OK) return mock_detach_result; + assert(mock_attached && mock_frame_depth == 0); + mock_attached = 0; + mock_detaches++; + return JNI_OK; +} + +static jboolean JNICALL mock_exception_check(JNIEnv *env) { + assert(env == &mock_env); + return mock_exception_pending ? JNI_TRUE : JNI_FALSE; +} + +static jthrowable JNICALL mock_exception_occurred(JNIEnv *env) { + assert(env == &mock_env && mock_exception_pending); + return mock_new_local(MOCK_THROWABLE); +} + +static void JNICALL mock_exception_clear(JNIEnv *env) { + assert(env == &mock_env && mock_exception_pending); + mock_exception_pending = 0; +} + +static jclass JNICALL mock_object_class(JNIEnv *env, jobject object) { + assert(env == &mock_env && !mock_exception_pending && object == MOCK_THROWABLE); + return mock_new_local(MOCK_THROWABLE_CLASS); +} + +static jmethodID JNICALL mock_method_id(JNIEnv *env, jclass cls, const char *name, + const char *signature) { + assert(env == &mock_env && !mock_exception_pending && cls == MOCK_THROWABLE_CLASS); + assert(strcmp(name, "toString") == 0); + assert(strcmp(signature, "()Ljava/lang/String;") == 0); + return MOCK_DESCRIBE; +} + +static jsize JNICALL mock_string_length(JNIEnv *env, jstring string) { + assert(env == &mock_env && !mock_exception_pending && string == MOCK_DESCRIPTION); + return sizeof(mock_exception_message) - 1; +} + +static const jchar *JNICALL mock_string_chars(JNIEnv *env, jstring string, + jboolean *copied) { + assert(env == &mock_env && !mock_exception_pending && string == MOCK_DESCRIPTION); + for (size_t i = 0; i < sizeof(mock_exception_message); i++) + mock_exception_chars[i] = (unsigned char) mock_exception_message[i]; + if (copied != NULL) *copied = JNI_FALSE; + return mock_exception_chars; +} + +static void JNICALL mock_release_string_chars(JNIEnv *env, jstring string, + const jchar *chars) { + assert(env == &mock_env && string == MOCK_DESCRIPTION && chars == mock_exception_chars); +} + +static jint JNICALL mock_push_frame(JNIEnv *env, jint capacity) { + assert(env == &mock_env && mock_attached && capacity > 0); + mock_pushes++; + if (mock_push_result != JNI_OK) return mock_push_result; + if (mock_frame_depth == mock_fail_frame_depth) return JNI_ERR; + assert(mock_frame_depth < 256); + mock_frame_capacities[mock_frame_depth] = capacity; + mock_frame_references[mock_frame_depth] = 0; + mock_frame_exception_references[mock_frame_depth] = 0; + mock_frame_depth++; + if (mock_frame_depth > mock_max_frame_depth) mock_max_frame_depth = mock_frame_depth; + return JNI_OK; +} + +static jobject JNICALL mock_pop_frame(JNIEnv *env, jobject result) { + assert(env == &mock_env && mock_frame_depth > 0 && result == NULL); + assert(mock_frame_exception_references[mock_frame_depth - 1] == 0); + mock_frame_depth--; + mock_pops++; + return NULL; +} + +static jobject JNICALL mock_current_thread(JNIEnv *env, jclass cls, jmethodID method, ...) { + (void) cls; (void) method; + assert(env == &mock_env); + return mock_new_local((jobject) (uintptr_t) 1); +} + +static jobject JNICALL mock_call_object(JNIEnv *env, jobject object, jmethodID method, ...) { + assert(env == &mock_env); + if (method == MOCK_DESCRIBE) { + assert(!mock_exception_pending && object == MOCK_THROWABLE); + return mock_new_local(MOCK_DESCRIPTION); + } + if (method == MOCK_NEXT) { + if (mock_next_failure) mock_exception_pending = 1; + return mock_new_local(mock_next_batch); + } + if (method == MOCK_DATA) return mock_new_local((jobject) mock_ipc_data); + return mock_new_local(mock_loader); +} + +static void JNICALL mock_set_loader(JNIEnv *env, jobject object, jmethodID method, ...) { + (void) object; + assert(env == &mock_env); + if (method == MOCK_CLOSE) { + assert(!mock_exception_pending); + mock_close_calls++; + if (mock_close_failure) mock_exception_pending = 1; + return; + } + if (method == MOCK_CANCEL) { + assert(!mock_exception_pending); + mock_cancel_calls++; + if (mock_cancel_failure) mock_exception_pending = 1; + return; + } + va_list args; + va_start(args, method); + mock_loader = va_arg(args, jobject); + va_end(args); +} + +static jobject JNICALL mock_new_global(JNIEnv *env, jobject object) { + assert(env == &mock_env && object != NULL); + return mock_new_global_failure ? NULL : object; +} + +static void JNICALL mock_delete_global(JNIEnv *env, jobject object) { + assert(env == &mock_env && object != NULL && mock_attached && mock_frame_depth > 0); + assert(!mock_exception_pending); + if (object == (jobject) (uintptr_t) 2) { + /* Final teardown must restore the caller's loader before deleting ours. */ + assert(mock_loader == (jobject) (uintptr_t) 3); + mock_loader_deletions++; + } + mock_globals_deleted++; +} + +static jboolean JNICALL mock_same_object(JNIEnv *env, jobject left, jobject right) { + assert(env == &mock_env); + return left == right; +} + +static void JNICALL mock_delete_local(JNIEnv *env, jobject object) { + assert(env == &mock_env && object != NULL && mock_frame_depth > 0); + assert(mock_frame_references[mock_frame_depth - 1] > 0); + mock_frame_references[mock_frame_depth - 1]--; + if (object == MOCK_THROWABLE || object == MOCK_THROWABLE_CLASS + || object == MOCK_DESCRIPTION) { + assert(mock_frame_exception_references[mock_frame_depth - 1] > 0); + mock_frame_exception_references[mock_frame_depth - 1]--; + } +} + +static jlong JNICALL mock_batch_row_count(JNIEnv *env, jobject object, jmethodID method, ...) { + (void) method; + assert(env == &mock_env && object == mock_next_batch); + return mock_row_count; +} + +static jsize JNICALL mock_array_length(JNIEnv *env, jarray array) { + assert(env == &mock_env && array != NULL); + return ((MockBytes *) array)->len; +} + +static void JNICALL mock_byte_array_region(JNIEnv *env, jbyteArray array, + jsize start, jsize length, jbyte *out) { + assert(env == &mock_env && array != NULL && start == 0); + mock_copy_calls++; + mock_copy_destination = out; + if (mock_copy_failure) { + mock_exception_pending = 1; + return; + } + MockBytes *bytes = (MockBytes *) array; + assert(bytes->len == length); + memcpy(out, bytes->data, (size_t) length); +} + +static void attachment_error(BridgeBuffer *error, const char *message) { + assert(error->data != NULL); + assert(error->len == strlen(message)); + assert(memcmp(error->data, message, error->len) == 0); + spark_connect_buffer_free(error); +} + +static void *detach_on_wrong_thread(void *attachment) { + BridgeBuffer error = {0}; + assert(spark_connect_engine_thread_detach(attachment, &error) == -1); + attachment_error(&error, + "JVM_ENGINE_ERROR:13:Java worker attachment belongs to another thread"); + return NULL; +} + +static jbyteArray JNICALL mock_fail_message_allocation(JNIEnv *env, jsize length) { + (void) length; + assert(env == &mock_env && !mock_exception_pending); + mock_message_allocation_calls++; + mock_exception_pending = 1; + return NULL; +} + +static jclass JNICALL mock_find_exception_class(JNIEnv *env, const char *name) { + assert(env == &mock_env && !mock_exception_pending); + assert(strcmp(name, "java/lang/IllegalStateException") == 0); + return mock_new_local(MOCK_THROWABLE_CLASS); +} + +static jint JNICALL mock_throw_new(JNIEnv *env, jclass type, const char *message) { + assert(env == &mock_env && !mock_exception_pending && type == MOCK_THROWABLE_CLASS); + assert(strcmp(message, "native Spark Connect operation failed") == 0); + mock_throw_new_calls++; + mock_exception_pending = 1; + return mock_throw_new_failure ? JNI_ERR : JNI_OK; +} + +static JavaVM *mock_vm(void) { + static const struct JNIInvokeInterface_ invocation = { + .GetEnv = mock_get_env, + .AttachCurrentThreadAsDaemon = mock_attach, + .DetachCurrentThread = mock_detach, + }; + static const struct JNINativeInterface_ native = { + .ExceptionCheck = mock_exception_check, + .ExceptionOccurred = mock_exception_occurred, + .ExceptionClear = mock_exception_clear, + .GetObjectClass = mock_object_class, + .GetMethodID = mock_method_id, + .GetStringLength = mock_string_length, + .GetStringChars = mock_string_chars, + .ReleaseStringChars = mock_release_string_chars, + .PushLocalFrame = mock_push_frame, + .PopLocalFrame = mock_pop_frame, + .CallStaticObjectMethod = mock_current_thread, + .CallObjectMethod = mock_call_object, + .CallVoidMethod = mock_set_loader, + .IsSameObject = mock_same_object, + .DeleteLocalRef = mock_delete_local, + .NewGlobalRef = mock_new_global, + .DeleteGlobalRef = mock_delete_global, + .CallLongMethod = mock_batch_row_count, + .GetArrayLength = mock_array_length, + .GetByteArrayRegion = mock_byte_array_region, + }; + mock_env = &native; + static JavaVM vm = &invocation; + return &vm; +} + +static void worker_attachments(void) { + JavaContext context = {.vm = mock_vm(), .loader = (jobject) (uintptr_t) 2}; + /* Two external owners prevent stack-backed context finalization in this test. */ + atomic_init(&context.references, 2); + EngineApi api = {.context = &context}; + EngineHandle factory = {.api = &api, .kind = ENGINE_FACTORY}; + EngineHandle query = {.api = &api, .kind = ENGINE_QUERY}; + BridgeBuffer error = {0}; + const char *invalid = "JVM_ENGINE_ERROR:13:invalid Java engine handle"; + assert(spark_connect_engine_thread_attach(NULL, &error) == NULL); + attachment_error(&error, invalid); + assert(spark_connect_engine_thread_attach(&query, &error) == NULL); + attachment_error(&error, invalid); + assert(mock_attaches == 0); + + mock_attach_result = JNI_ERR; + assert(spark_connect_engine_thread_attach(&factory, &error) == NULL); + attachment_error(&error, "JVM_ENGINE_ERROR:13:cannot attach Java engine worker"); + assert(atomic_load(&context.references) == 2 && mock_attaches == 0); + mock_attach_result = JNI_OK; + mock_get_env_result = JNI_EVERSION; + assert(spark_connect_engine_thread_attach(&factory, &error) == NULL); + attachment_error(&error, "JVM_ENGINE_ERROR:13:cannot access Java engine worker context"); + assert(atomic_load(&context.references) == 2); + mock_get_env_result = JNI_OK; + + void *outer = spark_connect_engine_thread_attach(&factory, &error); + assert(outer != NULL && error.data == NULL); + assert(mock_attaches == 1 && mock_named_attaches == 1); + char first_worker_name[sizeof(mock_worker_name)]; + strcpy(first_worker_name, mock_worker_name); + assert(atomic_load(&context.references) == 3); + void *inner = spark_connect_engine_thread_attach(&factory, &error); + assert(inner != NULL && mock_attaches == 1); + assert(atomic_load(&context.references) == 4); + assert(spark_connect_engine_thread_detach(inner, &error) == 0); + assert(mock_detaches == 0 && atomic_load(&context.references) == 3); + + mock_loader = (jobject) (uintptr_t) 3; + JavaScope first, second; + assert(scope_enter(&context, &first, &error) == 0); + assert(mock_loader == context.loader && mock_frame_depth == 1); + assert(scope_enter(&context, &second, &error) == 0); + assert(mock_frame_depth == 2 && mock_attaches == 1); + scope_leave(&second); + assert(mock_frame_depth == 1 && mock_loader == context.loader); + scope_leave(&first); + assert(mock_frame_depth == 0 && mock_loader == (jobject) (uintptr_t) 3); + assert(mock_pushes == 2 && mock_pops == 2 && mock_detaches == 0); + + mock_push_result = JNI_ERR; + assert(scope_enter(&context, &first, &error) == -1); + attachment_error(&error, "cannot allocate Java local reference frame"); + assert(mock_frame_depth == 0 && mock_pops == 2 && mock_detaches == 0); + mock_push_result = JNI_OK; + + pthread_t other; + assert(pthread_create(&other, NULL, detach_on_wrong_thread, outer) == 0); + assert(pthread_join(other, NULL) == 0); + assert(mock_detaches == 0 && atomic_load(&context.references) == 3); + mock_detach_result = JNI_ERR; + assert(spark_connect_engine_thread_detach(outer, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:13:cannot detach Java engine worker"); + assert(mock_detaches == 0 && atomic_load(&context.references) == 3); + mock_detach_result = JNI_OK; + assert(spark_connect_engine_thread_detach(outer, &error) == 0); + assert(mock_detaches == 1 && atomic_load(&context.references) == 2); + assert(spark_connect_engine_thread_detach(NULL, &error) == 0); + + /* An attachment borrowed from Java must remain attached after guard release. */ + mock_attached = 1; + outer = spark_connect_engine_thread_attach(&factory, &error); + assert(outer != NULL && mock_attaches == 1); + assert(spark_connect_engine_thread_detach(outer, &error) == 0); + assert(mock_attached && mock_detaches == 1 && atomic_load(&context.references) == 2); + mock_attached = 0; + + /* A new native attachment gets a distinct name, copied by the VM on entry. */ + outer = spark_connect_engine_thread_attach(&factory, &error); + assert(outer != NULL && error.data == NULL); + assert(mock_attaches == 2 && mock_named_attaches == 2); + assert(strcmp(first_worker_name, mock_worker_name) != 0); + assert(spark_connect_engine_thread_detach(outer, &error) == 0); + assert(mock_detaches == 2 && atomic_load(&context.references) == 2); + + /* The existing non-persistent path still balances attachment and local frames. */ + assert(scope_enter(&context, &first, &error) == 0); + assert(mock_attaches == 3 && mock_frame_depth == 1); + scope_leave(&first); + assert(mock_detaches == 3 && mock_frame_depth == 0 && mock_pushes == mock_pops + 1); + assert(mock_named_attaches == 2); + puts("C_WORKER_ATTACHMENTS_OK: nested ownership, same-thread release, scoped frames/loaders, failure retry"); +} + +static EngineHandle *mock_handle(EngineKind kind) { + JavaContext *context = calloc(1, sizeof(*context)); + EngineApi *api = calloc(1, sizeof(*api)); + EngineHandle *handle = calloc(1, sizeof(*handle)); + assert(context != NULL && api != NULL && handle != NULL); + context->vm = mock_vm(); + context->loader = (jobject) (uintptr_t) 2; + context->thread_class = (jclass) (uintptr_t) 5; + atomic_init(&context->references, 1); + api->context = context; + atomic_init(&api->references, 1); + for (int i = 0; i < ENGINE_EXCEPTION; i++) api->close[i] = MOCK_CLOSE; + api->next_ipc = MOCK_NEXT; + api->batch_data = MOCK_DATA; + api->batch_row_count = (jmethodID) (uintptr_t) 11; + api->cancel = MOCK_CANCEL; + handle->api = api; + handle->kind = kind; + handle->object = (jobject) (uintptr_t) 30; + mock_loader = (jobject) (uintptr_t) 3; + mock_close_calls = mock_globals_deleted = mock_loader_deletions = 0; + mock_close_failure = mock_new_global_failure = mock_next_failure = 0; + mock_next_batch = NULL; + mock_ipc_data = NULL; + mock_row_count = 0; + mock_copy_failure = 0; + mock_copy_calls = 0; + mock_copy_destination = NULL; + mock_cancel_calls = mock_cancel_failure = 0; + assert(!mock_attached && mock_frame_depth == 0 && !mock_exception_pending); + return handle; +} + +static void checked_cleanup_ownership(void) { + EngineHandle *handle = mock_handle(ENGINE_FACTORY); + EngineApi *api = handle->api; + JavaContext *context = api->context; + BridgeBuffer error = {0}; + int attaches = mock_attaches; + mock_attach_result = JNI_ERR; + assert(spark_connect_engine_free_checked(handle, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:13:cannot attach Rust worker to JVM"); + assert(mock_close_calls == 0 && mock_globals_deleted == 0); + assert(atomic_load(&api->references) == 1 && atomic_load(&context->references) == 1); + mock_attach_result = JNI_OK; + + mock_push_result = JNI_ERR; + assert(spark_connect_engine_free_checked(handle, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:13:cannot allocate Java local reference frame"); + assert(mock_close_calls == 0 && mock_globals_deleted == 0); + assert(atomic_load(&api->references) == 1 && atomic_load(&context->references) == 1); + mock_push_result = JNI_OK; + + mock_close_failure = 1; + assert(spark_connect_engine_free_checked(handle, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:13:injected child access failure"); + assert(mock_close_calls == 1 && mock_globals_deleted == 0); + assert(handle->api == api && handle->object == (jobject) (uintptr_t) 30); + assert(atomic_load(&api->references) == 1 && atomic_load(&context->references) == 1); + assert(!mock_attached && mock_frame_depth == 0); + mock_close_failure = 0; + assert(spark_connect_engine_free_checked(handle, &error) == 0 && error.data == NULL); + assert(mock_close_calls == 2 && mock_globals_deleted == 3 && mock_loader_deletions == 1); + /* Each attempt has one scope; final global deletion must not reattach. */ + assert(mock_attaches == attaches + 3 && !mock_attached && mock_frame_depth == 0); + + handle = mock_handle(ENGINE_FACTORY); + context = handle->api->context; + void *worker = spark_connect_engine_thread_attach(handle, &error); + assert(worker != NULL && atomic_load(&context->references) == 2); + assert(spark_connect_engine_free_checked(handle, &error) == -1); + attachment_error(&error, + "JVM_ENGINE_ERROR:13:Java engine worker attachments remain during factory cleanup"); + assert(mock_close_calls == 0 && mock_globals_deleted == 0); + assert(spark_connect_engine_thread_detach(worker, &error) == 0); + assert(atomic_load(&context->references) == 1); + assert(spark_connect_engine_free_checked(handle, &error) == 0); + assert(mock_close_calls == 1 && mock_globals_deleted == 3); + + puts("C_CHECKED_CLEANUP_OK: attach/Java failures retain owners; worker guards prevent factory free; final globals use the original scope"); +} + +typedef struct { + uint8_t data[4]; + size_t len; + int calls; + int fail; +} MockIpcDestination; + +static uint8_t *mock_allocate_ipc(void *context, size_t len) { + MockIpcDestination *destination = context; + assert(destination != NULL); + destination->calls++; + destination->len = len; + if (destination->fail) return NULL; + assert(len <= sizeof(destination->data)); + return destination->data; +} + +static void ipc_transfer(void) { + EngineHandle *query = mock_handle(ENGINE_QUERY); + BridgeBuffer error = {0}; + MockIpcDestination destination = {0}; + int64_t count = -1; + assert(spark_connect_engine_next_ipc(NULL, mock_allocate_ipc, &destination, + &count, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:13:invalid Java engine handle"); + assert(destination.calls == 0 && count == 0); + assert(spark_connect_engine_next_ipc(query, NULL, &destination, &count, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:13:missing Java engine IPC destination"); + assert(spark_connect_engine_next_ipc(query, mock_allocate_ipc, &destination, + NULL, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:13:missing Java engine IPC destination"); + mock_attach_result = JNI_ERR; + assert(spark_connect_engine_next_ipc(query, mock_allocate_ipc, &destination, + &count, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:13:cannot attach Rust worker to JVM"); + mock_attach_result = JNI_OK; + + assert(spark_connect_engine_next_ipc(query, mock_allocate_ipc, &destination, + &count, &error) == 0); + assert(error.data == NULL && destination.calls == 0 && count == 0); + mock_next_failure = 1; + assert(spark_connect_engine_next_ipc(query, mock_allocate_ipc, &destination, + &count, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:13:injected child access failure"); + assert(destination.calls == 0 && mock_copy_calls == 0 && count == 0); + mock_next_failure = 0; + + uint8_t data[] = {0, 255, 3, 7}; + MockBytes bytes = {.len = sizeof(data), .data = data}; + mock_next_batch = (jobject) (uintptr_t) 31; + mock_ipc_data = (jbyteArray) &bytes; + mock_row_count = 2; + assert(spark_connect_engine_next_ipc(query, mock_allocate_ipc, &destination, + &count, &error) == 1); + assert(error.data == NULL && destination.len == sizeof(data) && count == 2); + assert(destination.calls == 1 && mock_copy_calls == 1); + assert(mock_copy_destination == (jbyte *) destination.data); + assert(memcmp(destination.data, data, sizeof(data)) == 0); + /* The returned copy remains valid independently of Java's next-batch storage. */ + data[0] = 99; + assert(destination.data[0] == 0); + + bytes.len = 0; + mock_row_count = 0; + assert(spark_connect_engine_next_ipc(query, mock_allocate_ipc, &destination, + &count, &error) == 1); + assert(error.data == NULL && destination.len == 0 && count == 0); + assert(destination.calls == 2 && mock_copy_calls == 1); + bytes.len = sizeof(data); + + mock_row_count = -1; + assert(spark_connect_engine_next_ipc(query, mock_allocate_ipc, &destination, + &count, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:13:negative Java engine IPC row count"); + assert(destination.calls == 2 && mock_copy_calls == 1 && count == 0); + mock_row_count = 0; + bytes.len = 64 * 1024 * 1024 + 1; + assert(spark_connect_engine_next_ipc(query, mock_allocate_ipc, &destination, + &count, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:8:Spark Arrow batch exceeds JNI limit"); + assert(destination.calls == 2 && mock_copy_calls == 1 && count == 0); + bytes.len = sizeof(data); + mock_ipc_data = NULL; + assert(spark_connect_engine_next_ipc(query, mock_allocate_ipc, &destination, + &count, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:13:Java engine returned no IPC data"); + assert(destination.calls == 2 && mock_copy_calls == 1 && count == 0); + mock_ipc_data = (jbyteArray) &bytes; + + destination.fail = 1; + count = -1; + assert(spark_connect_engine_next_ipc(query, mock_allocate_ipc, &destination, + &count, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:8:cannot allocate Rust IPC buffer"); + assert(destination.calls == 3 && mock_copy_calls == 1 && count == 0); + assert(mock_frame_depth == 0 && !mock_exception_pending); + destination.fail = 0; + + mock_copy_failure = 1; + assert(spark_connect_engine_next_ipc(query, mock_allocate_ipc, &destination, + &count, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:13:injected child access failure"); + assert(destination.calls == 4 && mock_copy_calls == 2 && count == 0); + assert(mock_copy_destination == (jbyte *) destination.data); + assert(mock_frame_depth == 0 && !mock_exception_pending); + mock_copy_failure = 0; + assert(spark_connect_engine_free_checked(query, &error) == 0); + puts("C_IPC_TRANSFER_OK: direct caller-owned destination, empty batches, allocation/copy errors, and local-reference cleanup"); +} + +static void checked_cancellation(void) { + EngineHandle *task = mock_handle(ENGINE_TASK); + EngineApi *api = task->api; + JavaContext *context = api->context; + BridgeBuffer error = {0}; + assert(spark_connect_engine_cancel(NULL, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:13:invalid Java engine handle"); + EngineHandle wrong_kind = {.api = api, .kind = ENGINE_QUERY}; + assert(spark_connect_engine_cancel(&wrong_kind, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:13:invalid Java engine handle"); + assert(mock_cancel_calls == 0); + + mock_attach_result = JNI_ERR; + assert(spark_connect_engine_cancel(task, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:13:cannot attach Rust worker to JVM"); + assert(mock_cancel_calls == 0 && mock_globals_deleted == 0); + assert(atomic_load(&api->references) == 1 && atomic_load(&context->references) == 1); + mock_attach_result = JNI_OK; + // A failed delivery retains the same task, allowing a later caller to retry. + assert(spark_connect_engine_cancel(task, &error) == 0 && error.data == NULL); + assert(mock_cancel_calls == 1); + + mock_push_result = JNI_ERR; + assert(spark_connect_engine_cancel(task, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:13:cannot allocate Java local reference frame"); + assert(mock_cancel_calls == 1); + mock_push_result = JNI_OK; + mock_cancel_failure = 1; + assert(spark_connect_engine_cancel(task, &error) == -1); + attachment_error(&error, "JVM_ENGINE_ERROR:13:injected child access failure"); + assert(mock_cancel_calls == 2 && mock_globals_deleted == 0); + assert(!mock_attached && mock_frame_depth == 0 && !mock_exception_pending); + assert(atomic_load(&api->references) == 1 && atomic_load(&context->references) == 1); + mock_cancel_failure = 0; + assert(spark_connect_engine_free_checked(task, &error) == 0); + puts("C_CHECKED_CANCEL_OK: failed delivery and Java exceptions reported without consuming task ownership"); +} + +static void throw_error_allocation_failure(void) { + (void) mock_vm(); + JNIEnv saved_env = mock_env; + struct JNINativeInterface_ error_native = *mock_env; + // Install these restrictive fault mocks only for the throw_error cases. + error_native.NewByteArray = mock_fail_message_allocation; + error_native.FindClass = mock_find_exception_class; + error_native.ThrowNew = mock_throw_new; + mock_env = &error_native; + assert(!mock_attached && mock_frame_depth == 0 && !mock_exception_pending); + mock_attached = 1; + assert(mock_push_frame(&mock_env, 32) == JNI_OK); + for (int failure = 0; failure < 2; failure++) { + BridgeBuffer error = {0}; + set_error(&error, "injected native stop failure"); + mock_throw_new_failure = failure; + throw_error(&mock_env, &error); + assert(error.data == NULL && error.len == 0); + // Both the fallback throwable and a fallback allocation failure stay pending. + assert(mock_exception_pending); + assert(mock_frame_references[mock_frame_depth - 1] == 0); + mock_exception_clear(&mock_env); + } + assert(mock_message_allocation_calls == 2 && mock_throw_new_calls == 2); + // An already pending exception is preserved without allocating another message. + BridgeBuffer error = {0}; + set_error(&error, "injected native stop failure"); + mock_exception_pending = 1; + throw_error(&mock_env, &error); + assert(mock_exception_pending && error.data == NULL && error.len == 0); + assert(mock_message_allocation_calls == 2 && mock_throw_new_calls == 2); + mock_exception_clear(&mock_env); + mock_throw_new_failure = 0; + mock_pop_frame(&mock_env, NULL); + mock_attached = 0; + mock_env = saved_env; + puts("C_THROW_ERROR_OK: message allocation failure throws fallback and preserves pending exceptions"); +} + +int main(void) { + analysis_preconditions(); + worker_attachments(); + checked_cleanup_ownership(); + ipc_transfer(); + checked_cancellation(); + throw_error_allocation_failure(); + return 0; +} diff --git a/crates/spark-connect-server/jvm/build.sh b/crates/spark-connect-server/jvm/build.sh new file mode 100755 index 00000000..2723036d --- /dev/null +++ b/crates/spark-connect-server/jvm/build.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +if [[ $# -gt 1 || ( $# -eq 1 && "$1" != "--tests" ) ]]; then + echo "usage: $0 [--tests]" >&2 + exit 1 +fi + +PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +: "${SPARK_HOME:?Set SPARK_HOME to an existing Spark 4.2 assembly with shaded Connect gRPC jars}" +SPARK_JARS="$SPARK_HOME/jars" +if [[ -d "$SPARK_HOME/assembly/target/scala-2.13/jars" ]]; then + SPARK_JARS="$SPARK_HOME/assembly/target/scala-2.13/jars" +fi +if [[ ! -d "$SPARK_JARS" ]]; then + echo "Spark jar directory does not exist: $SPARK_JARS" >&2 + exit 1 +fi +JAVAC="${JAVA_HOME:+$JAVA_HOME/bin/}javac" +JAR="${JAVA_HOME:+$JAVA_HOME/bin/}jar" +BUILD_DIR="$PROJECT_DIR/.build/jvm" +mkdir -p "$BUILD_DIR" +# Concurrent builders keep their classes private. Publish only complete jars so +# a running launcher never observes a removed or partially written application jar. +BUILD_STAGE="$(mktemp -d "$BUILD_DIR/.build.XXXXXX")" +trap 'rm -rf -- "$BUILD_STAGE"' EXIT +mkdir -p "$BUILD_STAGE/classes" +MAIN_SOURCES=() +while IFS= read -r -d '' source; do + MAIN_SOURCES+=("$source") +done < <(find "$PROJECT_DIR/jvm/src/main/java" -name '*.java' -type f -print0) +"$JAVAC" -proc:none -cp "$SPARK_JARS/*" -d "$BUILD_STAGE/classes" \ + "${MAIN_SOURCES[@]}" +mkdir -p "$BUILD_STAGE/classes/META-INF" +cp "$PROJECT_DIR/../../LICENSE" "$BUILD_STAGE/classes/META-INF/LICENSE" +if [[ -f "$PROJECT_DIR/../../NOTICE" ]]; then + cp "$PROJECT_DIR/../../NOTICE" "$BUILD_STAGE/classes/META-INF/NOTICE" +fi +NATIVE_VERSION="$("${NATIVE_BUILD_PYTHON:-python3}" "$PROJECT_DIR/jvm/package-native.py" --print-version)" +printf 'artifact.version=%s\nnative.abi.version=1\n' "$NATIVE_VERSION" \ + > "$BUILD_STAGE/classes/META-INF/spark-connect-native-adapter.properties" +"$JAR" --create --file "$BUILD_STAGE/spark-connect-native-bridge.jar" -C "$BUILD_STAGE/classes" . +if [[ "${1:-}" == "--tests" ]]; then + mkdir -p "$BUILD_STAGE/test-classes" + TEST_SOURCES=() + while IFS= read -r -d '' source; do + TEST_SOURCES+=("$source") + done < <(find "$PROJECT_DIR/jvm/src/test/java" -name '*.java' -type f -print0) + "$JAVAC" -proc:none -cp "$SPARK_JARS/*:$BUILD_STAGE/classes" -d "$BUILD_STAGE/test-classes" \ + "${TEST_SOURCES[@]}" + "$JAR" --create --file "$BUILD_STAGE/bridge-test.jar" \ + -C "$BUILD_STAGE/classes" . -C "$BUILD_STAGE/test-classes" . +fi +# A compilation failure preserves previously published jars. Callers must honor +# a nonzero exit status and must not launch those older jars after a failed build. +# Both jars are self-contained, so publishing them independently is safe. +mv -f -- "$BUILD_STAGE/spark-connect-native-bridge.jar" "$BUILD_DIR/spark-connect-native-bridge.jar" +echo "$BUILD_DIR/spark-connect-native-bridge.jar" +if [[ "${1:-}" == "--tests" ]]; then + mv -f -- "$BUILD_STAGE/bridge-test.jar" "$BUILD_DIR/bridge-test.jar" + echo "$BUILD_DIR/bridge-test.jar" +fi diff --git a/crates/spark-connect-server/jvm/engine_acceptance.py b/crates/spark-connect-server/jvm/engine_acceptance.py new file mode 100755 index 00000000..99381fc9 --- /dev/null +++ b/crates/spark-connect-server/jvm/engine_acceptance.py @@ -0,0 +1,794 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Official PySpark and raw gRPC checks for the Rust frontend/JVM Engine backend.""" + +import argparse +import concurrent.futures +import datetime +from decimal import Decimal +import hashlib +import json +import logging +from pathlib import Path +import time +import tempfile +import traceback +import uuid + + +class RawSession: + def __init__(self, remote, timeout): + import grpc + from pyspark.sql.connect.proto import base_pb2 as base + from pyspark.sql.connect.proto import base_pb2_grpc as rpc + + self.base = base + self.timeout = timeout + self.session_id = str(uuid.uuid4()) + self.user = base.UserContext(user_id="jvm-engine-acceptance") + self.server_id = None + self.released = False + address = remote.removeprefix("sc://").split("/", 1)[0] + self.channel = grpc.insecure_channel(address) + self.stub = rpc.SparkConnectServiceStub(self.channel) + + def identity(self): + fields = {"session_id": self.session_id, "user_context": self.user} + if self.server_id is not None: + fields["client_observed_server_side_session_id"] = self.server_id + return fields + + def observe(self, response): + assert response.session_id == self.session_id, response + assert response.server_side_session_id, response + if self.server_id is not None: + assert response.server_side_session_id == self.server_id, response + self.server_id = response.server_side_session_id + + def config(self, name, payload): + request = self.base.ConfigRequest( + **self.identity(), operation=self.base.ConfigRequest.Operation(**{name: payload}) + ) + response = self.stub.Config(request, timeout=self.timeout) + self.observe(response) + return {pair.key: pair.value if pair.HasField("value") else None + for pair in response.pairs} + + def execute(self, relation, operation_id=None): + operation_id = operation_id or str(uuid.uuid4()) + request = self.base.ExecutePlanRequest( + **self.identity(), operation_id=operation_id, + plan=self.base.Plan(root=relation) + ) + return operation_id, self.stub.ExecutePlan(request, timeout=self.timeout) + + def collect(self, relation, expected_arrow_schema=None): + import pyarrow as pa + + operation_id, stream = self.execute(relation) + rows = [] + complete = False + arrow_batches = 0 + response_ids = set() + try: + for response in stream: + self.observe(response) + assert response.operation_id == operation_id, response + assert response.response_id and response.response_id not in response_ids, response + response_ids.add(response.response_id) + if response.HasField("arrow_batch"): + table = pa.ipc.open_stream(response.arrow_batch.data).read_all() + assert table.num_rows == response.arrow_batch.row_count, response + if expected_arrow_schema is not None: + assert table.schema.equals(expected_arrow_schema, check_metadata=False), ( + table.schema, expected_arrow_schema + ) + arrow_batches += 1 + rows.extend(table.to_pylist()) + if response.HasField("result_complete"): + complete = True + assert complete, "query ended without result_complete" + assert expected_arrow_schema is None or arrow_batches, "query returned no Arrow schema" + return rows + finally: + stream.cancel() + + def release(self, timeout=None): + response = self.stub.ReleaseSession( + self.base.ReleaseSessionRequest(session_id=self.session_id, user_context=self.user), + timeout=self.timeout if timeout is None else timeout, + ) + self.released = True + assert response.session_id == self.session_id, response + if self.server_id is not None: + assert response.server_side_session_id == self.server_id, response + return response + + def close(self): + try: + if not self.released: + self.release() + finally: + self.channel.close() + + +def expect_rpc_error(action, codes, contains=None): + import grpc + + try: + action() + except grpc.RpcError as error: + assert error.code() in codes, (error.code(), error.details()) + if contains is not None: + assert contains in error.details(), error.details() + return {"code": error.code().name, "details": error.details()} + raise AssertionError("request unexpectedly succeeded") + + +def main(): + if not __debug__: + raise RuntimeError("acceptance checks require Python assertions; disable optimized mode") + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--remote", required=True, help="sc://host:port") + parser.add_argument("--output", type=Path, help="write the checked JSON report here") + parser.add_argument("--timeout", type=float, default=60) + parser.add_argument("--cancel-timeout", type=float, default=15) + args = parser.parse_args() + if not args.remote.startswith("sc://"): + parser.error("--remote must start with sc://") + if args.timeout <= 0 or args.cancel_timeout <= 0: + parser.error("timeouts must be positive") + + import grpc + import pyspark + from pyspark.sql import SparkSession, functions as F, types as T + from pyspark.sql.connect.proto import base_pb2 as base + from pyspark.sql.connect.proto import base_pb2_grpc as rpc + from pyspark.sql.connect.proto import commands_pb2 as commands + from pyspark.sql.connect.proto import relations_pb2 as relations + + logging.getLogger("SQLQueryContextLogger").setLevel(logging.CRITICAL) + checks = [] + raw_sessions = [] + spark = None + + def check(name, action): + started = time.monotonic() + try: + result = action() + checks.append({"name": name, "passed": True, + "seconds": round(time.monotonic() - started, 3), "result": result}) + except Exception as error: + checks.append({"name": name, "passed": False, + "seconds": round(time.monotonic() - started, 3), + "error": repr(error), "traceback": traceback.format_exc()}) + + def raw_session(): + session = RawSession(args.remote, args.timeout) + raw_sessions.append(session) + return session + + def sql(query): + return relations.Relation(sql=relations.SQL(query=query)) + + try: + spark = SparkSession.builder.remote(args.remote).create() + + def builder_configuration(): + key = "spark.acceptance.builder." + uuid.uuid4().hex + shuffle = "spark.sql.shuffle.partitions" + previous = spark.conf.get(shuffle) + try: + configured = (SparkSession.builder.remote(args.remote) + .config(key, "builder-value") + .config(shuffle, "3").getOrCreate()) + assert configured.session_id == spark.session_id + assert configured.conf.get(key) == "builder-value" + assert configured.conf.get(shuffle) == "3" + return {"builder_silent_set": True, "unregistered_spark_key": True} + finally: + spark.conf.set(shuffle, previous) + spark.conf.unset(key) + + check("official_builder_configuration", builder_configuration) + + def dataframe_join(): + left = spark.range(4).withColumnRenamed("id", "left_id") + right = spark.range(2, 6).withColumnRenamed("id", "right_id") + rows = left.join(right, F.col("left_id") == F.col("right_id")).orderBy( + "left_id" + ).collect() + actual = [tuple(row) for row in rows] + assert actual == [(2, 2), (3, 3)], actual + return actual + + check("official_dataframe_join", dataframe_join) + + def unaliased_computations(): + frame = spark.range(4) + assert [row[0] for row in frame.select(F.col("id") + 1).collect()] == [1, 2, 3, 4] + assert frame.agg(F.sum("id")).collect()[0][0] == 6 + grouped = (frame.select((F.col("id") % 2).alias("k"), "id") + .groupBy("k").sum("id").orderBy("k").collect()) + assert [tuple(row) for row in grouped] == [(0, 2), (1, 4)] + session = raw_session() + rows = session.collect(sql("SELECT sum(id) FROM range(4)")) + assert [list(row.values()) for row in rows] == [[6]], rows + return {"sql_and_dataframe_unaliased_results": True} + + check("official_and_raw_unaliased_computations", unaliased_computations) + + def catalyst_types(): + frame = spark.sql( + "SELECT transform(array(1, 2, 3), x -> x * x) AS squares, " + "CAST('123.45' AS DECIMAL(10, 2)) AS amount, " + "map('Ï€', CAST(7 AS BIGINT)) AS labels, CAST(NULL AS INT) AS missing, " + "1 AS required, array(1, CAST(NULL AS INT)) AS nullable_array, " + "map('x', CAST(NULL AS INT)) AS nullable_map" + ) + schema = frame.schema + assert schema["squares"].dataType == T.ArrayType(T.IntegerType(), False), schema + assert schema["amount"].dataType == T.DecimalType(10, 2), schema + assert schema["labels"].dataType == T.MapType(T.StringType(), T.LongType(), False), schema + assert schema["missing"].nullable and not schema["required"].nullable, schema + assert schema["nullable_array"].dataType.containsNull, schema + assert schema["nullable_map"].dataType.valueContainsNull, schema + row = frame.collect()[0] + assert row.squares == [1, 4, 9] and row.amount == Decimal("123.45"), row + assert row.labels == {"Ï€": 7} and row.missing is None and row.required == 1, row + assert row.nullable_array == [1, None] and row.nullable_map == {"x": None}, row + assert spark.range(0).collect() == [], "empty query returned rows" + return {"schema": schema.jsonValue(), "amount": str(row.amount), + "squares": row.squares, "empty_rows": 0} + + check("official_catalyst_decimal_array_map_nullability", catalyst_types) + + def ipc_decimal_and_empty_children(): + # These layouts place decimal data after eight-byte IPC padding and exercise + # missing initial offsets in empty nested children before the Rust import. + cases = [ + ("SELECT CAST('1.23' AS DECIMAL(10,2)) AS amount", + [{"amount": Decimal("1.23")}]), + ("SELECT true AS flag, CAST('-45.67' AS DECIMAL(10,2)) AS amount, " + "CAST(7 AS TINYINT) AS small, CAST('0.01' AS DECIMAL(10,2)) AS other", + [{"flag": True, "amount": Decimal("-45.67"), "small": 7, + "other": Decimal("0.01")}]), + ("SELECT array(CAST('1.23' AS DECIMAL(10,2)), CAST(NULL AS DECIMAL(10,2))) AS amounts, " + "named_struct('amount', CAST('-45.67' AS DECIMAL(10,2))) AS nested, " + "map('x', CAST('0.01' AS DECIMAL(10,2))) AS mapping", + [{"amounts": [Decimal("1.23"), None], "nested": {"amount": Decimal("-45.67")}, + "mapping": [("x", Decimal("0.01"))]}]), + ("SELECT CAST(array() AS ARRAY) AS items, " + "array(CAST(array() AS ARRAY)) AS nested, " + "CAST(map() AS MAP>) AS mapping", + [{"items": [], "nested": [[]], "mapping": []}]), + ] + session = raw_session() + for large_vars in (False, True): + session.config("set", base.ConfigRequest.Set(pairs=[base.KeyValue( + key="spark.sql.execution.arrow.useLargeVarTypes", + value=str(large_vars).lower())])) + for statement, expected in cases: + actual = session.collect(sql(statement)) + assert actual == expected, (statement, large_vars, actual, expected) + return {"query_layouts": len(cases), "large_variable_type_settings": 2, + "decimal_values_and_empty_children_preserved": True} + + check("raw_ipc_decimal_alignment_and_empty_nested_children", ipc_decimal_and_empty_children) + + def arrow_edge_types(large_vars): + import pyarrow as pa + + query = ( + "SELECT NULL AS untyped_null, true AS true_value, false AS false_value, " + "CAST(NULL AS BOOLEAN) AS null_boolean, '' AS empty_text, 'π😀' AS unicode_text, " + "X'' AS empty_binary, X'00FFCF80F09F9880' AS binary_value, " + "named_struct('empty', '', 'inner', named_struct('flag', true, 'missing', NULL)) " + "AS nested, TIMESTAMP '2026-01-02 03:04:05.123456' AS instant, " + "TIMESTAMP_NTZ '2026-01-02 03:04:05.123456' AS wall_time, " + "DATE '2026-01-02' AS calendar_day" + ) + text = pa.large_string() if large_vars else pa.string() + binary = pa.large_binary() if large_vars else pa.binary() + arrow_schema = pa.schema([ + pa.field("untyped_null", pa.null(), nullable=True), + pa.field("true_value", pa.bool_(), nullable=False), + pa.field("false_value", pa.bool_(), nullable=False), + pa.field("null_boolean", pa.bool_(), nullable=True), + pa.field("empty_text", text, nullable=False), + pa.field("unicode_text", text, nullable=False), + pa.field("empty_binary", binary, nullable=False), + pa.field("binary_value", binary, nullable=False), + pa.field("nested", pa.struct([ + pa.field("empty", text, nullable=False), + pa.field("inner", pa.struct([ + pa.field("flag", pa.bool_(), nullable=False), + pa.field("missing", pa.null(), nullable=True), + ]), nullable=False), + ]), nullable=False), + pa.field("instant", pa.timestamp("us", tz="UTC"), nullable=False), + pa.field("wall_time", pa.timestamp("us"), nullable=False), + pa.field("calendar_day", pa.date32(), nullable=False), + ]) + instant = datetime.datetime(2026, 1, 2, 3, 4, 5, 123456, + tzinfo=datetime.timezone.utc) + expected = { + "untyped_null": None, "true_value": True, "false_value": False, + "null_boolean": None, "empty_text": "", "unicode_text": "π😀", + "empty_binary": b"", "binary_value": bytes.fromhex("00ffcf80f09f9880"), + "nested": {"empty": "", "inner": {"flag": True, "missing": None}}, + "instant": instant, "wall_time": instant.replace(tzinfo=None), + "calendar_day": datetime.date(2026, 1, 2), + } + session = raw_session() + key = "spark.sql.execution.arrow.useLargeVarTypes" + session.config("set", base.ConfigRequest.Set(pairs=[ + base.KeyValue(key=key, value=str(large_vars).lower()), + base.KeyValue(key="spark.sql.session.timeZone", value="UTC"), + ])) + actual = session.collect(sql(query), expected_arrow_schema=arrow_schema) + assert actual == [expected], actual + + spark_schema = T.StructType([ + T.StructField("untyped_null", T.NullType(), True), + T.StructField("true_value", T.BooleanType(), False), + T.StructField("false_value", T.BooleanType(), False), + T.StructField("null_boolean", T.BooleanType(), True), + T.StructField("empty_text", T.StringType(), False), + T.StructField("unicode_text", T.StringType(), False), + T.StructField("empty_binary", T.BinaryType(), False), + T.StructField("binary_value", T.BinaryType(), False), + T.StructField("nested", T.StructType([ + T.StructField("empty", T.StringType(), False), + T.StructField("inner", T.StructType([ + T.StructField("flag", T.BooleanType(), False), + T.StructField("missing", T.NullType(), True), + ]), False), + ]), False), + T.StructField("instant", T.TimestampType(), False), + T.StructField("wall_time", T.TimestampNTZType(), False), + T.StructField("calendar_day", T.DateType(), False), + ]) + previous = {name: spark.conf.get(name) for name in + (key, "spark.sql.session.timeZone")} + try: + spark.conf.set(key, str(large_vars).lower()) + spark.conf.set("spark.sql.session.timeZone", "UTC") + frame = spark.sql(query) + assert frame.schema == spark_schema, (frame.schema, spark_schema) + official = [row.asDict(recursive=True) for row in frame.collect()] + official_expected = dict(expected) + # Official PySpark collect converts TimestampType to the client's local timezone. + official_expected["instant"] = instant.astimezone().replace(tzinfo=None) + assert official == [official_expected], official + finally: + for name, value in previous.items(): + spark.conf.set(name, value) + return {"large_var_types": large_vars, "rows": 1, + "arrow_schema": str(arrow_schema), "spark_schema": spark_schema.jsonValue(), + "timestamp_utc": instant.isoformat(), "binary_hex": expected["binary_value"].hex(), + "exact_values_and_nullability": True} + + for large_vars in (False, True): + check("raw_and_official_arrow_edge_types_large_vars_" + str(large_vars).lower(), + lambda large_vars=large_vars: arrow_edge_types(large_vars)) + + def config_isolation_and_rollback(): + first, second = raw_session(), raw_session() + key = "spark-connect.acceptance." + uuid.uuid4().hex + created = key + ".new" + first.config("set", base.ConfigRequest.Set(pairs=[base.KeyValue(key=key, value="before 😀")])) + assert first.config("get", base.ConfigRequest.Get(keys=[key])) == {key: "before 😀"} + assert second.config("get_option", base.ConfigRequest.GetOption(keys=[key])) == {key: None} + assert first.server_id != second.server_id + shuffle = "spark.sql.shuffle.partitions" + previous = first.config("get", base.ConfigRequest.Get(keys=[shuffle]))[shuffle] + failure = expect_rpc_error( + lambda: first.config("set", base.ConfigRequest.Set(pairs=[ + base.KeyValue(key=key, value="after"), + base.KeyValue(key=created, value="created"), + base.KeyValue(key=shuffle, value="invalid-integer"), + ])), {grpc.StatusCode.INVALID_ARGUMENT} + ) + actual = first.config("get_option", base.ConfigRequest.GetOption(keys=[key, created, shuffle])) + assert actual == {key: "before 😀", created: None, shuffle: previous}, actual + return {"isolated": True, "rollback_restored_existing_and_removed_new": True, + "failure": failure} + + check("raw_config_isolation_and_atomic_rollback", config_isolation_and_rollback) + + def silent_configuration(): + session = raw_session() + before = "spark.acceptance.silent.before." + uuid.uuid4().hex + after = "spark.acceptance.silent.after." + uuid.uuid4().hex + shuffle = "spark.sql.shuffle.partitions" + static = "spark.sql.warehouse.dir" + previous = session.config("get", base.ConfigRequest.Get(keys=[shuffle]))[shuffle] + request = base.ConfigRequest( + **session.identity(), operation=base.ConfigRequest.Operation( + set=base.ConfigRequest.Set(silent=True, pairs=[ + base.KeyValue(key=before, value="before"), + base.KeyValue(key=shuffle, value="invalid-integer"), + base.KeyValue(key=static, value="/ignored-static-setting"), + base.KeyValue(key=after, value="after"), + ]))) + response = session.stub.Config(request, timeout=args.timeout) + session.observe(response) + assert not response.pairs, response + assert len(response.warnings) == 2, response + assert any(shuffle in warning for warning in response.warnings), response + assert any(static in warning for warning in response.warnings), response + values = session.config("get", base.ConfigRequest.Get(keys=[before, after, shuffle])) + assert values == {before: "before", after: "after", shuffle: previous}, values + return {"valid_pairs_applied": True, "invalid_pairs_warned": 2} + + check("raw_silent_config_continues_after_invalid_and_static_keys", silent_configuration) + + def nullable_config_default(): + missing = "spark-connect.null-default." + uuid.uuid4().hex + assert spark.conf.get(missing, None) is None + session = raw_session() + values = session.config("get_with_default", base.ConfigRequest.GetWithDefault( + pairs=[base.KeyValue(key=missing)])) + assert values == {missing: None}, values + return {"official_null_default": True, "raw_absent_value_is_null": True} + + check("official_and_raw_nullable_config_default", nullable_config_default) + + def unsupported_plans(): + session = raw_session() + name = "engine_forbidden_" + uuid.uuid4().hex + queries = { + "sql_write": sql(f"CREATE TEMP VIEW {name} AS SELECT 1 AS id"), + "sql_script": sql(f"BEGIN CREATE TEMP VIEW {name} AS SELECT 1 AS id; END"), + "with_relations": relations.Relation( + with_relations=relations.WithRelations(root=sql("SELECT 1 AS id")) + ), + } + result = {} + for label, relation in queries.items(): + result[label] = expect_rpc_error( + lambda relation=relation: session.collect(relation), + {grpc.StatusCode.UNIMPLEMENTED}, + ) + result["view_absent"] = expect_rpc_error( + lambda: session.collect(sql(f"SELECT * FROM {name}")), + {grpc.StatusCode.INVALID_ARGUMENT}, contains=name, + ) + return result + + check("raw_unsupported_sql_writes_scripts_with_relations", unsupported_plans) + + def unsupported_spark_schemas(): + session = raw_session() + cases = { + "collated_string": sql("SELECT 'a' COLLATE UTF8_LCASE AS c"), + "field_metadata": spark.range(1).select( + F.col("id").alias("id", metadata={"unit": "seconds"}) + )._plan.plan(spark.client), + } + result = {} + for name, relation in cases.items(): + request = base.AnalyzePlanRequest( + **session.identity(), schema=base.AnalyzePlanRequest.Schema( + plan=base.Plan(root=relation))) + result[name] = { + "schema": expect_rpc_error( + lambda: session.stub.AnalyzePlan(request, timeout=args.timeout), + {grpc.StatusCode.UNIMPLEMENTED}), + "execute": expect_rpc_error( + lambda: session.collect(relation), {grpc.StatusCode.UNIMPLEMENTED}), + } + return result + + check("raw_spark_only_schema_information_is_rejected", unsupported_spark_schemas) + + def shuffle_cleanup_guard(): + session = raw_session() + key = "spark.sql.classic.shuffleDependency.fileCleanup.enabled" + previous = session.config("get_all", base.ConfigRequest.GetAll()) + try: + session.config("set", base.ConfigRequest.Set(pairs=[base.KeyValue(key=key, value="true")])) + failure = expect_rpc_error( + lambda: session.collect(sql("SELECT SUM(id) AS total FROM range(100)")), + {grpc.StatusCode.UNIMPLEMENTED}) + session.config("set", base.ConfigRequest.Set(pairs=[base.KeyValue(key=key, value="false")])) + assert session.collect(sql("SELECT 2 AS answer")) == [{"answer": 2}] + finally: + if key in previous: + session.config("set", base.ConfigRequest.Set(pairs=[base.KeyValue(key=key, value=previous[key])])) + else: + session.config("unset", base.ConfigRequest.Unset(keys=[key])) + restored = session.config("get_all", base.ConfigRequest.GetAll()) + assert (key in restored) == (key in previous), (previous, restored) + assert restored.get(key) == previous.get(key), (previous, restored) + return {"unsupported_setting": failure, "session_recovered": True} + + check("raw_per_fetch_execution_rejects_shuffle_file_cleanup", shuffle_cleanup_guard) + + def rejected_inserts_leave_data_unchanged(): + def sql_string(value): + return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'" + + import pyarrow as pa + import pyarrow.parquet as pq + + session = raw_session() + with tempfile.TemporaryDirectory(prefix="engine-write-guard-quote'-") as directory: + literal = "quote' and backslash\\value" + assert session.collect(sql("SELECT " + sql_string(literal) + " AS value")) == [ + {"value": literal}] + path = Path(directory) + pq.write_table(pa.table({"id": pa.array([7], type=pa.int64())}), path / "seed.parquet") + target = "parquet.`" + directory.replace("`", "``") + "`" + + def snapshot(): + return {str(file.relative_to(path)): hashlib.sha256(file.read_bytes()).hexdigest() + for file in path.rglob("*") if file.is_file()} + + before = snapshot() + assert session.collect(sql(f"SELECT * FROM {target}")) == [{"id": 7}] + results = {} + for label, statement in { + "insert": f"INSERT INTO {target} VALUES (99)", + "overwrite": f"INSERT OVERWRITE TABLE {target} VALUES (99)", + "cte_insert": f"WITH source AS (SELECT 99 AS id) INSERT INTO {target} SELECT * FROM source", + "execute_immediate_insert": "EXECUTE IMMEDIATE " + + sql_string(f"INSERT INTO {target} VALUES (99)"), + "directory_overwrite": "INSERT OVERWRITE DIRECTORY " + + sql_string(directory) + " USING parquet SELECT 99 AS id", + }.items(): + relation = sql(statement) + results[label] = { + "execute_root": expect_rpc_error( + lambda: session.collect(relation), {grpc.StatusCode.UNIMPLEMENTED}) + } + for field, message in (("schema", base.AnalyzePlanRequest.Schema), + ("is_local", base.AnalyzePlanRequest.IsLocal), + ("is_streaming", base.AnalyzePlanRequest.IsStreaming)): + request = base.AnalyzePlanRequest( + **session.identity(), **{field: message(plan=base.Plan(root=relation))}) + results[label][field] = expect_rpc_error( + lambda: session.stub.AnalyzePlan(request, timeout=args.timeout), + {grpc.StatusCode.UNIMPLEMENTED}) + # This is the command shape emitted by official PySpark SparkSession.sql(). + command = commands.Command(sql_command=commands.SqlCommand(input=relation)) + request = base.ExecutePlanRequest( + **session.identity(), operation_id=str(uuid.uuid4()), + plan=base.Plan(command=command)) + results[label]["sql_command"] = expect_rpc_error( + lambda: list(session.stub.ExecutePlan(request, timeout=args.timeout)), + {grpc.StatusCode.UNIMPLEMENTED}) + assert snapshot() == before, "rejected SQL changed data files: " + label + assert session.collect(sql(f"SELECT * FROM {target}")) == [{"id": 7}] + return {"target_rows": 1, "target_value": 7, "files_unchanged": True, + "rejections": results} + + check("raw_insert_overwrite_rejected_without_data_mutation", rejected_inserts_leave_data_unchanged) + + def user_error_statuses(): + session = raw_session() + key = "spark-connect.missing." + uuid.uuid4().hex + return { + "missing_config": expect_rpc_error( + lambda: session.config("get", base.ConfigRequest.Get(keys=[key])), + {grpc.StatusCode.NOT_FOUND}, contains=key), + "invalid_sql": expect_rpc_error( + lambda: session.collect(sql("SELECT FROM")), {grpc.StatusCode.INVALID_ARGUMENT}), + "missing_column": expect_rpc_error( + lambda: session.collect(sql("SELECT absent_column FROM range(1)")), + {grpc.StatusCode.INVALID_ARGUMENT}, contains="absent_column"), + } + + check("raw_spark_user_error_statuses", user_error_statuses) + + def unicode_error(): + session = raw_session() + marker = "missing_Ï€_😀_" + uuid.uuid4().hex + return expect_rpc_error( + lambda: session.collect(sql(f"SELECT * FROM `{marker}`")), + {grpc.StatusCode.INVALID_ARGUMENT}, contains=marker, + ) + + check("raw_unicode_jvm_error", unicode_error) + + def release_cancels_and_session_recovers(): + session = raw_session() + operation_id, stream = session.execute( + sql("SELECT SUM(id % 13) AS total FROM range(1000000000000)") + ) + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as workers: + future = None + try: + header = next(stream) + session.observe(header) + assert header.HasField("schema") and header.operation_id == operation_id, header + + def consume(): + return expect_rpc_error(lambda: list(stream), {grpc.StatusCode.CANCELLED}) + + future = workers.submit(consume) + time.sleep(0.3) + assert not future.done(), "long query ended before cancellation" + started = time.monotonic() + released = session.stub.ReleaseExecute( + base.ReleaseExecuteRequest( + **session.identity(), operation_id=operation_id, + release_all=base.ReleaseExecuteRequest.ReleaseAll(), + ), timeout=args.cancel_timeout, + ) + session.observe(released) + assert released.operation_id == operation_id, released + cancelled = future.result(timeout=args.cancel_timeout) + cancel_seconds = time.monotonic() - started + finally: + stream.cancel() + assert session.collect(sql("SELECT 7 AS value")) == [{"value": 7}] + return {"cancel_seconds": round(cancel_seconds, 3), "rpc": cancelled, + "same_session_recovered": True} + + check("raw_release_all_cancellation_and_same_session_recovery", release_cancels_and_session_recovers) + + range_rows = 1_000_000 + + def partially_consumed_range(session): + import pyarrow as pa + + session.config("set", base.ConfigRequest.Set(pairs=[base.KeyValue( + key="spark.sql.execution.arrow.maxRecordsPerBatch", value="256" + )])) + operation_id, stream = session.execute(relations.Relation( + range=relations.Range(start=0, end=range_rows, step=1, num_partitions=64) + )) + try: + for response in stream: + session.observe(response) + assert response.operation_id == operation_id, response + assert not response.HasField("result_complete"), "range completed before data" + if response.HasField("arrow_batch"): + table = pa.ipc.open_stream(response.arrow_batch.data).read_all() + assert 0 < table.num_rows <= 256, table.num_rows + assert table.num_rows == response.arrow_batch.row_count, response + assert table.schema.equals(pa.schema([ + pa.field("id", pa.int64(), nullable=False) + ]), check_metadata=False), table.schema + assert table.to_pylist() == [ + {"id": value} for value in range(table.num_rows) + ], table + return operation_id, stream, table.num_rows + raise AssertionError("range ended before its first Arrow batch") + except BaseException: + stream.cancel() + raise + + def drain_interrupted_stream(session, operation_id, stream, initial_rows): + def consume(): + rows = initial_rows + complete = False + try: + for response in stream: + session.observe(response) + assert response.operation_id == operation_id, response + if response.HasField("arrow_batch"): + assert response.arrow_batch.row_count > 0, response + rows += response.arrow_batch.row_count + assert rows <= range_rows, rows + if response.HasField("result_complete"): + complete = True + except grpc.RpcError as error: + assert error.code() == grpc.StatusCode.CANCELLED, ( + error.code(), error.details() + ) + return {"terminal": error.code().name, "rows_observed": rows} + # The transport can already have buffered the complete result when interrupted. + assert complete and rows == range_rows, (complete, rows) + return {"terminal": "result_complete", "rows_observed": rows} + + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as workers: + future = workers.submit(consume) + try: + return future.result(timeout=args.cancel_timeout) + finally: + stream.cancel() + + def release_partially_consumed_session(): + session = raw_session() + operation_id, stream, first_rows = partially_consumed_range(session) + started = time.monotonic() + try: + session.release(timeout=args.cancel_timeout) + terminal = drain_interrupted_stream(session, operation_id, stream, first_rows) + release_seconds = time.monotonic() - started + finally: + stream.cancel() + released = expect_rpc_error( + lambda: session.collect(sql("SELECT 1 AS value")), + {grpc.StatusCode.FAILED_PRECONDITION}, contains="released", + ) + recovery = raw_session() + assert recovery.collect(sql("SELECT 11 AS value")) == [{"value": 11}] + return {"first_batch_rows": first_rows, "range_rows": range_rows, + "release_seconds": round(release_seconds, 3), + "stream": terminal, "released_session": released, + "fresh_session_recovered": True} + + check("raw_release_session_after_partial_arrow_stream", release_partially_consumed_session) + + def channel_close_cancels_partial_stream(): + session = raw_session() + operation_id, stream, first_rows = partially_consumed_range(session) + previous_channel = session.channel + address = args.remote.removeprefix("sc://").split("/", 1)[0] + # Keep session cleanup and the recovery probe usable after closing the active channel. + session.channel = grpc.insecure_channel(address) + session.stub = rpc.SparkConnectServiceStub(session.channel) + started = time.monotonic() + try: + previous_channel.close() + terminal = drain_interrupted_stream(session, operation_id, stream, first_rows) + cancel_seconds = time.monotonic() - started + finally: + stream.cancel() + previous_channel.close() + assert session.collect(sql("SELECT 13 AS value")) == [{"value": 13}] + return {"first_batch_rows": first_rows, "range_rows": range_rows, + "cancel_seconds": round(cancel_seconds, 3), + "stream": terminal, "same_session_recovered_on_new_channel": True} + + check("raw_channel_close_after_partial_arrow_stream", channel_close_cancels_partial_stream) + + def released_session_cannot_reappear(): + session = raw_session() + payload = base.ConfigRequest.Get(keys=["spark.sql.session.timeZone"]) + session.config("get", payload) + server_id = session.server_id + session.release() + config = expect_rpc_error( + lambda: session.config("get", payload), {grpc.StatusCode.FAILED_PRECONDITION}, + contains="released", + ) + execute = expect_rpc_error( + lambda: session.collect(sql("SELECT 1")), {grpc.StatusCode.FAILED_PRECONDITION}, + contains="released", + ) + return {"server_side_session_id": server_id, "config": config, "execute": execute} + + check("raw_release_session_tombstone", released_session_cannot_reappear) + except Exception as error: + checks.append({"name": "client_setup", "passed": False, "error": repr(error), + "traceback": traceback.format_exc()}) + finally: + for index, session in enumerate(reversed(raw_sessions)): + check("raw_session_cleanup_" + str(index), session.close) + if spark is not None: + check("official_session_cleanup", spark.stop) + + report = {"schema_version": 1, "profile": "jvm-engine", "remote": args.remote, + "client_versions": {"pyspark": pyspark.__version__, "grpc": grpc.__version__}, + "checks": checks, "passed": bool(checks) and all(item["passed"] for item in checks)} + serialized = json.dumps(report, indent=2, ensure_ascii=False) + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(serialized + "\n") + print(serialized, flush=True) + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/crates/spark-connect-server/jvm/native-licenses/LICENSE-binary b/crates/spark-connect-server/jvm/native-licenses/LICENSE-binary new file mode 100644 index 00000000..ab9045c9 --- /dev/null +++ b/crates/spark-connect-server/jvm/native-licenses/LICENSE-binary @@ -0,0 +1,1805 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------------------------------------------------------ +This project bundles some components that are also licensed under the Apache +License Version 2.0. Each is listed with its license expression. The components +whose expression offers Apache-2.0 as an option are used under Apache-2.0. + +ahash 0.8.12 (MIT OR Apache-2.0) +arrow 59.3.2 (Apache-2.0) +arrow-arith 59.3.2 (Apache-2.0) +arrow-buffer 59.3.2 (Apache-2.0) +arrow-cast 59.3.2 (Apache-2.0) +arrow-data 59.3.2 (Apache-2.0) +arrow-ipc 59.3.2 (Apache-2.0) +arrow-ord 59.3.2 (Apache-2.0) +arrow-row 59.3.2 (Apache-2.0) +arrow-schema 59.3.2 (Apache-2.0) +arrow-select 59.3.2 (Apache-2.0) +arrow-string 59.3.2 (Apache-2.0) +atomic-waker 1.1.2 (Apache-2.0 OR MIT) +base64 0.22.1 (MIT OR Apache-2.0) +base64 0.23.1 (MIT OR Apache-2.0) +bitflags 2.13.0 (MIT OR Apache-2.0) +cfg-if 1.0.4 (MIT OR Apache-2.0) +chrono 0.4.45 (MIT OR Apache-2.0) +core-foundation-sys 0.8.7 (MIT OR Apache-2.0) +equivalent 1.0.2 (Apache-2.0 OR MIT) +flatbuffers 25.12.19 (Apache-2.0) +fnv 1.0.7 (Apache-2.0 / MIT) +futures 0.3.32 (MIT OR Apache-2.0) +futures-channel 0.3.32 (MIT OR Apache-2.0) +futures-core 0.3.32 (MIT OR Apache-2.0) +futures-executor 0.3.32 (MIT OR Apache-2.0) +futures-io 0.3.32 (MIT OR Apache-2.0) +futures-sink 0.3.32 (MIT OR Apache-2.0) +futures-task 0.3.32 (MIT OR Apache-2.0) +futures-util 0.3.32 (MIT OR Apache-2.0) +getrandom 0.3.4 (MIT OR Apache-2.0) +getrandom 0.4.2 (MIT OR Apache-2.0) +half 2.7.1 (MIT OR Apache-2.0) +hashbrown 0.17.1 (MIT OR Apache-2.0) +http 1.4.2 (MIT OR Apache-2.0) +httparse 1.10.1 (MIT OR Apache-2.0) +httpdate 1.0.3 (MIT OR Apache-2.0) +hyper-timeout 0.5.2 (MIT OR Apache-2.0) +iana-time-zone 0.1.65 (MIT OR Apache-2.0) +indexmap 2.14.0 (Apache-2.0 OR MIT) +itoa 1.0.18 (MIT OR Apache-2.0) +lexical-core 1.0.6 (MIT/Apache-2.0) +lexical-parse-float 1.0.6 (MIT/Apache-2.0) +lexical-parse-integer 1.0.6 (MIT/Apache-2.0) +lexical-util 1.0.7 (MIT/Apache-2.0) +lexical-write-float 1.0.6 (MIT/Apache-2.0) +lexical-write-integer 1.0.6 (MIT/Apache-2.0) +libc 0.2.186 (MIT OR Apache-2.0) +mime 0.3.17 (MIT OR Apache-2.0) +num-bigint 0.5.1 (MIT OR Apache-2.0) +num-complex 0.4.6 (MIT OR Apache-2.0) +num-integer 0.1.46 (MIT OR Apache-2.0) +num-traits 0.2.19 (MIT OR Apache-2.0) +once_cell 1.21.4 (MIT OR Apache-2.0) +percent-encoding 2.3.2 (MIT OR Apache-2.0) +pin-project 1.1.13 (Apache-2.0 OR MIT) +pin-project-lite 0.2.17 (Apache-2.0 OR MIT) +prost 0.14.4 (Apache-2.0) +prost-types 0.14.4 (Apache-2.0) +regex 1.12.4 (MIT OR Apache-2.0) +regex-automata 0.4.14 (MIT OR Apache-2.0) +regex-syntax 0.8.11 (MIT OR Apache-2.0) +ryu 1.0.23 (Apache-2.0 OR BSL-1.0) +serde_core 1.0.228 (MIT OR Apache-2.0) +smallvec 1.15.1 (MIT OR Apache-2.0) +socket2 0.6.4 (MIT OR Apache-2.0) +sync_wrapper 1.0.2 (Apache-2.0) +uuid 1.23.3 (Apache-2.0 OR MIT) +zerocopy 0.8.52 (BSD-2-Clause OR Apache-2.0 OR MIT) +zstd-safe 7.2.4 (MIT OR Apache-2.0) + +------------------------------------------------------------------------------------ +This product bundles various third-party components under other open source +licenses. This section summarizes those components and their licenses. Their +license texts follow the summary. + +Apache-2.0 AND MIT +------------------ +arrow-array 59.3.2 + +BSD-3-Clause AND BSD-2-Clause +----------------------------- +zstd-sys 2.1.3+zstd.1.5.7 + +MIT +--- +atoi 2.0.0 +axum 0.8.9 +axum-core 0.5.6 +bytes 1.11.1 +h2 0.4.14 +http-body 1.0.1 +http-body-util 0.1.3 +hyper 1.10.1 +hyper-util 0.1.20 +libm 0.2.16 +lz4_flex 0.14.0 +mio 1.2.1 +slab 0.4.12 +tokio 1.52.3 +tokio-stream 0.1.18 +tokio-util 0.7.18 +tonic 0.14.6 +tonic-prost 0.14.6 +tower 0.5.3 +tower-layer 0.3.3 +tower-service 0.3.3 +tracing 0.1.44 +tracing-core 0.1.36 +try-lock 0.2.5 +twox-hash 2.1.2 +want 0.3.1 +zstd 0.13.3 + +MIT AND BSD-3-Clause +-------------------- +matchit 0.8.4 + +Unlicense OR MIT +---------------- +aho-corasick 1.1.4 +memchr 2.8.1 + +------------------------------------------------------------------------------------ +The license of aho-corasick 1.1.4 (Unlicense OR MIT), from COPYING: + +This project is dual-licensed under the Unlicense and MIT licenses. + +You may use this code under the terms of either license. + +------------------------------------------------------------------------------------ +The license of aho-corasick 1.1.4 (Unlicense OR MIT), from LICENSE-MIT: + +The MIT License (MIT) + +Copyright (c) 2015 Andrew Gallant + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of aho-corasick 1.1.4 (Unlicense OR MIT), from UNLICENSE: + +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to + +------------------------------------------------------------------------------------ +The license of arrow-array 59.3.2 (Apache-2.0 AND MIT), from LICENSE-MIT: + +MIT License + +Copyright (c) 2020-2022 Oliver Margetts + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +------------------------------------------------------------------------------------ +The license of atoi 2.0.0 (MIT), from LICENSE: + +MIT License + +Copyright (c) 2017 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +------------------------------------------------------------------------------------ +The license of axum 0.8.9 (MIT), from LICENSE: + +Copyright (c) 2019 axum Contributors + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of axum-core 0.5.6 (MIT), from LICENSE: + +MIT License + +Copyright (c) 2019–2025 axum Contributors + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of bytes 1.11.1 (MIT), from LICENSE: + +Copyright (c) 2018 Carl Lerche + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of h2 0.4.14 (MIT), from LICENSE: + +Copyright (c) 2017 h2 authors + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of http-body 1.0.1 (MIT), from LICENSE: + +Copyright (c) 2019-2024 Sean McArthur & Hyper Contributors + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of http-body-util 0.1.3 (MIT), from LICENSE: + +Copyright (c) 2019-2025 Sean McArthur & Hyper Contributors + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of hyper 1.10.1 (MIT), from LICENSE: + +Copyright (c) 2014-2026 Sean McArthur + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of hyper-util 0.1.20 (MIT), from LICENSE: + +Copyright (c) 2023-2025 Sean McArthur + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of libm 0.2.16 (MIT), from LICENSE.txt: + +rust-lang/libm as a whole is available for use under the MIT license: + +------------------------------------------------------------------------------ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +------------------------------------------------------------------------------ + +As a contributor, you agree that your code can be used under either the MIT +license or the Apache-2.0 license: + +------------------------------------------------------------------------------ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +------------------------------------------------------------------------------ + +This Rust library contains the following copyrights: + + Copyright (c) 2018 Jorge Aparicio + +Portions of this software are derived from third-party works licensed under +terms compatible with the above MIT license: + +* musl libc https://www.musl-libc.org/. This library contains the following + copyright: + + Copyright © 2005-2020 Rich Felker, et al. + +* The CORE-MATH project https://core-math.gitlabpages.inria.fr/. CORE-MATH + routines are available under the MIT license on a per-file basis. + +The musl libc COPYRIGHT file also includes the following notice relevant to +math portions of the library: + +------------------------------------------------------------------------------ +Much of the math library code (src/math/* and src/complex/*) is +Copyright © 1993,2004 Sun Microsystems or +Copyright © 2003-2011 David Schultz or +Copyright © 2003-2009 Steven G. Kargl or +Copyright © 2003-2009 Bruce D. Evans or +Copyright © 2008 Stephen L. Moshier or +Copyright © 2017-2018 Arm Limited +and labelled as such in comments in the individual source files. All +have been licensed under extremely permissive terms. +------------------------------------------------------------------------------ + +Copyright notices are retained in src/* files where relevant. + +------------------------------------------------------------------------------------ +The license of lz4_flex 0.14.0 (MIT), from LICENSE: + +The MIT License (MIT) + +Copyright (c) 2020 Pascal Seitz + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of matchit 0.8.4 (MIT AND BSD-3-Clause), from LICENSE: + +MIT License + +Copyright (c) 2022 Ibraheem Ahmed + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +------------------------------------------------------------------------------------ +The license of matchit 0.8.4 (MIT AND BSD-3-Clause), from LICENSE.httprouter: + +BSD 3-Clause License + +Copyright (c) 2013, Julien Schmidt +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------------------------------------------------------------------------------------ +The license of memchr 2.8.1 (Unlicense OR MIT), from COPYING: + +This project is dual-licensed under the Unlicense and MIT licenses. + +You may use this code under the terms of either license. + +------------------------------------------------------------------------------------ +The license of memchr 2.8.1 (Unlicense OR MIT), from LICENSE-MIT: + +The MIT License (MIT) + +Copyright (c) 2015 Andrew Gallant + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of memchr 2.8.1 (Unlicense OR MIT), from UNLICENSE: + +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to + +------------------------------------------------------------------------------------ +The license of mio 1.2.1 (MIT), from LICENSE: + +Copyright (c) 2014 Carl Lerche and other MIO contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of slab 0.4.12 (MIT), from LICENSE: + +Copyright (c) 2019 Carl Lerche + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of tokio 1.52.3 (MIT), from LICENSE: + +MIT License + +Copyright (c) Tokio Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +------------------------------------------------------------------------------------ +The license of tokio-stream 0.1.18 (MIT), from LICENSE: + +MIT License + +Copyright (c) Tokio Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +------------------------------------------------------------------------------------ +The license of tokio-util 0.7.18 (MIT), from LICENSE: + +MIT License + +Copyright (c) Tokio Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +------------------------------------------------------------------------------------ +The license of tonic 0.14.6 (MIT), from LICENSE: + +Copyright (c) 2025 Lucio Franco + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of tonic-prost 0.14.6 (MIT), from LICENSE: + +Copyright (c) 2025 Lucio Franco + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of tower 0.5.3 (MIT), from LICENSE: + +Copyright (c) 2019 Tower Contributors + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of tower-layer 0.3.3 (MIT), from LICENSE: + +Copyright (c) 2019 Tower Contributors + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of tower-service 0.3.3 (MIT), from LICENSE: + +Copyright (c) 2019 Tower Contributors + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of tracing 0.1.44 (MIT), from LICENSE: + +Copyright (c) 2019 Tokio Contributors + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of tracing-core 0.1.36 (MIT), from LICENSE: + +Copyright (c) 2019 Tokio Contributors + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of try-lock 0.2.5 (MIT), from LICENSE: + +Copyright (c) 2018-2023 Sean McArthur +Copyright (c) 2016 Alex Crichton + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of twox-hash 2.1.2 (MIT), from LICENSE.txt: + +The MIT License (MIT) + +Copyright (c) 2015 Jake Goulding + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +------------------------------------------------------------------------------------ +The license of want 0.3.1 (MIT), from LICENSE: + +Copyright (c) 2018-2019 Sean McArthur + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of zstd 0.13.3 (MIT), from LICENSE: + +The MIT License (MIT) +Copyright (c) 2016 Alexandre Bury + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +------------------------------------------------------------------------------------ +The license of zstd-sys 2.1.3+zstd.1.5.7 (BSD-3-Clause AND BSD-2-Clause), from LICENSE: + +BSD 3-Clause License + +Copyright (c) 2026, Alexandre Bury + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------------------------------------------------------------------------------------ +The license of zstd-sys 2.1.3+zstd.1.5.7 (BSD-3-Clause AND BSD-2-Clause), from LICENSE.BSD-2-Clause: + +The decoders for legacy formats, in zstd/lib/legacy/, include code under the +2-clause BSD license: + +BSD 2-Clause License + +Copyright (C) 2013-2016, Yann Collet. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------------------------------------------------------------------------------------ +The license of zstd-sys 2.1.3+zstd.1.5.7 (BSD-3-Clause AND BSD-2-Clause), from LICENSE.BSD-3-Clause: + +The auto-generated bindings are under the 3-clause BSD license: + +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------------------------------------------------------------------------------------ +The notice in the source of futures-channel 0.3.32 (MIT OR Apache-2.0), in src/mpsc/queue.rs: + +/* Copyright (c) 2010-2011 Dmitry Vyukov. All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY DMITRY VYUKOV "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL DMITRY VYUKOV OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of Dmitry Vyukov. + */ + +------------------------------------------------------------------------------------ +The notice in the source of lexical-parse-float 1.0.6 (MIT/Apache-2.0), in src/libm.rs: + +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +------------------------------------------------------------------------------------ +The notice in the source of lexical-util 1.0.7 (MIT/Apache-2.0), in src/libm.rs: + +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunSoft, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +------------------------------------------------------------------------------------ +The notice in the source of libm 0.2.16 (MIT), in src/math/acos.rs, src/math/asin.rs, src/math/atan2.rs, src/math/j0.rs, + src/math/j1.rs, src/math/jn.rs, src/math/k_cos.rs, src/math/k_sin.rs, + src/math/lgamma_r.rs, src/math/log.rs, src/math/log10.rs, + src/math/log2.rs, src/math/rem_pio2_large.rs: + +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunSoft, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +------------------------------------------------------------------------------------ +The notice in the source of libm 0.2.16 (MIT), in src/math/acosf.rs, src/math/asinf.rs, src/math/atan.rs, src/math/atan2f.rs, + src/math/atanf.rs, src/math/cbrtf.rs, src/math/cos.rs, src/math/cosf.rs, + src/math/erf.rs, src/math/erff.rs, src/math/expf.rs, src/math/expm1.rs, + src/math/expm1f.rs, src/math/j0f.rs, src/math/j1f.rs, src/math/jnf.rs, + src/math/k_cosf.rs, src/math/k_sinf.rs, src/math/lgammaf_r.rs, + src/math/log10f.rs, src/math/log1p.rs, src/math/log1pf.rs, + src/math/log2f.rs, src/math/logf.rs, src/math/powf.rs, + src/math/rem_pio2f.rs, src/math/sin.rs, src/math/sincos.rs, + src/math/sincosf.rs, src/math/sinf.rs, src/math/tan.rs, src/math/tanf.rs: + +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +------------------------------------------------------------------------------------ +The notice in the source of libm 0.2.16 (MIT), in src/math/cbrt.rs: + +/* origin: core-math/src/binary64/cbrt/cbrt.c + * Copyright (c) 2021-2022 Alexei Sibidanov. + * Ported to Rust in 2025 by Trevor Gross. + */ + +------------------------------------------------------------------------------------ +The notice in the source of libm 0.2.16 (MIT), in src/math/exp.rs, src/math/pow.rs: + +/* + * ==================================================== + * Copyright (C) 2004 by Sun Microsystems, Inc. All rights reserved. + * + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +------------------------------------------------------------------------------------ +The notice in the source of libm 0.2.16 (MIT), in src/math/exp2.rs, src/math/exp2f.rs: + +// origin: FreeBSD /usr/src/lib/msun/src/s_exp2.c */ +//- +// Copyright (c) 2005 David Schultz +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +// OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +// SUCH DAMAGE. + +------------------------------------------------------------------------------------ +The notice in the source of libm 0.2.16 (MIT), in src/math/k_tan.rs, src/math/k_tanf.rs: + +// origin: FreeBSD /usr/src/lib/msun/src/k_tan.c */ +// +// ==================================================== +// Copyright 2004 Sun Microsystems, Inc. All Rights Reserved. +// +// Permission to use, copy, modify, and distribute this +// software is freely granted, provided that this notice +// is preserved. +// ==================================================== + +------------------------------------------------------------------------------------ +The notice in the source of libm 0.2.16 (MIT), in src/math/rem_pio2.rs: + +// origin: FreeBSD /usr/src/lib/msun/src/e_rem_pio2.c +// +// ==================================================== +// Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. +// +// Developed at SunPro, a Sun Microsystems, Inc. business. +// Permission to use, copy, modify, and distribute this +// software is freely granted, provided that this notice +// is preserved. +// ==================================================== +// +// Optimized by Bruce D. Evans. */ + +------------------------------------------------------------------------------------ +The notice in the source of zstd-sys 2.1.3+zstd.1.5.7 (BSD-3-Clause AND BSD-2-Clause), in zstd/lib/common/threading.c: + +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +------------------------------------------------------------------------------------ +The notice in the source of zstd-sys 2.1.3+zstd.1.5.7 (BSD-3-Clause AND BSD-2-Clause), in zstd/lib/dictBuilder/divsufsort.c: + +/* + * divsufsort.c for libdivsufsort-lite + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +------------------------------------------------------------------------------------ +The notice in the source of zstd-sys 2.1.3+zstd.1.5.7 (BSD-3-Clause AND BSD-2-Clause), in zstd/lib/legacy/zstd_v01.c, zstd/lib/legacy/zstd_v02.c, + zstd/lib/legacy/zstd_v03.c, zstd/lib/legacy/zstd_v04.c, + zstd/lib/legacy/zstd_v05.c, zstd/lib/legacy/zstd_v06.c, + zstd/lib/legacy/zstd_v07.c: + +/* + * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ diff --git a/crates/spark-connect-server/jvm/native-licenses/NOTICE-binary b/crates/spark-connect-server/jvm/native-licenses/NOTICE-binary new file mode 100644 index 00000000..16b06af7 --- /dev/null +++ b/crates/spark-connect-server/jvm/native-licenses/NOTICE-binary @@ -0,0 +1,26 @@ +Apache Spark Connect Rust Client +Copyright 2023 and onwards The Apache Software Foundation. + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +------------------------------------------------------------------------------------ +This product bundles the following components licensed under the Apache License +2.0, whose NOTICE files require the notices below. Each notice is reproduced +as its component ships it. + +=== NOTICE FOR arrow 59.3.2, arrow-arith 59.3.2, arrow-array 59.3.2, arrow-buffer 59.3.2, arrow-cast 59.3.2, arrow-data 59.3.2, arrow-ipc 59.3.2, arrow-ord 59.3.2, arrow-row 59.3.2, arrow-schema 59.3.2, arrow-select 59.3.2, arrow-string 59.3.2 === +Apache Arrow +Copyright 2016-2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This product includes software from the chronoutil crate (MIT) + * Copyright (c) 2020-2022 Oliver Margetts + * https://github.com/olliemath/chronoutil + +This product includes software from the compact-thrift project (Apache 2.0) + * Copyright Jörn Horstmann + * https://github.com/jhorstmann/compact-thrift +=== END OF NOTICE FOR arrow 59.3.2, arrow-arith 59.3.2, arrow-array 59.3.2, arrow-buffer 59.3.2, arrow-cast 59.3.2, arrow-data 59.3.2, arrow-ipc 59.3.2, arrow-ord 59.3.2, arrow-row 59.3.2, arrow-schema 59.3.2, arrow-select 59.3.2, arrow-string 59.3.2 === diff --git a/crates/spark-connect-server/jvm/native-licenses/dependencies.json b/crates/spark-connect-server/jvm/native-licenses/dependencies.json new file mode 100644 index 00000000..f7b0d4a0 --- /dev/null +++ b/crates/spark-connect-server/jvm/native-licenses/dependencies.json @@ -0,0 +1,2123 @@ +{ + "inputs": { + "Cargo.lock": "e35b527aa1e55e460424fafa0a9631a266c1e4c178cf4a7a17e20a136ed13dbe", + "Cargo.toml": "7643b23addd58bdbeecd687297526c3ddc8c3b85f0e019d9ef334bf8c0e3dbb6", + "LICENSE": "0d1dda3ddbb607c0ce754eddb23aa9b25b96f4f937b79b32605614c9ff1bfb96", + "NOTICE": "f34ebcb547d17c3d7aab2d024c4685d4b17d49182480727b70cad2eccfc35274", + "crates/spark-connect-proto/Cargo.toml": "5b43301089f8496478820fa2b063e544c2255da7cf8f1a33240ea915f2fe28ec", + "crates/spark-connect-server/Cargo.toml": "9ec4b61425a7dce2121c78381d5504e25ae38a1bfe887ada79400cb055e59fb9", + "crates/spark-connect-server/jni/Cargo.toml": "069fe443f8cd2b869644b9a22ec6b1a4ea53b7dd93e40b3e0dce5f57fa4087a1", + "scripts/gen_binary_licenses.py": "59e36ee5da2b4e5cf17ede555cc6854e268e7f11137da72a1eac98ebf26f0608", + "scripts/gen_native_binary_licenses.py": "018456a70b85ea09640d5410de56259ecb2906debd7617b99e1d79cd52ecbe0b" + }, + "outputs": { + "LICENSE-binary": "a4d3b9700aed77ea2962f4e8afbf06267f364162d9832d3cfa96ce5a00adda84", + "NOTICE-binary": "56b4d3334d28705f5764f12c9dd72f8ae141fde0ecdc95565998a223583cf737" + }, + "root_package": "spark-connect-jni", + "schema_version": 1, + "targets": { + "aarch64-apple-darwin": { + "registry": [ + [ + "ahash", + "0.8.12", + "MIT OR Apache-2.0" + ], + [ + "aho-corasick", + "1.1.4", + "Unlicense OR MIT" + ], + [ + "arrow", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-arith", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-array", + "59.3.2", + "Apache-2.0 AND MIT" + ], + [ + "arrow-buffer", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-cast", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-data", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-ipc", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-ord", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-row", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-schema", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-select", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-string", + "59.3.2", + "Apache-2.0" + ], + [ + "atoi", + "2.0.0", + "MIT" + ], + [ + "atomic-waker", + "1.1.2", + "Apache-2.0 OR MIT" + ], + [ + "axum", + "0.8.9", + "MIT" + ], + [ + "axum-core", + "0.5.6", + "MIT" + ], + [ + "base64", + "0.22.1", + "MIT OR Apache-2.0" + ], + [ + "base64", + "0.23.1", + "MIT OR Apache-2.0" + ], + [ + "bitflags", + "2.13.0", + "MIT OR Apache-2.0" + ], + [ + "bytes", + "1.11.1", + "MIT" + ], + [ + "cfg-if", + "1.0.4", + "MIT OR Apache-2.0" + ], + [ + "chrono", + "0.4.45", + "MIT OR Apache-2.0" + ], + [ + "core-foundation-sys", + "0.8.7", + "MIT OR Apache-2.0" + ], + [ + "equivalent", + "1.0.2", + "Apache-2.0 OR MIT" + ], + [ + "flatbuffers", + "25.12.19", + "Apache-2.0" + ], + [ + "fnv", + "1.0.7", + "Apache-2.0 / MIT" + ], + [ + "futures", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-channel", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-core", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-executor", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-io", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-sink", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-task", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-util", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "getrandom", + "0.3.4", + "MIT OR Apache-2.0" + ], + [ + "getrandom", + "0.4.2", + "MIT OR Apache-2.0" + ], + [ + "h2", + "0.4.14", + "MIT" + ], + [ + "half", + "2.7.1", + "MIT OR Apache-2.0" + ], + [ + "hashbrown", + "0.17.1", + "MIT OR Apache-2.0" + ], + [ + "http", + "1.4.2", + "MIT OR Apache-2.0" + ], + [ + "http-body", + "1.0.1", + "MIT" + ], + [ + "http-body-util", + "0.1.3", + "MIT" + ], + [ + "httparse", + "1.10.1", + "MIT OR Apache-2.0" + ], + [ + "httpdate", + "1.0.3", + "MIT OR Apache-2.0" + ], + [ + "hyper", + "1.10.1", + "MIT" + ], + [ + "hyper-timeout", + "0.5.2", + "MIT OR Apache-2.0" + ], + [ + "hyper-util", + "0.1.20", + "MIT" + ], + [ + "iana-time-zone", + "0.1.65", + "MIT OR Apache-2.0" + ], + [ + "indexmap", + "2.14.0", + "Apache-2.0 OR MIT" + ], + [ + "itoa", + "1.0.18", + "MIT OR Apache-2.0" + ], + [ + "lexical-core", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "lexical-parse-float", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "lexical-parse-integer", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "lexical-util", + "1.0.7", + "MIT/Apache-2.0" + ], + [ + "lexical-write-float", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "lexical-write-integer", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "libc", + "0.2.186", + "MIT OR Apache-2.0" + ], + [ + "libm", + "0.2.16", + "MIT" + ], + [ + "lz4_flex", + "0.14.0", + "MIT" + ], + [ + "matchit", + "0.8.4", + "MIT AND BSD-3-Clause" + ], + [ + "memchr", + "2.8.1", + "Unlicense OR MIT" + ], + [ + "mime", + "0.3.17", + "MIT OR Apache-2.0" + ], + [ + "mio", + "1.2.1", + "MIT" + ], + [ + "num-bigint", + "0.5.1", + "MIT OR Apache-2.0" + ], + [ + "num-complex", + "0.4.6", + "MIT OR Apache-2.0" + ], + [ + "num-integer", + "0.1.46", + "MIT OR Apache-2.0" + ], + [ + "num-traits", + "0.2.19", + "MIT OR Apache-2.0" + ], + [ + "once_cell", + "1.21.4", + "MIT OR Apache-2.0" + ], + [ + "percent-encoding", + "2.3.2", + "MIT OR Apache-2.0" + ], + [ + "pin-project", + "1.1.13", + "Apache-2.0 OR MIT" + ], + [ + "pin-project-lite", + "0.2.17", + "Apache-2.0 OR MIT" + ], + [ + "prost", + "0.14.4", + "Apache-2.0" + ], + [ + "prost-types", + "0.14.4", + "Apache-2.0" + ], + [ + "regex", + "1.12.4", + "MIT OR Apache-2.0" + ], + [ + "regex-automata", + "0.4.14", + "MIT OR Apache-2.0" + ], + [ + "regex-syntax", + "0.8.11", + "MIT OR Apache-2.0" + ], + [ + "ryu", + "1.0.23", + "Apache-2.0 OR BSL-1.0" + ], + [ + "serde_core", + "1.0.228", + "MIT OR Apache-2.0" + ], + [ + "slab", + "0.4.12", + "MIT" + ], + [ + "smallvec", + "1.15.1", + "MIT OR Apache-2.0" + ], + [ + "socket2", + "0.6.4", + "MIT OR Apache-2.0" + ], + [ + "sync_wrapper", + "1.0.2", + "Apache-2.0" + ], + [ + "tokio", + "1.52.3", + "MIT" + ], + [ + "tokio-stream", + "0.1.18", + "MIT" + ], + [ + "tokio-util", + "0.7.18", + "MIT" + ], + [ + "tonic", + "0.14.6", + "MIT" + ], + [ + "tonic-prost", + "0.14.6", + "MIT" + ], + [ + "tower", + "0.5.3", + "MIT" + ], + [ + "tower-layer", + "0.3.3", + "MIT" + ], + [ + "tower-service", + "0.3.3", + "MIT" + ], + [ + "tracing", + "0.1.44", + "MIT" + ], + [ + "tracing-core", + "0.1.36", + "MIT" + ], + [ + "try-lock", + "0.2.5", + "MIT" + ], + [ + "twox-hash", + "2.1.2", + "MIT" + ], + [ + "uuid", + "1.23.3", + "Apache-2.0 OR MIT" + ], + [ + "want", + "0.3.1", + "MIT" + ], + [ + "zerocopy", + "0.8.52", + "BSD-2-Clause OR Apache-2.0 OR MIT" + ], + [ + "zstd", + "0.13.3", + "MIT" + ], + [ + "zstd-safe", + "7.2.4", + "MIT OR Apache-2.0" + ], + [ + "zstd-sys", + "2.1.3+zstd.1.5.7", + "BSD-3-Clause AND BSD-2-Clause" + ] + ], + "workspace": [ + [ + "apache-spark-connect-proto", + "4.4.0" + ], + [ + "spark-connect-jni", + "0.1.0" + ], + [ + "spark-connect-server", + "0.1.0" + ] + ] + }, + "aarch64-unknown-linux-gnu": { + "registry": [ + [ + "ahash", + "0.8.12", + "MIT OR Apache-2.0" + ], + [ + "aho-corasick", + "1.1.4", + "Unlicense OR MIT" + ], + [ + "arrow", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-arith", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-array", + "59.3.2", + "Apache-2.0 AND MIT" + ], + [ + "arrow-buffer", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-cast", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-data", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-ipc", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-ord", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-row", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-schema", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-select", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-string", + "59.3.2", + "Apache-2.0" + ], + [ + "atoi", + "2.0.0", + "MIT" + ], + [ + "atomic-waker", + "1.1.2", + "Apache-2.0 OR MIT" + ], + [ + "axum", + "0.8.9", + "MIT" + ], + [ + "axum-core", + "0.5.6", + "MIT" + ], + [ + "base64", + "0.22.1", + "MIT OR Apache-2.0" + ], + [ + "base64", + "0.23.1", + "MIT OR Apache-2.0" + ], + [ + "bitflags", + "2.13.0", + "MIT OR Apache-2.0" + ], + [ + "bytes", + "1.11.1", + "MIT" + ], + [ + "cfg-if", + "1.0.4", + "MIT OR Apache-2.0" + ], + [ + "chrono", + "0.4.45", + "MIT OR Apache-2.0" + ], + [ + "equivalent", + "1.0.2", + "Apache-2.0 OR MIT" + ], + [ + "flatbuffers", + "25.12.19", + "Apache-2.0" + ], + [ + "fnv", + "1.0.7", + "Apache-2.0 / MIT" + ], + [ + "futures", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-channel", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-core", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-executor", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-io", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-sink", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-task", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-util", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "getrandom", + "0.3.4", + "MIT OR Apache-2.0" + ], + [ + "getrandom", + "0.4.2", + "MIT OR Apache-2.0" + ], + [ + "h2", + "0.4.14", + "MIT" + ], + [ + "half", + "2.7.1", + "MIT OR Apache-2.0" + ], + [ + "hashbrown", + "0.17.1", + "MIT OR Apache-2.0" + ], + [ + "http", + "1.4.2", + "MIT OR Apache-2.0" + ], + [ + "http-body", + "1.0.1", + "MIT" + ], + [ + "http-body-util", + "0.1.3", + "MIT" + ], + [ + "httparse", + "1.10.1", + "MIT OR Apache-2.0" + ], + [ + "httpdate", + "1.0.3", + "MIT OR Apache-2.0" + ], + [ + "hyper", + "1.10.1", + "MIT" + ], + [ + "hyper-timeout", + "0.5.2", + "MIT OR Apache-2.0" + ], + [ + "hyper-util", + "0.1.20", + "MIT" + ], + [ + "iana-time-zone", + "0.1.65", + "MIT OR Apache-2.0" + ], + [ + "indexmap", + "2.14.0", + "Apache-2.0 OR MIT" + ], + [ + "itoa", + "1.0.18", + "MIT OR Apache-2.0" + ], + [ + "lexical-core", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "lexical-parse-float", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "lexical-parse-integer", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "lexical-util", + "1.0.7", + "MIT/Apache-2.0" + ], + [ + "lexical-write-float", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "lexical-write-integer", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "libc", + "0.2.186", + "MIT OR Apache-2.0" + ], + [ + "libm", + "0.2.16", + "MIT" + ], + [ + "lz4_flex", + "0.14.0", + "MIT" + ], + [ + "matchit", + "0.8.4", + "MIT AND BSD-3-Clause" + ], + [ + "memchr", + "2.8.1", + "Unlicense OR MIT" + ], + [ + "mime", + "0.3.17", + "MIT OR Apache-2.0" + ], + [ + "mio", + "1.2.1", + "MIT" + ], + [ + "num-bigint", + "0.5.1", + "MIT OR Apache-2.0" + ], + [ + "num-complex", + "0.4.6", + "MIT OR Apache-2.0" + ], + [ + "num-integer", + "0.1.46", + "MIT OR Apache-2.0" + ], + [ + "num-traits", + "0.2.19", + "MIT OR Apache-2.0" + ], + [ + "once_cell", + "1.21.4", + "MIT OR Apache-2.0" + ], + [ + "percent-encoding", + "2.3.2", + "MIT OR Apache-2.0" + ], + [ + "pin-project", + "1.1.13", + "Apache-2.0 OR MIT" + ], + [ + "pin-project-lite", + "0.2.17", + "Apache-2.0 OR MIT" + ], + [ + "prost", + "0.14.4", + "Apache-2.0" + ], + [ + "prost-types", + "0.14.4", + "Apache-2.0" + ], + [ + "regex", + "1.12.4", + "MIT OR Apache-2.0" + ], + [ + "regex-automata", + "0.4.14", + "MIT OR Apache-2.0" + ], + [ + "regex-syntax", + "0.8.11", + "MIT OR Apache-2.0" + ], + [ + "ryu", + "1.0.23", + "Apache-2.0 OR BSL-1.0" + ], + [ + "serde_core", + "1.0.228", + "MIT OR Apache-2.0" + ], + [ + "slab", + "0.4.12", + "MIT" + ], + [ + "smallvec", + "1.15.1", + "MIT OR Apache-2.0" + ], + [ + "socket2", + "0.6.4", + "MIT OR Apache-2.0" + ], + [ + "sync_wrapper", + "1.0.2", + "Apache-2.0" + ], + [ + "tokio", + "1.52.3", + "MIT" + ], + [ + "tokio-stream", + "0.1.18", + "MIT" + ], + [ + "tokio-util", + "0.7.18", + "MIT" + ], + [ + "tonic", + "0.14.6", + "MIT" + ], + [ + "tonic-prost", + "0.14.6", + "MIT" + ], + [ + "tower", + "0.5.3", + "MIT" + ], + [ + "tower-layer", + "0.3.3", + "MIT" + ], + [ + "tower-service", + "0.3.3", + "MIT" + ], + [ + "tracing", + "0.1.44", + "MIT" + ], + [ + "tracing-core", + "0.1.36", + "MIT" + ], + [ + "try-lock", + "0.2.5", + "MIT" + ], + [ + "twox-hash", + "2.1.2", + "MIT" + ], + [ + "uuid", + "1.23.3", + "Apache-2.0 OR MIT" + ], + [ + "want", + "0.3.1", + "MIT" + ], + [ + "zerocopy", + "0.8.52", + "BSD-2-Clause OR Apache-2.0 OR MIT" + ], + [ + "zstd", + "0.13.3", + "MIT" + ], + [ + "zstd-safe", + "7.2.4", + "MIT OR Apache-2.0" + ], + [ + "zstd-sys", + "2.1.3+zstd.1.5.7", + "BSD-3-Clause AND BSD-2-Clause" + ] + ], + "workspace": [ + [ + "apache-spark-connect-proto", + "4.4.0" + ], + [ + "spark-connect-jni", + "0.1.0" + ], + [ + "spark-connect-server", + "0.1.0" + ] + ] + }, + "x86_64-apple-darwin": { + "registry": [ + [ + "ahash", + "0.8.12", + "MIT OR Apache-2.0" + ], + [ + "aho-corasick", + "1.1.4", + "Unlicense OR MIT" + ], + [ + "arrow", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-arith", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-array", + "59.3.2", + "Apache-2.0 AND MIT" + ], + [ + "arrow-buffer", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-cast", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-data", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-ipc", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-ord", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-row", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-schema", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-select", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-string", + "59.3.2", + "Apache-2.0" + ], + [ + "atoi", + "2.0.0", + "MIT" + ], + [ + "atomic-waker", + "1.1.2", + "Apache-2.0 OR MIT" + ], + [ + "axum", + "0.8.9", + "MIT" + ], + [ + "axum-core", + "0.5.6", + "MIT" + ], + [ + "base64", + "0.22.1", + "MIT OR Apache-2.0" + ], + [ + "base64", + "0.23.1", + "MIT OR Apache-2.0" + ], + [ + "bitflags", + "2.13.0", + "MIT OR Apache-2.0" + ], + [ + "bytes", + "1.11.1", + "MIT" + ], + [ + "cfg-if", + "1.0.4", + "MIT OR Apache-2.0" + ], + [ + "chrono", + "0.4.45", + "MIT OR Apache-2.0" + ], + [ + "core-foundation-sys", + "0.8.7", + "MIT OR Apache-2.0" + ], + [ + "equivalent", + "1.0.2", + "Apache-2.0 OR MIT" + ], + [ + "flatbuffers", + "25.12.19", + "Apache-2.0" + ], + [ + "fnv", + "1.0.7", + "Apache-2.0 / MIT" + ], + [ + "futures", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-channel", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-core", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-executor", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-io", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-sink", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-task", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-util", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "getrandom", + "0.3.4", + "MIT OR Apache-2.0" + ], + [ + "getrandom", + "0.4.2", + "MIT OR Apache-2.0" + ], + [ + "h2", + "0.4.14", + "MIT" + ], + [ + "half", + "2.7.1", + "MIT OR Apache-2.0" + ], + [ + "hashbrown", + "0.17.1", + "MIT OR Apache-2.0" + ], + [ + "http", + "1.4.2", + "MIT OR Apache-2.0" + ], + [ + "http-body", + "1.0.1", + "MIT" + ], + [ + "http-body-util", + "0.1.3", + "MIT" + ], + [ + "httparse", + "1.10.1", + "MIT OR Apache-2.0" + ], + [ + "httpdate", + "1.0.3", + "MIT OR Apache-2.0" + ], + [ + "hyper", + "1.10.1", + "MIT" + ], + [ + "hyper-timeout", + "0.5.2", + "MIT OR Apache-2.0" + ], + [ + "hyper-util", + "0.1.20", + "MIT" + ], + [ + "iana-time-zone", + "0.1.65", + "MIT OR Apache-2.0" + ], + [ + "indexmap", + "2.14.0", + "Apache-2.0 OR MIT" + ], + [ + "itoa", + "1.0.18", + "MIT OR Apache-2.0" + ], + [ + "lexical-core", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "lexical-parse-float", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "lexical-parse-integer", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "lexical-util", + "1.0.7", + "MIT/Apache-2.0" + ], + [ + "lexical-write-float", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "lexical-write-integer", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "libc", + "0.2.186", + "MIT OR Apache-2.0" + ], + [ + "libm", + "0.2.16", + "MIT" + ], + [ + "lz4_flex", + "0.14.0", + "MIT" + ], + [ + "matchit", + "0.8.4", + "MIT AND BSD-3-Clause" + ], + [ + "memchr", + "2.8.1", + "Unlicense OR MIT" + ], + [ + "mime", + "0.3.17", + "MIT OR Apache-2.0" + ], + [ + "mio", + "1.2.1", + "MIT" + ], + [ + "num-bigint", + "0.5.1", + "MIT OR Apache-2.0" + ], + [ + "num-complex", + "0.4.6", + "MIT OR Apache-2.0" + ], + [ + "num-integer", + "0.1.46", + "MIT OR Apache-2.0" + ], + [ + "num-traits", + "0.2.19", + "MIT OR Apache-2.0" + ], + [ + "once_cell", + "1.21.4", + "MIT OR Apache-2.0" + ], + [ + "percent-encoding", + "2.3.2", + "MIT OR Apache-2.0" + ], + [ + "pin-project", + "1.1.13", + "Apache-2.0 OR MIT" + ], + [ + "pin-project-lite", + "0.2.17", + "Apache-2.0 OR MIT" + ], + [ + "prost", + "0.14.4", + "Apache-2.0" + ], + [ + "prost-types", + "0.14.4", + "Apache-2.0" + ], + [ + "regex", + "1.12.4", + "MIT OR Apache-2.0" + ], + [ + "regex-automata", + "0.4.14", + "MIT OR Apache-2.0" + ], + [ + "regex-syntax", + "0.8.11", + "MIT OR Apache-2.0" + ], + [ + "ryu", + "1.0.23", + "Apache-2.0 OR BSL-1.0" + ], + [ + "serde_core", + "1.0.228", + "MIT OR Apache-2.0" + ], + [ + "slab", + "0.4.12", + "MIT" + ], + [ + "smallvec", + "1.15.1", + "MIT OR Apache-2.0" + ], + [ + "socket2", + "0.6.4", + "MIT OR Apache-2.0" + ], + [ + "sync_wrapper", + "1.0.2", + "Apache-2.0" + ], + [ + "tokio", + "1.52.3", + "MIT" + ], + [ + "tokio-stream", + "0.1.18", + "MIT" + ], + [ + "tokio-util", + "0.7.18", + "MIT" + ], + [ + "tonic", + "0.14.6", + "MIT" + ], + [ + "tonic-prost", + "0.14.6", + "MIT" + ], + [ + "tower", + "0.5.3", + "MIT" + ], + [ + "tower-layer", + "0.3.3", + "MIT" + ], + [ + "tower-service", + "0.3.3", + "MIT" + ], + [ + "tracing", + "0.1.44", + "MIT" + ], + [ + "tracing-core", + "0.1.36", + "MIT" + ], + [ + "try-lock", + "0.2.5", + "MIT" + ], + [ + "twox-hash", + "2.1.2", + "MIT" + ], + [ + "uuid", + "1.23.3", + "Apache-2.0 OR MIT" + ], + [ + "want", + "0.3.1", + "MIT" + ], + [ + "zerocopy", + "0.8.52", + "BSD-2-Clause OR Apache-2.0 OR MIT" + ], + [ + "zstd", + "0.13.3", + "MIT" + ], + [ + "zstd-safe", + "7.2.4", + "MIT OR Apache-2.0" + ], + [ + "zstd-sys", + "2.1.3+zstd.1.5.7", + "BSD-3-Clause AND BSD-2-Clause" + ] + ], + "workspace": [ + [ + "apache-spark-connect-proto", + "4.4.0" + ], + [ + "spark-connect-jni", + "0.1.0" + ], + [ + "spark-connect-server", + "0.1.0" + ] + ] + }, + "x86_64-unknown-linux-gnu": { + "registry": [ + [ + "ahash", + "0.8.12", + "MIT OR Apache-2.0" + ], + [ + "aho-corasick", + "1.1.4", + "Unlicense OR MIT" + ], + [ + "arrow", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-arith", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-array", + "59.3.2", + "Apache-2.0 AND MIT" + ], + [ + "arrow-buffer", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-cast", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-data", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-ipc", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-ord", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-row", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-schema", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-select", + "59.3.2", + "Apache-2.0" + ], + [ + "arrow-string", + "59.3.2", + "Apache-2.0" + ], + [ + "atoi", + "2.0.0", + "MIT" + ], + [ + "atomic-waker", + "1.1.2", + "Apache-2.0 OR MIT" + ], + [ + "axum", + "0.8.9", + "MIT" + ], + [ + "axum-core", + "0.5.6", + "MIT" + ], + [ + "base64", + "0.22.1", + "MIT OR Apache-2.0" + ], + [ + "base64", + "0.23.1", + "MIT OR Apache-2.0" + ], + [ + "bitflags", + "2.13.0", + "MIT OR Apache-2.0" + ], + [ + "bytes", + "1.11.1", + "MIT" + ], + [ + "cfg-if", + "1.0.4", + "MIT OR Apache-2.0" + ], + [ + "chrono", + "0.4.45", + "MIT OR Apache-2.0" + ], + [ + "equivalent", + "1.0.2", + "Apache-2.0 OR MIT" + ], + [ + "flatbuffers", + "25.12.19", + "Apache-2.0" + ], + [ + "fnv", + "1.0.7", + "Apache-2.0 / MIT" + ], + [ + "futures", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-channel", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-core", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-executor", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-io", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-sink", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-task", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "futures-util", + "0.3.32", + "MIT OR Apache-2.0" + ], + [ + "getrandom", + "0.3.4", + "MIT OR Apache-2.0" + ], + [ + "getrandom", + "0.4.2", + "MIT OR Apache-2.0" + ], + [ + "h2", + "0.4.14", + "MIT" + ], + [ + "half", + "2.7.1", + "MIT OR Apache-2.0" + ], + [ + "hashbrown", + "0.17.1", + "MIT OR Apache-2.0" + ], + [ + "http", + "1.4.2", + "MIT OR Apache-2.0" + ], + [ + "http-body", + "1.0.1", + "MIT" + ], + [ + "http-body-util", + "0.1.3", + "MIT" + ], + [ + "httparse", + "1.10.1", + "MIT OR Apache-2.0" + ], + [ + "httpdate", + "1.0.3", + "MIT OR Apache-2.0" + ], + [ + "hyper", + "1.10.1", + "MIT" + ], + [ + "hyper-timeout", + "0.5.2", + "MIT OR Apache-2.0" + ], + [ + "hyper-util", + "0.1.20", + "MIT" + ], + [ + "iana-time-zone", + "0.1.65", + "MIT OR Apache-2.0" + ], + [ + "indexmap", + "2.14.0", + "Apache-2.0 OR MIT" + ], + [ + "itoa", + "1.0.18", + "MIT OR Apache-2.0" + ], + [ + "lexical-core", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "lexical-parse-float", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "lexical-parse-integer", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "lexical-util", + "1.0.7", + "MIT/Apache-2.0" + ], + [ + "lexical-write-float", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "lexical-write-integer", + "1.0.6", + "MIT/Apache-2.0" + ], + [ + "libc", + "0.2.186", + "MIT OR Apache-2.0" + ], + [ + "libm", + "0.2.16", + "MIT" + ], + [ + "lz4_flex", + "0.14.0", + "MIT" + ], + [ + "matchit", + "0.8.4", + "MIT AND BSD-3-Clause" + ], + [ + "memchr", + "2.8.1", + "Unlicense OR MIT" + ], + [ + "mime", + "0.3.17", + "MIT OR Apache-2.0" + ], + [ + "mio", + "1.2.1", + "MIT" + ], + [ + "num-bigint", + "0.5.1", + "MIT OR Apache-2.0" + ], + [ + "num-complex", + "0.4.6", + "MIT OR Apache-2.0" + ], + [ + "num-integer", + "0.1.46", + "MIT OR Apache-2.0" + ], + [ + "num-traits", + "0.2.19", + "MIT OR Apache-2.0" + ], + [ + "once_cell", + "1.21.4", + "MIT OR Apache-2.0" + ], + [ + "percent-encoding", + "2.3.2", + "MIT OR Apache-2.0" + ], + [ + "pin-project", + "1.1.13", + "Apache-2.0 OR MIT" + ], + [ + "pin-project-lite", + "0.2.17", + "Apache-2.0 OR MIT" + ], + [ + "prost", + "0.14.4", + "Apache-2.0" + ], + [ + "prost-types", + "0.14.4", + "Apache-2.0" + ], + [ + "regex", + "1.12.4", + "MIT OR Apache-2.0" + ], + [ + "regex-automata", + "0.4.14", + "MIT OR Apache-2.0" + ], + [ + "regex-syntax", + "0.8.11", + "MIT OR Apache-2.0" + ], + [ + "ryu", + "1.0.23", + "Apache-2.0 OR BSL-1.0" + ], + [ + "serde_core", + "1.0.228", + "MIT OR Apache-2.0" + ], + [ + "slab", + "0.4.12", + "MIT" + ], + [ + "smallvec", + "1.15.1", + "MIT OR Apache-2.0" + ], + [ + "socket2", + "0.6.4", + "MIT OR Apache-2.0" + ], + [ + "sync_wrapper", + "1.0.2", + "Apache-2.0" + ], + [ + "tokio", + "1.52.3", + "MIT" + ], + [ + "tokio-stream", + "0.1.18", + "MIT" + ], + [ + "tokio-util", + "0.7.18", + "MIT" + ], + [ + "tonic", + "0.14.6", + "MIT" + ], + [ + "tonic-prost", + "0.14.6", + "MIT" + ], + [ + "tower", + "0.5.3", + "MIT" + ], + [ + "tower-layer", + "0.3.3", + "MIT" + ], + [ + "tower-service", + "0.3.3", + "MIT" + ], + [ + "tracing", + "0.1.44", + "MIT" + ], + [ + "tracing-core", + "0.1.36", + "MIT" + ], + [ + "try-lock", + "0.2.5", + "MIT" + ], + [ + "twox-hash", + "2.1.2", + "MIT" + ], + [ + "uuid", + "1.23.3", + "Apache-2.0 OR MIT" + ], + [ + "want", + "0.3.1", + "MIT" + ], + [ + "zerocopy", + "0.8.52", + "BSD-2-Clause OR Apache-2.0 OR MIT" + ], + [ + "zstd", + "0.13.3", + "MIT" + ], + [ + "zstd-safe", + "7.2.4", + "MIT OR Apache-2.0" + ], + [ + "zstd-sys", + "2.1.3+zstd.1.5.7", + "BSD-3-Clause AND BSD-2-Clause" + ] + ], + "workspace": [ + [ + "apache-spark-connect-proto", + "4.4.0" + ], + [ + "spark-connect-jni", + "0.1.0" + ], + [ + "spark-connect-server", + "0.1.0" + ] + ] + } + } +} diff --git a/crates/spark-connect-server/jvm/package-native.py b/crates/spark-connect-server/jvm/package-native.py new file mode 100755 index 00000000..263480e5 --- /dev/null +++ b/crates/spark-connect-server/jvm/package-native.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Package already-built JNI libraries; this command does not cross-compile them.""" + +import argparse +import hashlib +import importlib.util +import os +from pathlib import Path +import platform +import re +import shutil +import struct +import tempfile +import zipfile + + +PROJECT = Path(__file__).resolve().parent.parent +REPOSITORY = PROJECT.parent.parent +LICENSE_SPEC = importlib.util.spec_from_file_location( + "native_binary_licenses", REPOSITORY / "scripts/gen_native_binary_licenses.py" +) +native_licenses = importlib.util.module_from_spec(LICENSE_SPEC) +LICENSE_SPEC.loader.exec_module(native_licenses) +ABI_VERSION = "1" +MAX_LIBRARY_SIZE = 512 * 1024 * 1024 +MAX_METADATA_SIZE = 16 * 1024 +PLATFORMS = { + "linux-x86_64": ("libspark_connect_jni.so", "elf", 62), + "linux-aarch64": ("libspark_connect_jni.so", "elf", 183), + "macos-x86_64": ("libspark_connect_jni.dylib", "macho", 0x01000007), + "macos-aarch64": ("libspark_connect_jni.dylib", "macho", 0x0100000C), +} +ADAPTER_METADATA = "META-INF/spark-connect-native-adapter.properties" +SEMVER_NUMBER = r"(?:0|[1-9][0-9]*)" +SEMVER_PRERELEASE = r"(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)" +SEMVER = re.compile( + rf"{SEMVER_NUMBER}\.{SEMVER_NUMBER}\.{SEMVER_NUMBER}" + rf"(?:-{SEMVER_PRERELEASE}(?:\.{SEMVER_PRERELEASE})*)?" + r"(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?" +) + + +def version_from_manifest(manifest): + # This unpublished crate has a literal version; avoid requiring a TOML dependency or + # Python 3.11 merely to read it. Fail closed if the manifest changes its version form. + packages = re.findall( + r"(?ms)^[ \t]*\[package\][ \t]*(?:#[^\n]*)?\n(.*?)(?=^[ \t]*\[|\Z)", + manifest, + ) + if len(packages) != 1: + raise ValueError("JNI crate must declare one literal SemVer package version") + declarations = re.findall(r"^[ \t]*version(?:[ \t]*=.*|\..*)$", packages[0], re.MULTILINE) + if len(declarations) != 1: + raise ValueError("JNI crate must declare one literal SemVer package version") + literal = re.fullmatch(r'[ \t]*version[ \t]*=[ \t]*"([^"\\\r\n]*)"[ \t]*(?:#[^\r\n]*)?', + declarations[0]) + if literal is None or SEMVER.fullmatch(literal.group(1)) is None: + raise ValueError("JNI crate must declare one literal SemVer package version") + return literal.group(1) + + +def artifact_version(): + return version_from_manifest((PROJECT / "jni/Cargo.toml").read_text(encoding="utf-8")) + + +def host_platform(): + system = {"Linux": "linux", "Darwin": "macos"}.get(platform.system()) + arch = {"x86_64": "x86_64", "amd64": "x86_64", "arm64": "aarch64", + "aarch64": "aarch64"}.get(platform.machine().lower()) + if system is None or arch is None: + raise ValueError("native JARs support only Linux/macOS on x86_64 or aarch64") + return f"{system}-{arch}" + + +def validate_library(data, classifier): + """Reject wrong-platform input rather than publishing a mislabeled artifact.""" + if classifier not in PLATFORMS: + raise ValueError(f"unsupported platform: {classifier}") + if len(data) > MAX_LIBRARY_SIZE: + raise ValueError("native library exceeds the 512 MiB packaging limit") + _, kind, machine = PLATFORMS[classifier] + if kind == "elf": + if len(data) < 64 or data[:7] != b"\x7fELF\x02\x01\x01": + raise ValueError(f"{classifier} requires a 64-bit little-endian ELF library") + file_type, actual_machine, version = struct.unpack_from(" MAX_METADATA_SIZE: + raise ValueError("native metadata exceeds the 16 KiB limit") + try: + lines = data.decode("ascii").splitlines() + except UnicodeDecodeError as error: + raise ValueError("native metadata must be ASCII") from error + result = {} + for line in lines: + if not line.strip() or line.startswith(("#", "!")): + continue + if "=" not in line or "\\" in line: + raise ValueError("native metadata uses unsupported Properties syntax") + key, value = line.split("=", 1) + if key in result or key != key.strip() or value != value.strip(): + raise ValueError("native metadata contains duplicate or padded properties") + result[key] = value + return result + + +def licenses(): + # These assets cover spark-connect-jni's own normal dependency closure on all + # four targets. Check their Cargo/source and content hashes before packaging. + return native_licenses.load_bundle(REPOSITORY) + + +def write_jar(path, entries): + """Use stable archive metadata and atomically replace only completed output.""" + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + os.close(descriptor) + try: + with zipfile.ZipFile(temporary, "w") as archive: + for name, data in sorted(entries.items()): + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.create_system = 3 + info.external_attr = 0o100644 << 16 + archive.writestr(info, data, compress_type=zipfile.ZIP_DEFLATED, compresslevel=9) + os.chmod(temporary, 0o644) + os.replace(temporary, path) + finally: + Path(temporary).unlink(missing_ok=True) + write_checksum(path) + return path + + +def write_checksum(path): + # Publish a complete checksum file; JAR and checksum replacement remain separate. + digest = hashlib.sha256(path.read_bytes()).hexdigest() + destination = path.with_suffix(path.suffix + ".sha256") + descriptor, temporary = tempfile.mkstemp(prefix=f".{destination.name}.", dir=path.parent) + os.close(descriptor) + try: + Path(temporary).write_text(f"{digest} {path.name}\n", encoding="ascii") + os.chmod(temporary, 0o644) + os.replace(temporary, destination) + finally: + Path(temporary).unlink(missing_ok=True) + + +def checked_archive(path): + archive = zipfile.ZipFile(path) + names = archive.namelist() + if len(names) != len(set(names)): + archive.close() + raise ValueError(f"duplicate JAR resources in {path}") + return archive + + +def check_version(metadata, version): + if metadata.get("artifact.version") != version: + raise ValueError("native and adapter artifact versions must match the JNI crate") + if metadata.get("native.abi.version") != ABI_VERSION: + raise ValueError("unsupported native ABI version") + + +def package_library(library, output_dir, classifier, version): + library = Path(library) + if not library.is_absolute() or not library.is_file(): + raise ValueError("--library must name an existing absolute library path") + filename = PLATFORMS[classifier][0] + if library.name != filename: + raise ValueError(f"{classifier} library must be named {filename}") + if library.stat().st_size > MAX_LIBRARY_SIZE: + raise ValueError("native library exceeds the 512 MiB packaging limit") + data = library.read_bytes() + validate_library(data, classifier) + prefix = f"META-INF/native/{classifier}/" + entries = licenses() + entries[prefix + filename] = data + entries[prefix + "native.properties"] = properties({ + "artifact.version": version, + "native.abi.version": ABI_VERSION, + "library.name": filename, + "library.sha256": hashlib.sha256(data).hexdigest(), + }) + return write_jar(output_dir / f"spark-connect-native-{version}-{classifier}.jar", entries) + + +def copy_adapter(adapter, output_dir, version): + adapter = Path(adapter) + destination = output_dir / f"spark-connect-native-adapter-{version}.jar" + descriptor, temporary = tempfile.mkstemp(prefix=f".{destination.name}.", dir=output_dir) + os.close(descriptor) + try: + # Validate the private copy: concurrent builds can replace adapter by rename. + # Copying first also keeps source == destination safe. + shutil.copyfile(adapter, temporary) + with checked_archive(temporary) as archive: + try: + info = archive.getinfo(ADAPTER_METADATA) + except KeyError as error: + raise ValueError("adapter JAR is missing generated version metadata") from error + if info.file_size > MAX_METADATA_SIZE: + raise ValueError("adapter metadata exceeds the 16 KiB limit") + check_version(parse_properties(archive.read(info)), version) + os.chmod(temporary, 0o644) + os.replace(temporary, destination) + finally: + Path(temporary).unlink(missing_ok=True) + write_checksum(destination) + return destination + + +def combine_jars(jars, output_dir, version, *, macos_only=False): + bundle = licenses() + entries = dict(bundle) + seen = set() + required = {name for name in PLATFORMS if not macos_only or name.startswith("macos-")} + artifact = "macos-universal" if macos_only else "all" + for path in jars: + with checked_archive(path) as archive: + metadata_names = [name for name in archive.namelist() + if re.fullmatch(r"META-INF/native/[^/]+/native\.properties", name)] + if len(metadata_names) != 1: + raise ValueError("combine inputs must each contain exactly one platform") + metadata_name = metadata_names[0] + classifier = metadata_name.split("/")[2] + if classifier not in PLATFORMS or classifier in seen: + raise ValueError(f"unsupported or duplicate platform in combine inputs: {classifier}") + if classifier not in required: + raise ValueError(f"{artifact} JAR does not accept platform: {classifier}") + info = archive.getinfo(metadata_name) + if info.file_size > MAX_METADATA_SIZE: + raise ValueError("native metadata exceeds the 16 KiB limit") + metadata_data = archive.read(info) + metadata = parse_properties(metadata_data) + check_version(metadata, version) + filename = PLATFORMS[classifier][0] + library_name = f"META-INF/native/{classifier}/{filename}" + if metadata.get("library.name") != filename: + raise ValueError("native metadata contains an unexpected library name") + expected_names = set(bundle) | {metadata_name, library_name} + if set(archive.namelist()) != expected_names: + raise ValueError("platform JAR contains unexpected or missing resources") + for name, data in bundle.items(): + if archive.getinfo(name).file_size != len(data) or archive.read(name) != data: + raise ValueError("platform JAR license resources do not match this source") + info = archive.getinfo(library_name) + if info.file_size > MAX_LIBRARY_SIZE: + raise ValueError("native library exceeds the 512 MiB packaging limit") + data = archive.read(info) + validate_library(data, classifier) + if hashlib.sha256(data).hexdigest() != metadata.get("library.sha256"): + raise ValueError("native library checksum does not match metadata") + entries[metadata_name] = metadata_data + entries[library_name] = data + seen.add(classifier) + missing = required - seen + if missing: + label = ("macOS universal JAR requires both macOS platforms" if macos_only + else "all-platform JAR requires every supported platform") + raise ValueError(label + "; missing: " + + ", ".join(sorted(missing))) + return write_jar(output_dir / f"spark-connect-native-{version}-{artifact}.jar", entries) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--print-version", action="store_true", help="print the shared JNI artifact version") + mode.add_argument("--library", type=Path, help="absolute path to an already-built native library") + mode.add_argument("--combine", type=Path, nargs="+", help="combine platform JARs into an all-platform JAR") + mode.add_argument("--combine-macos", type=Path, nargs="+", + help="combine both macOS platform JARs into a macOS universal JAR") + parser.add_argument("--platform", choices=sorted(PLATFORMS), help="default: current OS/architecture") + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--adapter-jar", type=Path, help="also copy and version-check the compiled adapter") + args = parser.parse_args() + try: + version = artifact_version() + if args.print_version: + if args.platform or args.output_dir or args.adapter_jar: + parser.error("--print-version cannot be combined with packaging options") + print(version) + return + if args.output_dir is None: + parser.error("--output-dir is required for packaging") + args.output_dir.mkdir(parents=True, exist_ok=True) + if args.combine or args.combine_macos: + if args.platform: + parser.error("--platform cannot be used with --combine or --combine-macos") + result = combine_jars(args.combine or args.combine_macos, args.output_dir, version, + macos_only=args.combine_macos is not None) + else: + result = package_library(args.library, args.output_dir, args.platform or host_platform(), version) + print(result) + if args.adapter_jar: + print(copy_adapter(args.adapter_jar, args.output_dir, version)) + except (OSError, ValueError, KeyError, zipfile.BadZipFile) as error: + parser.exit(1, f"native packaging failed: {error}\n") + + +if __name__ == "__main__": + main() diff --git a/crates/spark-connect-server/jvm/run-engine.sh b/crates/spark-connect-server/jvm/run-engine.sh new file mode 100755 index 00000000..bfeaa244 --- /dev/null +++ b/crates/spark-connect-server/jvm/run-engine.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +usage() { + echo "usage: $0 [listen-address]" >&2 + echo " $0 --native-jar [listen-address]" >&2 + exit 1 +} +[[ $# -ge 1 ]] || usage +NATIVE_JAR="" +if [[ "$1" == "--native-jar" ]]; then + [[ $# -ge 2 && $# -le 3 && "$2" == /* && -f "$2" && "$2" != *,* ]] || usage + NATIVE_JAR="$2" + NATIVE_INPUT=--bundled + LISTEN_ADDRESS="${3:-127.0.0.1:15002}" +else + [[ $# -le 2 && "$1" == /* && -f "$1" ]] || usage + NATIVE_INPUT="$1" + LISTEN_ADDRESS="${2:-127.0.0.1:15002}" +fi +PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +"$PROJECT_DIR/jvm/build.sh" +unset SPARK_REMOTE +export SPARK_LOCAL_IP="${SPARK_LOCAL_IP:-127.0.0.1}" +export SPARK_CONNECT_MODE=0 +SPARK_ARGUMENTS=(--class org.apache.spark.sql.connect.nativebridge.engine.EmbeddedSparkEngine + --master "${SPARK_MASTER:-local[2]}") +if [[ -n "$NATIVE_JAR" ]]; then + SPARK_ARGUMENTS+=(--jars "$NATIVE_JAR") +fi +exec "$SPARK_HOME/bin/spark-submit" "${SPARK_ARGUMENTS[@]}" \ + "$PROJECT_DIR/.build/jvm/spark-connect-native-bridge.jar" "$NATIVE_INPUT" "$LISTEN_ADDRESS" diff --git a/crates/spark-connect-server/jvm/run_acceptance.py b/crates/spark-connect-server/jvm/run_acceptance.py new file mode 100644 index 00000000..51896356 --- /dev/null +++ b/crates/spark-connect-server/jvm/run_acceptance.py @@ -0,0 +1,314 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Launch the native server inside Spark and run reproducible lifecycle/API acceptance.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import shlex +import signal +import socket +import subprocess +import sys +import tempfile +import time + + +SHUTDOWN_COMPLETE = "Embedded Spark Engine shutdown complete" + + +def shutdown_error(content, exit_code, marker=SHUTDOWN_COMPLETE): + if re.search(r"shutdown failed|JNI stop is incomplete|native stop incomplete", content, + flags=re.IGNORECASE): + return "embedded server reported failed or incomplete shutdown" + if re.search(r"JNI cleanup is incomplete|Java engine handle cleanup failed", content, + flags=re.IGNORECASE): + return "embedded server reported failed resource cleanup" + if marker not in content.splitlines(): + return "embedded server omitted successful cleanup marker" + if exit_code not in (0, -signal.SIGTERM, 128 + signal.SIGTERM): + return "embedded server exited abnormally" + return None + + +def record_jni_check(report, content): + report["jni_check_clean"] = not re.search( + r"WARNING in native method|FATAL ERROR in native method", content) + if not report["jni_check_clean"]: + report.setdefault("error", "embedded server triggered a JNI check warning") + + +def listener_owner_pids(port, *, proc_root=Path("/proc")): + """Inspect listener ownership with lsof, falling back to Linux /proc.""" + lsof = shutil.which("lsof") + if lsof is not None: + ownership = subprocess.run( + [lsof, "-nP", "-iTCP:" + str(port), "-sTCP:LISTEN", "-Fp"], + capture_output=True, text=True, check=True, timeout=10) + return sorted({int(line[1:]) for line in ownership.stdout.splitlines() + if line.startswith("p")}) + if not (proc_root / "net" / "tcp").is_file(): + raise RuntimeError("listener ownership check requires lsof or Linux /proc") + inodes = set() + for table in (proc_root / "net" / "tcp", proc_root / "net" / "tcp6"): + if not table.is_file(): + continue + for line in table.read_text().splitlines()[1:]: + fields = line.split() + if fields[3] == "0A" and int(fields[1].rsplit(":", 1)[1], 16) == port: + inodes.add("socket:[" + fields[9] + "]") + owners = set() + for process in proc_root.iterdir(): + if not process.name.isdecimal(): + continue + try: + for descriptor in (process / "fd").iterdir(): + try: + if os.readlink(descriptor) in inodes: + owners.add(int(process.name)) + except (FileNotFoundError, PermissionError, ProcessLookupError): + continue + except (FileNotFoundError, PermissionError, ProcessLookupError): + continue + return sorted(owners) + + +def ipc_validation_commands(common, tests_jar): + ipc = ["--class", "org.apache.spark.sql.connect.nativebridge.engine.IpcBatchSmokeTest", + str(tests_jar)] + cluster = list(common) + cluster[cluster.index("--master") + 1] = "local-cluster[1,1,1024]" + cluster[cluster.index("--driver-java-options") + 1] += " -XX:ActiveProcessorCount=2" + cluster += ["--executor-memory", "512m", "--executor-cores", "1", + "--conf", "spark.executor.extraJavaOptions=-Xcheck:jni -XX:ActiveProcessorCount=2", + "--conf", "spark.cores.max=1", "--conf", "spark.default.parallelism=2", + "--conf", "spark.task.maxFailures=1", + "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", + "--conf", "spark.kryo.registrationRequired=true", + "--conf", "spark.rpc.netty.dispatcher.numThreads=2", + "--conf", "spark.shuffle.io.serverThreads=2", + "--conf", "spark.shuffle.io.clientThreads=2"] + return ( + ("ipc-separate-executor", cluster + ipc, "IPC_BATCH_SMOKE_OK"), + ("ipc-result-size", common + ["--conf", "spark.driver.maxResultSize=1m"] + + ipc + ["--result-size-only"], "IPC_RESULT_SIZE_SMOKE_OK"), + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--library", required=True, type=Path, help="built JNI .dylib or .so") + parser.add_argument("--native-jar", type=Path, + help="also exercise classpath loading from this native platform/all JAR") + parser.add_argument("--spark-home", type=Path, default=os.environ.get("SPARK_HOME")) + parser.add_argument("--python", default=sys.executable, help="official PySpark client/worker Python") + parser.add_argument("--rust-example", type=Path, help="optional prebuilt official Rust client example") + parser.add_argument("--output", type=Path) + args = parser.parse_args() + if args.spark_home is None: + parser.error("set SPARK_HOME or pass --spark-home") + if sys.flags.optimize: + parser.error("acceptance checks require Python assertions; disable optimized mode") + root = Path(__file__).resolve().parents[1] + library = args.library.resolve(strict=True) + native_jar = args.native_jar.resolve(strict=True) if args.native_jar else None + spark_home = args.spark_home.resolve(strict=True) + reports = root / ".build" / "jvm-native" + reports.mkdir(parents=True, exist_ok=True) + # Keep every run's logs/client report separate so stale evidence cannot pass a rerun. + build = Path(tempfile.mkdtemp(prefix="engine-", dir=reports)) + output = args.output or reports / "engine-acceptance.json" + spark_submit = spark_home / "bin" / "spark-submit" + if not spark_submit.is_file(): + parser.error("Spark launcher does not exist: " + str(spark_submit)) + production_jar = root / ".build" / "jvm" / "spark-connect-native-bridge.jar" + tests_jar = root / ".build" / "jvm" / "bridge-test.jar" + for artifact in (production_jar, tests_jar): + if not artifact.is_file(): + parser.error("missing Java artifact; first run jvm/build.sh --tests: " + str(artifact)) + env = os.environ.copy() + env.update({"SPARK_HOME": str(spark_home), "SPARK_LOCAL_IP": "127.0.0.1", + "SPARK_CONNECT_MODE": "0", "PYSPARK_PYTHON": args.python, "TZ": "UTC"}) + env.pop("SPARK_REMOTE", None) + env.pop("PYSPARK_REMOTE_INIT_CONF_LEN", None) + common = [str(spark_submit), "--master", "local[2]", + "--driver-memory", "1g", "--driver-java-options", "-Xcheck:jni", + "--conf", "spark.ui.enabled=false", "--conf", "spark.driver.host=127.0.0.1", + "--conf", "spark.driver.bindAddress=127.0.0.1", "--conf", "spark.sql.shuffle.partitions=2", + "--conf", "spark.pyspark.python=" + args.python, + "--conf", "spark.sql.warehouse.dir=" + str(build / "warehouse")] + marker = SHUTDOWN_COMPLETE + report = {"backend": "spark-jvm", "library": str(library), "spark_home": str(spark_home), + "native_jar": str(native_jar) if native_jar else None, + "run_directory": str(build), "results": []} + server = None + log = None + address = None + + def run(name, command, timeout=180): + started = time.monotonic() + path = build / (name + ".log") + try: + completed = subprocess.run(command, cwd=root, env=env, capture_output=True, + text=True, timeout=timeout) + except subprocess.TimeoutExpired as error: + def decoded(value): + return value.decode(errors="replace") if isinstance(value, bytes) else value or "" + path.write_text(decoded(error.stdout) + decoded(error.stderr)) + report["results"].append({"name": name, "command": command, "timed_out": True, + "seconds": round(time.monotonic() - started, 3), + "log": str(path)}) + raise RuntimeError(name + " timed out; see " + str(path)) from error + path.write_text(completed.stdout + completed.stderr) + result = {"name": name, "command": command, "exit_code": completed.returncode, + "seconds": round(time.monotonic() - started, 3), "log": str(path)} + report["results"].append(result) + print(json.dumps(result), flush=True) + if completed.returncode != 0: + raise RuntimeError(name + " failed; see " + str(path)) + if re.search(r"WARNING in native method|FATAL ERROR in native method", path.read_text()): + raise RuntimeError(name + " triggered a JNI check warning") + return completed + + try: + for name, class_name, success in ( + ("engine-smoke", "EngineSmokeTest", "ENGINE_SMOKE_OK"), + ("ipc-smoke", "IpcBatchSmokeTest", "IPC_BATCH_SMOKE_OK"), + ): + completed = run(name, common + ["--class", + "org.apache.spark.sql.connect.nativebridge.engine." + class_name, + str(tests_jar)]) + if success not in completed.stdout: + raise RuntimeError(name + " success marker missing") + for name, command, success in ipc_validation_commands(common, tests_jar): + completed = run(name, command) + if success not in completed.stdout: + raise RuntimeError(name + " success marker missing") + engine_lifecycle = run("engine-lifecycle", common + ["--class", + "org.apache.spark.sql.connect.nativebridge.engine.NativeEngineLifecycleTest", + str(tests_jar), str(library)]) + if "NATIVE_ENGINE_LIFECYCLE_OK" not in engine_lifecycle.stdout: + raise RuntimeError("native engine lifecycle success marker missing") + cleanup_failure = run("cleanup-failure", common + ["--class", + "org.apache.spark.sql.connect.nativebridge.engine.CleanupFailureTest", + str(tests_jar), str(library)]) + if "CLEANUP_FAILURE_OK" not in cleanup_failure.stdout: + raise RuntimeError("cleanup failure retention success marker missing") + # A private JVM temp root lets the check prove extracted library cleanup at exit. + native_temp = build / "native-temp" + native_temp.mkdir() + packaged_options = (["--jars", str(native_jar), "--conf", + "spark.driver.extraJavaOptions=-Xcheck:jni -Djava.io.tmpdir=" + + shlex.quote(str(native_temp))] if native_jar else []) + command = common + packaged_options + ["--class", + "org.apache.spark.sql.connect.nativebridge.engine.EmbeddedSparkEngine", + str(production_jar), + "--bundled" if native_jar else str(library), "127.0.0.1:0"] + log_path = build / "server.log" + log = log_path.open("w") + server = subprocess.Popen(command, cwd=root, env=env, stdout=log, stderr=subprocess.STDOUT) + report.update({"server_command": command, "server_pid": server.pid, "server_log": str(log_path)}) + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + content = log_path.read_text(errors="replace") + listening = re.search(r"Embedded Spark Connect listening on (\S+)", content) + identity = re.search(r"Embedded Spark Connect JVM PID (\d+); Java Connect listener absent", content) + if listening and identity: + address = listening.group(1) + report["reported_jvm_pid"] = int(identity.group(1)) + if int(identity.group(1)) != server.pid: + raise RuntimeError("Spark launcher and JVM PID differ") + break + if server.poll() is not None: + raise RuntimeError("embedded server exited during startup; see " + str(log_path)) + time.sleep(0.1) + else: + raise RuntimeError("embedded server startup timed out; see " + str(log_path)) + owners = listener_owner_pids(int(address.rsplit(":", 1)[1])) + report["listener_owner_pids"] = owners + if owners != [server.pid]: + raise RuntimeError("native listener is not owned exclusively by the Spark JVM") + if native_jar: + extracted = list(native_temp.rglob("libspark_connect_jni.*")) + if len(extracted) != 1: + raise RuntimeError("expected one extracted native library while server is running") + if hashlib.sha256(extracted[0].read_bytes()).digest() != hashlib.sha256(library.read_bytes()).digest(): + raise RuntimeError("packaged library differs from selected native artifact") + report["extracted_native_library"] = str(extracted[0]) + env["SPARK_REMOTE"] = "sc://" + address + report["remote"] = env["SPARK_REMOTE"] + client_report = build / "engine-client.json" + run("engine-client", [args.python, "-B", "jvm/engine_acceptance.py", "--remote", + env["SPARK_REMOTE"], "--output", str(client_report)], timeout=600) + evidence = json.loads(client_report.read_text()) + if (evidence.get("passed") is not True or not evidence.get("checks") + or any(item.get("passed") is not True for item in evidence["checks"])): + raise RuntimeError("direct engine client report did not pass") + report["engine_client_report"] = str(client_report) + if args.rust_example: + run("official-rust", [str(args.rust_example.resolve(strict=True))]) + except Exception as error: + report["error"] = repr(error) + except KeyboardInterrupt: + report["error"] = "acceptance interrupted" + finally: + if server is not None: + if server.poll() is not None and "error" not in report: + report["error"] = "embedded server exited before requested shutdown" + if server.poll() is None: + server.send_signal(signal.SIGTERM) + try: + server.wait(timeout=20) + except subprocess.TimeoutExpired: + server.kill() + server.wait() + report["forced_kill"] = True + report["server_exit_code"] = server.returncode + if log is not None: + log.close() + if report.get("extracted_native_library"): + extracted = Path(report["extracted_native_library"]) + report["extracted_library_cleaned"] = not extracted.exists() and not extracted.parent.exists() + if not report["extracted_library_cleaned"]: + report.setdefault("error", "extracted native library or private directory leaked at JVM exit") + if address is not None: + try: + with socket.create_connection(("127.0.0.1", int(address.rsplit(":", 1)[1])), timeout=1): + report["listener_leaked"] = True + except OSError: + report["listener_closed"] = True + if "server_log" in report: + content = Path(report["server_log"]).read_text(errors="replace") + record_jni_check(report, content) + report["shutdown_complete"] = marker in content.splitlines() + failure = shutdown_error(content, report["server_exit_code"], marker) + if failure is not None: + report.setdefault("error", failure) + report["passed"] = "error" not in report and not report.get("forced_kill") and not report.get("listener_leaked") + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps({"output": str(output), "passed": report["passed"]}), flush=True) + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/ArrowPartitionConverter.java b/crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/ArrowPartitionConverter.java new file mode 100644 index 00000000..c8773e1f --- /dev/null +++ b/crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/ArrowPartitionConverter.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.spark.sql.connect.nativebridge.engine; + +import java.io.Serializable; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.execution.arrow.ArrowConverters$; +import org.apache.spark.sql.types.StructType; + +import scala.Tuple2; +import scala.collection.AbstractIterator; +import scala.collection.Iterator; +import scala.runtime.AbstractFunction1; + +/** Only conversion settings are serialized; Spark owns Arrow allocation and task cleanup. */ +final class ArrowPartitionConverter + extends AbstractFunction1, Iterator>> + implements Serializable { + private static final long serialVersionUID = 1L; + private static final long MAX_ESTIMATED_BATCH_BYTES = 2 * 1024 * 1024; + private final StructType schema; + private final String timeZone; + private final boolean largeVarTypes; + private final int maxRows; + + ArrowPartitionConverter(StructType schema, String timeZone, boolean largeVarTypes, int maxRows) { + this.schema = schema; + this.timeZone = timeZone; + this.largeVarTypes = largeVarTypes; + this.maxRows = maxRows; + } + + @Override + public Iterator> apply(Iterator rows) { + var batches = + ArrowConverters$.MODULE$.toBatchWithSchemaIterator(rows, schema, maxRows, + MAX_ESTIMATED_BATCH_BYTES, timeZone, false, largeVarTypes); + return new AbstractIterator>() { + @Override + public boolean hasNext() { return batches.hasNext(); } + @Override + public Tuple2 next() { + byte[] data = SparkJvmEngine.checkedIpc(batches.next()); + return new Tuple2<>(data, batches.rowCountInLastBatch()); + } + }; + } +} diff --git a/crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/EmbeddedSparkEngine.java b/crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/EmbeddedSparkEngine.java new file mode 100644 index 00000000..bc48ca06 --- /dev/null +++ b/crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/EmbeddedSparkEngine.java @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.spark.sql.connect.nativebridge.engine; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BooleanSupplier; + +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.connect.service.SparkConnectService$; +import org.apache.spark.util.ShutdownHookManager; + +/** Spark-submit launcher: Rust protocol/session/operation ownership with Catalyst execution. */ +public final class EmbeddedSparkEngine { + static final String SHUTDOWN_COMPLETE = "Embedded Spark Engine shutdown complete"; + static final String STARTUP_CLEANUP_COMPLETE = + "Embedded Spark Engine startup failed; cleanup complete"; + private EmbeddedSparkEngine() {} + + public static void main(String[] arguments) throws InterruptedException { + if (arguments.length < 1 || arguments.length > 2) { + throw new IllegalArgumentException( + "usage: EmbeddedSparkEngine [listen-address]"); + } + String address = arguments.length == 2 ? arguments[1] : "127.0.0.1:15002"; + SparkSession session = SparkSession.builder().appName("Rust Spark Connect with Catalyst") + .config("spark.sql.artifact.isolation.enabled", "true") + .config("spark.sql.artifact.isolation.alwaysApplyClassloader", "true") + .getOrCreate(); + SparkJvmEngine engine = null; + NativeEngineServer server = null; + CountDownLatch stopped = new CountDownLatch(1); + AtomicBoolean launched = new AtomicBoolean(); + AtomicBoolean stopping = new AtomicBoolean(); + Object hook = null; + try { + if (SparkConnectService$.MODULE$.server() != null) { + throw new IllegalStateException("a Java Spark Connect listener was started"); + } + engine = new SparkJvmEngine(session); + server = "--bundled".equals(arguments[0]) + ? new NativeEngineServer(address, engine) + : new NativeEngineServer(arguments[0], address, engine); + // NativeEngineServer owns the engine after acquisition; the extra idempotent close also + // covers Java-side startup failures before acquisition and keeps one cleanup helper. + SparkJvmEngine shutdownEngine = engine; + NativeEngineServer shutdownServer = server; + hook = ShutdownHookManager.addShutdownHook( + ShutdownHookManager.SPARK_CONTEXT_SHUTDOWN_PRIORITY() + 1, () -> { + stopping.set(true); + shutdown(shutdownServer::close, shutdownEngine::close, session::stop, stopped, + !launched.get()); + return scala.runtime.BoxedUnit.UNIT; + }); + System.out.println("Embedded Spark Connect listening on " + server.address()); + System.out.println("Embedded Spark Connect JVM PID " + ProcessHandle.current().pid() + + "; Java Connect listener absent"); + launched.set(true); + awaitListener(stopped, server::isRunning, stopping::get); + } catch (Throwable error) { + stopping.set(true); + if (hook != null) { + try { + ShutdownHookManager.removeShutdownHook(hook); + } catch (Throwable cleanup) { + if (cleanup != error) error.addSuppressed(cleanup); + } + } + NativeEngineServer failedServer = server; + SparkJvmEngine failedEngine = engine; + // main owns this application's Spark lifetime, including a session reused by + // getOrCreate (for example, one initialized by a Spark plugin during startup). + shutdown(() -> { + if (failedServer != null) { + closeAfterListenerExit(failedServer::close, failedServer::isDrained, error); + } + }, + () -> { if (failedEngine != null) failedEngine.close(); }, session::stop, stopped, + !launched.get()); + throw error; + } + } + + static void awaitListener(CountDownLatch stopped, BooleanSupplier running, + BooleanSupplier stopping) throws InterruptedException { + while (!stopped.await(100, TimeUnit.MILLISECONDS)) { + if (!running.getAsBoolean() && !stopping.getAsBoolean()) { + throw new IllegalStateException("native Spark Connect listener stopped unexpectedly"); + } + } + } + + static void closeAfterListenerExit(Runnable closeNative, BooleanSupplier drained, + Throwable listenerFailure) { + try { + closeNative.run(); + } catch (RuntimeException failure) { + if (!drained.getAsBoolean()) throw failure; + // The first close reports a terminal listener error while retaining the + // handle. Only confirmed drain permits this bounded cleanup retry. + try { + closeNative.run(); + } catch (RuntimeException retry) { + if (retry != failure) failure.addSuppressed(retry); + throw failure; + } + if (failure != listenerFailure) listenerFailure.addSuppressed(failure); + } + } + + static boolean shutdown(Runnable stopNative, Runnable stopEngine, Runnable stopSession, + CountDownLatch stopped, boolean startupFailure) { + try { + try { + stopNative.run(); + } catch (Throwable failure) { + System.err.println("Embedded Spark Engine shutdown failed: native stop incomplete; " + + "retaining engine and Spark resources while JNI work may remain"); + failure.printStackTrace(System.err); + return false; + } + Throwable failure = null; + try { + stopEngine.run(); + } catch (Throwable error) { + failure = error; + } + try { + stopSession.run(); + } catch (Throwable error) { + if (failure == null) failure = error; + else if (error != failure) failure.addSuppressed(error); + } + if (failure != null) { + System.err.println("Embedded Spark Engine shutdown failed: Spark cleanup did not finish"); + failure.printStackTrace(System.err); + return false; + } + System.out.println(startupFailure ? STARTUP_CLEANUP_COMPLETE : SHUTDOWN_COMPLETE); + return true; + } finally { + stopped.countDown(); + } + } +} diff --git a/crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/NativeEngineServer.java b/crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/NativeEngineServer.java new file mode 100644 index 00000000..bfb64793 --- /dev/null +++ b/crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/NativeEngineServer.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.spark.sql.connect.nativebridge.engine; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Objects; + +/** + * Loads the Rust protocol frontend with the typed JVM Engine implementation. + * + * The supplied engine is dedicated to this server: native startup acquires its ownership, + * successful server close closes it, and startup failure after acquisition closes it as well. + * Do not reuse or independently close the engine while this server is running. If construction + * throws, the caller should close the engine to cover failures before native acquisition + * (for example, library loading); engine close is idempotent. The caller continues to own the + * original SparkSession and SparkContext. + */ +public final class NativeEngineServer implements AutoCloseable { + private long handle; + + /** Load the native library from the matching platform JAR on the adapter's classpath. */ + public NativeEngineServer(String listenAddress, SparkJvmEngine engine) { + Objects.requireNonNull(listenAddress, "listenAddress"); + Objects.requireNonNull(engine, "engine"); + NativeLibraryLoader.loadBundled(); + start(listenAddress, engine); + } + + public NativeEngineServer(String absoluteLibraryPath, String listenAddress, SparkJvmEngine engine) { + Path library = Path.of(Objects.requireNonNull(absoluteLibraryPath, "absoluteLibraryPath")); + if (!library.isAbsolute()) throw new IllegalArgumentException("native library path must be absolute"); + Objects.requireNonNull(listenAddress, "listenAddress"); + Objects.requireNonNull(engine, "engine"); + NativeLibraryLoader.loadAbsolute(library); + start(listenAddress, engine); + } + + private void start(String listenAddress, SparkJvmEngine engine) { + handle = nativeStart(listenAddress.getBytes(StandardCharsets.UTF_8), engine); + if (handle == 0) throw new IllegalStateException("native engine returned an invalid handle"); + } + + public synchronized String address() { + if (handle == 0) throw new IllegalStateException("native listener is closed"); + return new String(nativeAddress(handle), StandardCharsets.UTF_8); + } + + /** Whether the native listener is still serving, independently of cleanup progress. */ + public synchronized boolean isRunning() { + return handle != 0 && nativeIsRunning(handle); + } + + // A terminal listener error can be fully drained but still need one close retry + // to release the factory. Unknown or incomplete drain must retain its resources. + synchronized boolean isDrained() { + return handle == 0 || nativeIsDrained(handle); + } + + /** + * Drain native work and close the owned engine. Repeated successful closes are harmless; + * failed teardown retains the handle and engine resources. A later close can retry a JNI + * attachment failure. A partial Java cleanup failure keeps reporting its original error + * until JVM exit rather than replaying cleanup that may not be idempotent. + */ + @Override + public synchronized void close() { + if (handle != 0) { + nativeStop(handle); + handle = 0; + } + } + + private static native long nativeStart(byte[] utf8Address, SparkJvmEngine engine); + private static native byte[] nativeAddress(long handle); + private static native void nativeStop(long handle); + private static native boolean nativeIsRunning(long handle); + private static native boolean nativeIsDrained(long handle); +} diff --git a/crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/NativeIpcBatch.java b/crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/NativeIpcBatch.java new file mode 100644 index 00000000..b0f639d3 --- /dev/null +++ b/crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/NativeIpcBatch.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.spark.sql.connect.nativebridge.engine; + +/** Driver-side holder for a self-contained Spark Arrow IPC stream and its row count. */ +public final class NativeIpcBatch { + private final byte[] data; + private final long rowCount; + + NativeIpcBatch(byte[] data, long rowCount) { + this.data = data; + this.rowCount = rowCount; + } + + /** Read-only bytes owned by this batch; JNI copies them before returning to native code. */ + public byte[] data() { return data; } + public long rowCount() { return rowCount; } +} diff --git a/crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/NativeLibraryLoader.java b/crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/NativeLibraryLoader.java new file mode 100644 index 00000000..135fd085 --- /dev/null +++ b/crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/NativeLibraryLoader.java @@ -0,0 +1,239 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.spark.sql.connect.nativebridge.engine; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermissions; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Enumeration; +import java.util.HexFormat; +import java.util.Locale; +import java.util.Objects; +import java.util.Properties; + +/** Loads a matched native artifact from this adapter's defining class loader. */ +final class NativeLibraryLoader { + private static final String ADAPTER_PROPERTIES = + "META-INF/spark-connect-native-adapter.properties"; + private static final String ABI_VERSION = "1"; + private static final int MAX_METADATA_BYTES = 16 * 1024; + private static final long MAX_LIBRARY_BYTES = 512L * 1024 * 1024; + private static Path loadedPath; + private static ClassLoader loadedResourceLoader; + private static String loadedPlatform; + + private NativeLibraryLoader() {} + + static Path loadBundled() { + return loadBundled(NativeLibraryLoader.class.getClassLoader(), + System.getProperty("os.name"), System.getProperty("os.arch")); + } + + static synchronized Path loadAbsolute(Path library) { + Objects.requireNonNull(library, "library"); + if (!library.isAbsolute()) { + throw new IllegalArgumentException("native library path must be absolute"); + } + try { + Path resolved = library.toRealPath(); + if (loadedPath != null) { + if (loadedPath.equals(resolved)) return loadedPath; + throw new IllegalStateException( + "a different Spark Connect native library is already loaded: " + loadedPath); + } + System.load(resolved.toString()); + loadedPath = resolved; + return resolved; + } catch (IOException error) { + throw new IllegalStateException("cannot read Spark Connect native library " + library, error); + } + } + + // Package visibility permits deterministic resource and platform tests without Spark jars. + static synchronized Path loadBundled(ClassLoader resources, String osName, String osArch) { + String platform = platform(osName, osArch); + if (loadedPath != null) { + if (loadedResourceLoader == resources && platform.equals(loadedPlatform)) return loadedPath; + throw new IllegalStateException("a Spark Connect native library is already loaded from " + + loadedPath + "; use one native artifact source per adapter class loader"); + } + Path library = extract(resources, osName, osArch); + try { + System.load(library.toString()); + loadedPath = library; + loadedResourceLoader = resources; + loadedPlatform = platform; + return library; + } catch (RuntimeException | Error failure) { + deleteExtraction(library, failure); + throw failure; + } + } + + static String platform(String osName, String osArch) { + String os = Objects.requireNonNull(osName, "os.name").toLowerCase(Locale.ROOT); + String arch = Objects.requireNonNull(osArch, "os.arch").toLowerCase(Locale.ROOT); + String operatingSystem; + if (os.equals("linux")) operatingSystem = "linux"; + else if (os.equals("mac os x") || os.equals("macos") || os.equals("darwin")) { + operatingSystem = "macos"; + } else { + throw new UnsupportedOperationException("unsupported Spark Connect native operating system: " + + osName + "; packaged libraries support Linux and macOS"); + } + String architecture; + if (arch.equals("amd64") || arch.equals("x86_64")) architecture = "x86_64"; + else if (arch.equals("aarch64") || arch.equals("arm64")) architecture = "aarch64"; + else { + throw new UnsupportedOperationException("unsupported Spark Connect native architecture: " + + osArch + "; packaged libraries support x86_64 and aarch64"); + } + return operatingSystem + "-" + architecture; + } + + static Path extract(ClassLoader resources, String osName, String osArch) { + Objects.requireNonNull(resources, "resource class loader"); + String platform = platform(osName, osArch); + String libraryName = "libspark_connect_jni." + (platform.startsWith("macos-") ? "dylib" : "so"); + String prefix = "META-INF/native/" + platform + "/"; + Path directory = null; + Path library = null; + try { + Properties adapter = properties(uniqueResource(resources, ADAPTER_PROPERTIES)); + String version = required(adapter, "artifact.version"); + requireEquals(ABI_VERSION, required(adapter, "native.abi.version"), "adapter native ABI"); + Properties nativeArtifact = properties( + uniqueResource(resources, prefix + "native.properties")); + requireEquals(version, required(nativeArtifact, "artifact.version"), + "native artifact version"); + requireEquals(ABI_VERSION, required(nativeArtifact, "native.abi.version"), "native ABI"); + requireEquals(libraryName, required(nativeArtifact, "library.name"), "native library name"); + String expectedHash = required(nativeArtifact, "library.sha256"); + if (!expectedHash.matches("[0-9a-f]{64}")) { + throw new IOException( + "native library SHA-256 must contain 64 lowercase hexadecimal digits"); + } + URL resource = uniqueResource(resources, prefix + libraryName); + directory = Files.createTempDirectory("spark-connect-native-", + PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------"))); + directory = directory.toRealPath(); + // Delete-on-exit is ordered in reverse registration order: file before directory. + directory.toFile().deleteOnExit(); + library = Files.createFile(directory.resolve(libraryName), + PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-------"))); + library.toFile().deleteOnExit(); + MessageDigest digest = sha256(); + long total = 0; + try (InputStream input = resource.openStream(); + OutputStream output = Files.newOutputStream(library, StandardOpenOption.WRITE)) { + byte[] buffer = new byte[64 * 1024]; + int read; + while ((read = input.read(buffer)) != -1) { + total += read; + if (total > MAX_LIBRARY_BYTES) throw new IOException("native library exceeds 512 MiB"); + digest.update(buffer, 0, read); + output.write(buffer, 0, read); + } + } + if (total == 0) throw new IOException("native library resource is empty"); + String actualHash = HexFormat.of().formatHex(digest.digest()); + if (!expectedHash.equals(actualHash)) { + throw new IOException("native library SHA-256 mismatch for " + platform); + } + return library; + } catch (IOException | RuntimeException failure) { + if (library != null) deleteExtraction(library, failure); + else if (directory != null) { + try { + Files.deleteIfExists(directory); + } catch (IOException cleanup) { + failure.addSuppressed(cleanup); + } + } + throw new IllegalStateException("cannot load bundled Spark Connect native library for " + + platform + ": " + failure.getMessage(), failure); + } + } + + private static URL uniqueResource(ClassLoader resources, String name) throws IOException { + Enumeration matches = resources.getResources(name); + if (!matches.hasMoreElements()) { + throw new IOException("missing classpath resource " + name + + "; include the adapter JAR and matching platform native JAR"); + } + URL first = matches.nextElement(); + if (matches.hasMoreElements()) { + throw new IOException("multiple classpath resources for " + name + + "; include exactly one matching artifact version"); + } + return first; + } + + private static Properties properties(URL resource) throws IOException { + byte[] bytes; + try (InputStream input = resource.openStream()) { + bytes = input.readNBytes(MAX_METADATA_BYTES + 1); + } + if (bytes.length > MAX_METADATA_BYTES) { + throw new IOException("native artifact metadata exceeds 16 KiB"); + } + Properties values = new Properties(); + values.load(new ByteArrayInputStream(bytes)); + return values; + } + + private static String required(Properties values, String key) throws IOException { + String value = values.getProperty(key); + if (value == null || value.isEmpty()) { + throw new IOException("missing native artifact property " + key); + } + return value; + } + + private static void requireEquals(String expected, String actual, String description) + throws IOException { + if (!expected.equals(actual)) { + throw new IOException(description + " mismatch: expected " + expected + ", found " + actual); + } + } + + private static MessageDigest sha256() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException impossible) { + throw new AssertionError("JDK must support SHA-256", impossible); + } + } + + private static void deleteExtraction(Path library, Throwable failure) { + try { + Files.deleteIfExists(library); + Files.deleteIfExists(library.getParent()); + } catch (IOException cleanup) { + failure.addSuppressed(cleanup); + } + } +} diff --git a/crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/SparkJvmEngine.java b/crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/SparkJvmEngine.java new file mode 100644 index 00000000..790ed658 --- /dev/null +++ b/crates/spark-connect-server/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine/SparkJvmEngine.java @@ -0,0 +1,1018 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.spark.sql.connect.nativebridge.engine; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.regex.Pattern; + +import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.spark.SparkContext; +import org.apache.spark.SparkEnv; +import org.apache.spark.SparkException; +import org.apache.spark.SparkThrowable; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.connect.proto.Relation; +import org.apache.spark.sql.AnalysisException; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.catalyst.analysis.UnresolvedExecuteImmediate; +import org.apache.spark.sql.catalyst.plans.logical.Command; +import org.apache.spark.sql.catalyst.plans.logical.InsertIntoDir; +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan; +import org.apache.spark.sql.catalyst.plans.logical.ParsedStatement; +import org.apache.spark.sql.classic.Dataset; +import org.apache.spark.sql.connect.common.InvalidPlanInput; +import org.apache.spark.sql.connect.planner.SparkConnectPlanner; +import org.apache.spark.sql.connect.service.SessionHolder; +import org.apache.spark.sql.errors.QueryExecutionErrors$; +import org.apache.spark.sql.execution.CommandExecutionMode; +import org.apache.spark.sql.execution.QueryExecution; +import org.apache.spark.sql.execution.SQLExecution; +import org.apache.spark.sql.execution.SparkPlan; +import org.apache.spark.sql.execution.arrow.ArrowConverters$; +import org.apache.spark.sql.types.ArrayType; +import org.apache.spark.sql.types.CharType; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.MapType; +import org.apache.spark.sql.types.StringType; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.types.UserDefinedType; +import org.apache.spark.sql.types.VarcharType; +import org.sparkproject.connect.protobuf.InvalidProtocolBufferException; +import org.sparkproject.connect.protobuf.Message; + +import scala.Option; +import scala.Tuple2; +import scala.reflect.ClassTag; +import scala.reflect.ClassTag$; +import scala.runtime.AbstractFunction0; +import scala.runtime.AbstractFunction1; + +/** + * A query engine for the Rust-owned frontend. The JNI boundary passes Catalyst inputs and Arrow + * batches, never service RPC requests, responses, observers, or operation registries. + * + * This intentionally targets the pinned Spark 4.2 assembly. SessionHolder is a planner resource + * container only: its service-managed close() and SparkSession.close() are never called. + * + * Passing this engine to NativeEngineServer transfers its lifetime to that server. Native stop + * closes it, and a failed native start can close it too; do not share or reuse it afterwards. + * close() is idempotent, so callers may also close it while cleaning up a failed constructor. + * The caller retains ownership of the original SparkSession and shared SparkContext. + */ +public final class SparkJvmEngine implements AutoCloseable { + static final int MAX_IPC_BYTES = 64 * 1024 * 1024; + private static final int MAX_BATCH_ROWS = 10000; + private static final String SHUFFLE_FILE_CLEANUP = + "spark.sql.classic.shuffleDependency.fileCleanup.enabled"; + private static final Set RELATIONS = Set.of( + "RANGE", "LOCAL_RELATION", "SQL", "PROJECT", "FILTER", "LIMIT", "OFFSET", "TAIL", + "JOIN", "SORT", "AGGREGATE", "DEDUPLICATE", "SET_OP", "SUBQUERY_ALIAS", "TO_DF", + "WITH_COLUMNS_RENAMED", "WITH_COLUMNS", "DROP", "REPARTITION", + "REPARTITION_BY_EXPRESSION", "HINT", "READ"); + + private final org.apache.spark.sql.classic.SparkSession base; + private final SparkContext sparkContext; + private final Pattern redactionPattern; + private final SparkEnv sparkEnv; + private final ClassLoader classLoader; + private final AtomicBoolean closed = new AtomicBoolean(); + private final Set sessions = ConcurrentHashMap.newKeySet(); + private final Set tasks = ConcurrentHashMap.newKeySet(); + private boolean closeComplete; + private Throwable closeFailure; + + public SparkJvmEngine(SparkSession session) { + if (!(Objects.requireNonNull(session, "session") + instanceof org.apache.spark.sql.classic.SparkSession)) { + throw new IllegalArgumentException("a classic Spark session is required"); + } + base = (org.apache.spark.sql.classic.SparkSession) session; + sparkContext = base.sparkContext(); + // SparkContext configuration is fixed after startup. Cache the host policy rather + // than cloning the whole SparkConf for each Config RPC or reading session overrides. + redactionPattern = sparkContext.getConf().get( + org.apache.spark.internal.config.package$.MODULE$.SECRET_REDACTION_PATTERN()).pattern(); + sparkEnv = SparkEnv.get(); + ClassLoader current = Thread.currentThread().getContextClassLoader(); + classLoader = current == null ? getClass().getClassLoader() : current; + } + + /** Allocate this control before invoking a blocking planner or cursor method. */ + public Task newTask(String description) { + ensureOpen(); + Task task = new Task(Objects.requireNonNull(description, "description")); + tasks.add(task); + if (closed.get()) task.cancel(); + return task; + } + + public Session openSession(String userId, String sessionId, Task task) { + Objects.requireNonNull(userId, "userId"); + Objects.requireNonNull(sessionId, "sessionId"); + return task.run(null, () -> { + ensureOpen(); + Session session = new Session(userId, sessionId); + sessions.add(session); + if (closed.get() || task.isCancelled()) { + session.close(); + task.check(); + ensureOpen(); + } + task.complete(); + return session; + }); + } + + boolean isClosed() { + return closed.get(); + } + + private void ensureOpen() { + if (closed.get()) throw new EngineException(9, "JVM engine is closed"); + } + + /** + * Idempotently release this engine's tasks and sessions without stopping the shared + * SparkContext. Once passed to NativeEngineServer, drain that server before calling this. + * A partial cleanup failure remains visible on later closes without replaying its side effects. + */ + @Override + public synchronized void close() { + throwFailure(closeFailure); + if (closeComplete) return; + closed.set(true); + Throwable failure = null; + for (Task task : tasks) { + try { task.close(); } catch (Throwable error) { failure = recordFailure(failure, error); } + } + for (Session session : sessions) { + try { session.close(); } catch (Throwable error) { failure = recordFailure(failure, error); } + } + closeFailure = failure; + throwFailure(failure); + closeComplete = true; + } + + @FunctionalInterface + private interface Action { T run() throws Exception; } + + /** A small stable error surface for the native adapter's tonic Status mapping. */ + public static final class EngineException extends RuntimeException { + private static final long serialVersionUID = 1L; + private final int code; + + public EngineException(int code, String message) { + super(message); + this.code = code; + } + + public EngineException(int code, String message, Throwable cause) { + super(message, cause); + this.code = code; + } + + public int code() { return code; } + } + + private static EngineException engineError(Exception error) { + if (error instanceof EngineException) return (EngineException) error; + if (error instanceof OutOfMemoryException) { + return arrowAllocationFailure((OutOfMemoryException) error); + } + if (error instanceof SparkException) { + // Executor conversion errors arrive inside Spark's task-failure wrapper. Preserve only + // resource errors from that cause chain; an arbitrary nested argument error is not a + // client validation failure. + Set seen = Collections.newSetFromMap(new IdentityHashMap<>()); + for (Throwable cause = error.getCause(); cause != null && seen.add(cause); + cause = cause.getCause()) { + if (cause instanceof EngineException && ((EngineException) cause).code() == 8) { + return new EngineException(8, cause.getMessage(), error); + } + } + } + int code = 13; + // The pinned planner uses InvalidPlanInput for malformed client plans even though its + // legacy message-only constructor labels them INTERNAL_ERROR. Its concrete type is the + // input-validation contract; arbitrary Spark internal exceptions still map to INTERNAL. + if (error instanceof InvalidProtocolBufferException || error instanceof InvalidPlanInput) { + code = 3; + } else if (error instanceof SparkThrowable && !((SparkThrowable) error).isInternalError()) { + if (error instanceof NoSuchElementException) code = 5; + else if (error instanceof AnalysisException || error instanceof IllegalArgumentException) code = 3; + } + return new EngineException(code, error.getMessage(), error); + } + + static EngineException arrowAllocationFailure(OutOfMemoryException error) { + return new EngineException(8, "JVM Arrow allocation failed: " + error.getMessage(), error); + } + + /** Cancellation is independent of a query/session lock and remains usable during preparation. */ + public final class Task implements AutoCloseable { + private final String jobGroup = "spark-connect-engine-" + UUID.randomUUID(); + private final String description; + // Partition, batch and task-step checks observe cross-thread cancellation. + // Compound task transitions remain locked. + private volatile boolean cancelled; + private boolean completed; + private boolean executionStarted; + private Thread running; + private final Object cancellation = new Object(); + private Throwable cancellationFailure; + + private Task(String description) { this.description = description; } + + public void cancel() { + synchronized (cancellation) { + throwFailure(cancellationFailure); + try { + boolean cancelJobs; + synchronized (this) { + if (completed || cancelled) return; + cancelled = true; + if (running != null) running.interrupt(); + cancelJobs = executionStarted || running != null; + } + // Active preparation/analysis can submit Spark jobs too (for example, parallel file + // listing). An idle task that never began fetching has no scheduler work to cancel. + if (cancelJobs) sparkContext.cancelJobGroupAndFutureJobs(jobGroup); + } catch (Throwable error) { + cancellationFailure = error; + throw error; + } + } + } + + private boolean isCancelled() { return cancelled; } + + private void check() { + if (cancelled) throw new EngineException(1, "JVM engine operation was cancelled"); + } + + private synchronized void complete() { completed = true; } + + private synchronized void beginExecution() { + check(); + executionStarted = true; + } + + private EngineException taskError(Exception error) { + if (isCancelled() && !(error instanceof EngineException)) { + return new EngineException(1, "JVM engine operation was cancelled", error); + } + return engineError(error); + } + + private T run(Session session, Action action) { + Thread thread = Thread.currentThread(); + boolean previousInterrupted; + synchronized (this) { + check(); + if (running != null) throw new EngineException(9, "task is already executing"); + previousInterrupted = Thread.interrupted(); + running = thread; + } + try { + // Capture can fail too (for example, a caller's context-class-loader accessor). Keep + // the running mark covered before any caller/Spark code and until restoration ends. + Properties previousProperties = sparkContext.getLocalProperties(); + ClassLoader previousLoader = thread.getContextClassLoader(); + SparkEnv previousEnv = SparkEnv.get(); + Option previousSession = SparkSession.getActiveSession(); + try { + if (session != null) session.activeTasks.add(this); + SparkEnv.set(sparkEnv); + thread.setContextClassLoader(classLoader); + sparkContext.setLocalProperties(new Properties()); + sparkContext.setJobGroup(jobGroup, description, true); + SparkSession.setActiveSession(session == null ? base : session.spark); + check(); + if (session == null) return action.run(); + session.ensureOpen(); + return session.holder.withSession( + new AbstractFunction1() { + @Override + public T apply(org.apache.spark.sql.classic.SparkSession ignored) { + try { + check(); + return action.run(); + } catch (InterruptedException error) { + throw new EngineException(1, "JVM engine operation was interrupted", error); + } catch (Exception error) { + throw taskError(error); + } + } + }); + } finally { + if (session != null) session.activeTasks.remove(this); + if (previousSession.isDefined()) SparkSession.setActiveSession(previousSession.get()); + else SparkSession.clearActiveSession(); + sparkContext.setLocalProperties(previousProperties); + thread.setContextClassLoader(previousLoader); + SparkEnv.set(previousEnv); + } + } catch (InterruptedException error) { + throw new EngineException(1, "JVM engine operation was interrupted", error); + } catch (Exception error) { + throw taskError(error); + } finally { + synchronized (this) { running = null; } + Thread.interrupted(); + if (previousInterrupted) thread.interrupt(); + } + } + + @Override + public void close() { + cancel(); + tasks.remove(this); + } + } + + /** Independent Catalyst catalog/configuration; identity and wire lifetime remain in Rust. */ + public final class Session implements AutoCloseable { + private final org.apache.spark.sql.classic.SparkSession spark; + private final SessionHolder holder; + private final ReentrantReadWriteLock catalog = new ReentrantReadWriteLock(true); + private final AtomicBoolean released = new AtomicBoolean(); + private final Set queries = ConcurrentHashMap.newKeySet(); + private final Set activeTasks = ConcurrentHashMap.newKeySet(); + private boolean closeComplete; + private Throwable closeFailure; + + private Session(String userId, String sessionId) { + spark = base.newSession(); + try { + holder = new SessionHolder(userId, sessionId, spark); + } catch (Throwable error) { + // Never close SparkSession: its clone shares the caller's SparkContext. Use the same + // owned-resource cleanup as a published session, retaining any partial cleanup failure + // for engine close even though this constructor cannot return a handle to JNI. + abandonUnpublished(error); + throw error; + } + } + + // Package-local so the unpublished ownership path can be exercised without breaking a + // Spark constructor. This cleanup deliberately never accesses the uninitialized holder. + void abandonUnpublished(Throwable error) { + sessions.add(this); + try { close(); } catch (Throwable cleanup) { recordFailure(error, cleanup); } + } + + private void ensureOpen() { + SparkJvmEngine.this.ensureOpen(); + if (released.get()) throw new EngineException(9, "JVM engine session is closed"); + } + + public Query prepare(byte[] relationBytes, Task task) { + Objects.requireNonNull(relationBytes, "relationBytes"); + return task.run(this, () -> { + // Preparation only analyzes read-only plans and registers a concurrent query handle. + // Share admission with metadata/config readers; config mutation and close stay exclusive. + catalog.readLock().lockInterruptibly(); + try { + Query query = new Query(this, analyzedFrame(relationBytes, task), task); + queries.add(query); + if (released.get()) { + query.close(); + ensureOpen(); + } + if (task.isCancelled()) { + query.close(); + task.check(); + } + return query; + } finally { + catalog.readLock().unlock(); + } + }); + } + + /** + * Analysis modes 0/1/2 are schema/local/streaming. A strict boolean byte precedes the same + * schema IPC used by prepare; schema and streaming modes return a zero prefix. + */ + public byte[] analyze(int opcode, byte[] relationBytes, Task task) { + Objects.requireNonNull(relationBytes, "relationBytes"); + return task.run(this, () -> { + if (opcode < 0 || opcode > 2) { + throw new EngineException(12, "unsupported analysis opcode: " + opcode); + } + catalog.readLock().lockInterruptibly(); + try { + Dataset frame = analyzedFrame(relationBytes, task); + PlanMetadata metadata = new PlanMetadata(this, frame); + byte[] output = new byte[metadata.schemaIpc.length + 1]; + output[0] = (byte) (opcode == 1 && metadata.local ? 1 : 0); + System.arraycopy(metadata.schemaIpc, 0, output, 1, metadata.schemaIpc.length); + synchronized (task) { + task.check(); + ensureOpen(); + task.complete(); + } + return output; + } finally { + catalog.readLock().unlock(); + } + }); + } + + // The caller holds a catalog read/write lock inside Task.run and its session/artifact scopes. + private Dataset analyzedFrame(byte[] relationBytes, Task task) throws IOException { + task.check(); + ensureOpen(); + if (Boolean.parseBoolean(spark.conf().get(SHUFFLE_FILE_CLEANUP))) { + throw new EngineException(12, SHUFFLE_FILE_CLEANUP + + " is unsupported with per-partition SQL execution scopes"); + } + Relation relation = Relation.parseFrom(relationBytes); + validateRelations(relation); + LogicalPlan logical = new SparkConnectPlanner(holder, Option.empty()) + .transformRelation(relation); + rejectCommands(logical); + // Dataset.ofRows uses ALL and may eagerly execute a write produced by analysis. + // Inspect with SKIP first, then keep that same execution through the Dataset constructor. + QueryExecution execution = spark.sessionState().executePlan(logical, CommandExecutionMode.SKIP()); + execution.assertAnalyzed(); + rejectCommands(execution.analyzed()); + Dataset frame = new Dataset<>(execution, Encoders.row(execution.analyzed().schema())); + if (frame.isStreaming()) { + throw new EngineException(12, "streaming queries are unsupported"); + } + return frame; + } + + private void rejectCommands(LogicalPlan logical) { + if (logical.exists(new AbstractFunction1() { + @Override + public Object apply(LogicalPlan node) { + // InsertIntoStatement extends ParsedStatement, and InsertIntoDir extends UnaryNode; + // neither is a Command until analysis. SQL scripts and dynamic SQL can execute + // their statements during analysis itself, before the post-analysis guard runs. + // CompoundBody already implements Command in the pinned assembly. Use actual types + // so a Spark internal API rename fails compilation instead of weakening the guard. + return node instanceof Command || node instanceof ParsedStatement + || node instanceof InsertIntoDir || node instanceof UnresolvedExecuteImmediate; + } + })) { + throw new EngineException(12, "SQL commands and external writes are unsupported"); + } + } + + /** + * Internal typed fields: non-negative big-endian signed 32-bit pair count, then UTF-8 key + * length/data and nullable value length/data (signed -1 means absent). No Spark Connect + * request/response envelopes cross here. + */ + public byte[] config(int opcode, byte[] fields, Task task) { + List input = decodePairs(fields); + return task.run(this, () -> { + Lock access = configLock(opcode); + access.lockInterruptibly(); + try { + task.check(); + ensureOpen(); + List output = new ArrayList<>(); + switch (opcode) { + case 1: + for (Pair pair : input) { + String value = spark.conf().get(pair.key); + if (isRedacted(redactionPattern, pair.key, value)) { + throw QueryExecutionErrors$.MODULE$.sqlConfigNotFoundError(pair.key); + } + output.add(new Pair(pair.key, value)); + } + break; + case 2: + for (Pair pair : input) { + Option stored = spark.conf().getOption(pair.key); + String value = stored.isDefined() ? stored.get() : null; + output.add(new Pair(pair.key, + isRedacted(redactionPattern, pair.key, value) ? null : value)); + } + break; + case 3: + for (Pair pair : input) { + String value = spark.conf().get(pair.key, pair.value); + output.add(new Pair(pair.key, + isRedacted(redactionPattern, pair.key, value) ? pair.value : value)); + } + break; + case 4: + if (input.size() > 1) throw new EngineException(3, "getAll accepts one prefix"); + String prefix = input.isEmpty() ? "" : input.get(0).key; + spark.conf().getAllAsJava().entrySet().stream() + .filter(entry -> !isRedacted(redactionPattern, entry.getKey(), entry.getValue())) + .filter(entry -> entry.getKey().startsWith(prefix)) + .sorted(Map.Entry.comparingByKey()) + .forEach(entry -> output.add(new Pair( + entry.getKey().substring(prefix.length()), entry.getValue()))); + break; + case 5: + for (Pair pair : input) { + output.add(new Pair(pair.key, Boolean.toString(spark.conf().isModifiable(pair.key)))); + } + break; + case 6: + case 7: + applyConfig(opcode, input, task); + break; + case 8: + // A separate result preserves setter-error provenance. JNI/task/admission + // failures still fail the call rather than becoming silent Set warnings. + if (input.size() != 1) throw new EngineException(3, "silent set accepts one key"); + String warning = applyConfig(opcode, input, task); + if (warning != null) output.add(new Pair(input.get(0).key, warning)); + break; + default: + throw new EngineException(12, "unsupported config opcode: " + opcode); + } + task.complete(); + return encodePairs(output); + } finally { + access.unlock(); + } + }); + } + + private Lock configLock(int opcode) { + // JNI contract: 1..5 are reads; 6..8 mutate. Unknown opcodes cannot come from Rust; + // conservatively keep them exclusive until the task's normal validation rejects them. + switch (opcode) { + case 1: + case 2: + case 3: + case 4: + case 5: + return catalog.readLock(); + default: + return catalog.writeLock(); + } + } + + private String applyConfig(int opcode, List input, Task task) { + boolean silent = opcode == 8; + Map previous = new LinkedHashMap<>(); + Set attempted = new LinkedHashSet<>(); + Map explicit = spark.conf().getAllAsJava(); + for (Pair pair : input) { + if (opcode == 6 && pair.value == null) throw new EngineException(3, "missing config value"); + if (!previous.containsKey(pair.key)) previous.put(pair.key, explicit.get(pair.key)); + } + task.check(); + Throwable setterFailure = null; + try { + for (Pair pair : input) { + task.check(); + // A setter can modify state before throwing, so record its key before calling it. + attempted.add(pair.key); + // RuntimeConfig accepts unregistered spark.* keys even when isModifiable is false; + // let its setters reject static/core entries and invalid registered values. Only this + // input-validation boundary maps an unstructured IllegalArgumentException to a client + // error; the same exception from arbitrary Spark execution is INTERNAL. + try { + if (opcode != 7) spark.conf().set(pair.key, pair.value); + else spark.conf().unset(pair.key); + } catch (Throwable error) { + setterFailure = error; + if (!silent && error instanceof IllegalArgumentException) { + if (!(error instanceof SparkThrowable) || !((SparkThrowable) error).isInternalError()) { + throw new EngineException(3, error.getMessage(), error); + } + } + throw error; + } + } + synchronized (task) { + task.check(); + task.complete(); + } + } catch (Throwable error) { + Map current = null; + boolean restored = true; + try { current = spark.conf().getAllAsJava(); } catch (Throwable inspection) { + recordFailure(error, inspection); + restored = false; + } + for (String key : attempted) { + String value = previous.get(key); + // Skip rejected static entries and other unchanged keys. If inspection failed, + // conservatively restore every attempted key; unvisited keys never need rollback. + if (current != null && Objects.equals(value, current.get(key))) continue; + try { + // Restore the accepted snapshot independently of RuntimeConfig restrictions that + // this batch may have changed, such as rejection of Spark core configuration keys. + if (value == null) spark.sessionState().conf().unsetConf(key); + else spark.sessionState().conf().setConfString(key, value); + } catch (Throwable rollback) { + if (rollback != error) error.addSuppressed(rollback); + restored = false; + } + } + if (silent && error == setterFailure && restored) { + // Match Spark's silent Set diagnostics without retaining rejected values/messages. + String reason = error instanceof SparkThrowable + ? ((SparkThrowable) error).getCondition() : null; + if (reason == null) reason = error.getClass().getName(); + synchronized (task) { + task.check(); + if (error instanceof InterruptedException || Thread.currentThread().isInterrupted()) { + throw new EngineException(1, "JVM engine operation was interrupted", error); + } + task.complete(); + } + return reason; + } + throw error; + } + return null; + } + + @Override + public synchronized void close() { + throwFailure(closeFailure); + if (closeComplete) return; + released.set(true); + Throwable failure = null; + for (Task task : activeTasks) { + try { task.cancel(); } catch (Throwable error) { failure = recordFailure(failure, error); } + } + for (Query query : queries) { + try { query.close(); } catch (Throwable error) { failure = recordFailure(failure, error); } + } + // A partial Spark cleanup is not generally replayable (for example, artifact cache + // cleanup decrements reference counts). Preserve the failure and its owners instead. + if (failure == null) { + catalog.writeLock().lock(); + try { + // These resources belong to this newSession only. Never close SessionHolder or Spark. + try { + spark.sessionState().catalog().clearTempTables(); + } catch (Throwable error) { + failure = recordFailure(failure, error); + } + try { + spark.artifactManager().close(); + } catch (Throwable error) { + failure = recordFailure(failure, error); + } + } finally { + catalog.writeLock().unlock(); + } + } + closeFailure = failure; + throwFailure(failure); + sessions.remove(this); + closeComplete = true; + } + } + + /** Shared metadata evaluation order for analysis and executable query preparation. */ + private static final class PlanMetadata { + private final StructType schema; + private final String timeZone; + private final boolean largeVarTypes; + private final int maxRows; + private final boolean local; + private final byte[] schemaIpc; + + private PlanMetadata(Session session, Dataset frame) { + schema = frame.schema(); + validateResultType(schema); + timeZone = session.spark.sessionState().conf().sessionLocalTimeZone(); + largeVarTypes = session.spark.sessionState().conf().arrowUseLargeVarTypes(); + int configured = session.spark.sessionState().conf().arrowMaxRecordsPerBatch(); + maxRows = configured > 0 ? configured : MAX_BATCH_ROWS; + local = frame.isLocal(); + schemaIpc = createSchemaIpc(schema, timeZone, largeVarTypes); + } + } + + /** An analyzed Catalyst query and a lazy pull cursor. Only one caller may fetch at a time. */ + public final class Query implements AutoCloseable { + private final Session session; + private final QueryExecution execution; + private final ArrowPartitionConverter converter; + private final Task task; + private final byte[] schemaIpc; + private final boolean local; + private final ReentrantLock cursor = new ReentrantLock(); + private boolean released; + private Throwable closeFailure; + private boolean ended; + private Throwable pullFailure; + private SparkPlan plan; + private boolean planCleanupAttempted; + private JavaRDD> arrowPartitions; + private int nextPartition; + private ArrayDeque> partitionBatches; + + private Query(Session session, Dataset frame, Task task) { + this.session = session; + this.execution = frame.queryExecution(); + PlanMetadata metadata = new PlanMetadata(session, frame); + this.converter = new ArrowPartitionConverter( + metadata.schema, metadata.timeZone, metadata.largeVarTypes, metadata.maxRows); + this.task = task; + this.local = metadata.local; + this.schemaIpc = metadata.schemaIpc; + } + + public byte[] schemaIpc() { return schemaIpc.clone(); } + public boolean isLocal() { return local; } + public boolean isStreaming() { return false; } + + private void loadNextPartition() { + task.check(); + SQLExecution.withNewExecutionId(execution, + Option.apply("spark-connect-engine-arrow-partition"), + new AbstractFunction0() { + @Override + public Object apply() { + if (arrowPartitions == null) { + if (plan == null) plan = execution.executedPlan(); + ClassTag> batchesClass = ClassTag$.MODULE$.apply(Tuple2.class); + arrowPartitions = JavaRDD.fromRDD( + plan.execute().mapPartitionsInternal(converter, false, batchesClass), + batchesClass); + } + if (nextPartition < arrowPartitions.getNumPartitions()) { + // The pending heap owner is one partition, matching Spark's local iterator. + // Conversion happens before transfer, avoiding compressed UnsafeRow round trips. + List>[] fetched = + arrowPartitions.collectPartitions(new int[] {nextPartition}); + task.check(); + nextPartition++; + partitionBatches = new ArrayDeque<>(fetched[0]); + } + return null; + } + }); + } + + /** Pull a self-contained IPC batch produced by Spark's executor-side converter. */ + public NativeIpcBatch nextIpcBatch() { + return task.run(session, () -> { + cursor.lockInterruptibly(); + try { + throwFailure(pullFailure); + if (released) throw new EngineException(9, "query cursor is closed"); + if (ended) return null; + task.beginExecution(); + try { + while (partitionBatches == null || partitionBatches.isEmpty()) { + partitionBatches = null; + if (arrowPartitions != null && nextPartition == arrowPartitions.getNumPartitions()) { + ended = true; + task.complete(); + return null; + } + loadNextPartition(); + } + task.check(); + Tuple2 batch = partitionBatches.removeFirst(); + return new NativeIpcBatch(batch._1(), batch._2()); + } catch (Throwable error) { + pullFailure = error instanceof InterruptedException + ? new EngineException(1, "JVM engine operation was interrupted", error) + : error instanceof Exception ? task.taskError((Exception) error) : error; + partitionBatches = null; + arrowPartitions = null; + ended = true; + throwFailure(pullFailure); + return null; + } + } finally { + cursor.unlock(); + } + }); + } + + @Override + public void close() { + Throwable failure = null; + try { task.close(); } catch (Throwable error) { failure = error; } + cursor.lock(); + try { + // Release buffered results and never replay partial plan cleanup. + partitionBatches = null; + arrowPartitions = null; + released = true; + if (!planCleanupAttempted) { + planCleanupAttempted = true; + try { + if (plan != null) plan.cleanupResources(); + } catch (Throwable error) { + failure = recordFailure(failure, error); + } + } + closeFailure = failure == null ? closeFailure : recordFailure(closeFailure, failure); + throwFailure(closeFailure); + session.queries.remove(this); + } finally { + cursor.unlock(); + } + } + } + + private static byte[] createSchemaIpc(StructType schema, String timeZone, boolean largeVarTypes) { + return checkedIpc(ArrowConverters$.MODULE$.createEmptyArrowBatch( + schema, timeZone, false, largeVarTypes)); + } + + // Package-local so tests can cover exact Spark types that earlier Spark planning can normalize. + static void validateResultType(DataType type) { + validateResultType(type, false); + } + + private static void validateResultType(DataType type, boolean nested) { + if (type instanceof UserDefinedType || type instanceof CharType || type instanceof VarcharType + || (type instanceof StringType && ((StringType) type).collationId() + != ((StringType) DataTypes.StringType).collationId())) { + throw new EngineException(12, "result type cannot be preserved by Arrow schema transport: " + + type.catalogString()); + } + if (type instanceof StructType) { + StructType original = (StructType) type; + StructField[] fields = org.apache.spark.sql.catalyst.util.package$.MODULE$ + .removeInternalMetadata(original).fields(); + Set names = nested ? new HashSet<>() : Collections.emptySet(); + for (int i = 0; i < fields.length; i++) { + StructField field = fields[i]; + if (nested && !names.add(field.name())) { + throw new EngineException(12, + "duplicate nested field names cannot be preserved by Arrow schema transport: " + + field.name()); + } + // CHAR/VARCHAR's raw type annotation is semantic even though its name is internal. + if (original.fields()[i].metadata().contains("__CHAR_VARCHAR_TYPE_STRING")) { + throw new EngineException(12, + "CHAR/VARCHAR result metadata cannot be preserved by Arrow schema transport: " + + field.name()); + } + if (!field.metadata().isEmpty()) { + throw new EngineException(12, + "result field metadata cannot be preserved by Arrow schema transport: " + field.name()); + } + validateResultType(field.dataType(), true); + } + } else if (type instanceof ArrayType) { + validateResultType(((ArrayType) type).elementType(), true); + } else if (type instanceof MapType) { + validateResultType(((MapType) type).keyType(), true); + validateResultType(((MapType) type).valueType(), true); + } + } + + static byte[] checkedIpc(byte[] bytes) { + if (bytes.length > MAX_IPC_BYTES) throw new EngineException(8, "Arrow batch exceeds 64 MiB"); + return bytes; + } + + private static Throwable recordFailure(Throwable previous, Throwable error) { + if (previous == null) return error; + if (previous != error) previous.addSuppressed(error); + return previous; + } + + private static void throwFailure(Throwable failure) { + if (failure instanceof Error) throw (Error) failure; + if (failure instanceof RuntimeException) throw (RuntimeException) failure; + if (failure != null) throw new EngineException(13, "JVM engine cleanup failed", failure); + } + + private static void validateRelations(Message message) { + if (message instanceof Relation) { + Relation relation = (Relation) message; + String kind = relation.getRelTypeCase().name(); + if (!RELATIONS.contains(kind)) { + throw new EngineException(12, "unsupported JVM engine relation: " + kind); + } + if (relation.hasRead() && !relation.getRead().hasNamedTable()) { + throw new EngineException(12, "only named-table reads are supported in this prototype"); + } + } + String type = message.getDescriptorForType().getFullName(); + if (type.endsWith("CommonInlineUserDefinedFunction") + || type.endsWith("CommonInlineUserDefinedTableFunction") + || type.endsWith("TypedAggregateExpression") || type.endsWith("ScalarScalaUDF")) { + throw new EngineException(12, "user-defined functions are unsupported in this prototype"); + } + for (Object field : message.getAllFields().values()) { + if (field instanceof Message) validateRelations((Message) field); + else if (field instanceof List) { + for (Object item : (List) field) { + if (item instanceof Message) validateRelations((Message) item); + } + } + } + } + + private static final class Pair { + private final String key; + private final String value; + private Pair(String key, String value) { this.key = key; this.value = value; } + } + + private static boolean isRedacted(Pattern pattern, String key, String value) { + return pattern.matcher(key).find() || value != null && pattern.matcher(value).find(); + } + + private static List decodePairs(byte[] bytes) { + Objects.requireNonNull(bytes, "fields"); + if (bytes.length > MAX_IPC_BYTES) throw new EngineException(8, "config fields exceed 64 MiB"); + try (DataInputStream input = new DataInputStream(new ByteArrayInputStream(bytes))) { + int count = input.readInt(); + if (count < 0 || count > input.available() / 8) { + throw new EngineException(3, "invalid config field count"); + } + List result = new ArrayList<>(count); + for (int i = 0; i < count; i++) result.add(new Pair(readString(input, false), readString(input, true))); + if (input.available() != 0) throw new EngineException(3, "trailing config field bytes"); + return result; + } catch (IOException error) { + throw new EngineException(3, "invalid config fields", error); + } + } + + private static String readString(DataInputStream input, boolean nullable) throws IOException { + int len = input.readInt(); + if (len == -1 && nullable) return null; + if (len < 0 || len > input.available()) throw new EngineException(3, "invalid config string length"); + byte[] bytes = input.readNBytes(len); + try { + return StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT).decode(ByteBuffer.wrap(bytes)).toString(); + } catch (CharacterCodingException error) { + throw new EngineException(3, "invalid config UTF-8", error); + } + } + + private static byte[] encodePairs(List pairs) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + DataOutputStream output = new DataOutputStream(bytes); + output.writeInt(pairs.size()); + for (Pair pair : pairs) { + writeString(output, pair.key); + writeString(output, pair.value); + } + return bytes.toByteArray(); + } + + private static void writeString(DataOutputStream output, String value) throws IOException { + if (value == null) output.writeInt(-1); + else { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + output.writeInt(bytes.length); + output.write(bytes); + } + } +} diff --git a/crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/CleanupFailureTest.java b/crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/CleanupFailureTest.java new file mode 100644 index 00000000..5d8e9c26 --- /dev/null +++ b/crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/CleanupFailureTest.java @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.spark.sql.connect.nativebridge.engine; + +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.spark.connect.proto.Range; +import org.apache.spark.connect.proto.Relation; +import org.apache.spark.rdd.RDD; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.catalyst.expressions.Attribute; +import org.apache.spark.sql.execution.SparkPlan; + +/** Deliberately failed owners stay retained until this separate test JVM exits. */ +public final class CleanupFailureTest { + private CleanupFailureTest() {} + + public static void main(String[] arguments) throws Exception { + require(arguments.length == 1, "pass the absolute native library path"); + SparkSession spark = SparkSession.builder().appName("Native cleanup failure retention") + .config("spark.sql.artifact.isolation.enabled", "true") + .config("spark.sql.artifact.isolation.alwaysApplyClassloader", "true") + .getOrCreate(); + spark.sparkContext().setLogLevel("ERROR"); + try { + taskCancellationFailure(spark); + querySessionAndFactoryFailure(spark); + unpublishedSessionFailure(spark); + nativeFactoryFailure(spark, arguments[0]); + require(!spark.sparkContext().isStopped(), "failed cleanup stopped the shared SparkContext"); + System.out.println("CLEANUP_FAILURE_OK: task/query/session/factory failures retain " + + "owners and original errors without replaying partial cleanup; native stop keeps " + + "its retry handle and reports repeated Java cleanup failure after listener close"); + } finally { + spark.stop(); + } + } + + private static void taskCancellationFailure(SparkSession spark) throws Exception { + SparkJvmEngine engine = new SparkJvmEngine(spark); + SparkJvmEngine.Task task = engine.newTask("failing cancellation"); + RuntimeException failure = new IllegalStateException("injected task interruption failure"); + AtomicInteger attempts = new AtomicInteger(); + Thread running = new Thread() { + @Override + public void interrupt() { + attempts.incrementAndGet(); + throw failure; + } + }; + field(task, "running").set(task, running); + expectSame(failure, task::cancel); + // A real interrupted call clears running in Task.run's finally block. Its later close + // must still report the cancellation failure even after that worker has left. + field(task, "running").set(task, null); + expectSame(failure, task::close); + expectSame(failure, task::close); + require(registry(engine, "tasks").contains(task), "failed Task cleanup lost its owner"); + expectSame(failure, engine::close); + expectSame(failure, engine::close); + require(attempts.get() == 1, "failed task cancellation was replayed"); + } + + private static void querySessionAndFactoryFailure(SparkSession spark) throws Exception { + FailedQuery fixture = new FailedQuery(spark); + expectSame(fixture.failure, fixture.query::close); + expectSame(fixture.failure, fixture.query::close); + require(registry(fixture.session, "queries").contains(fixture.query), + "failed Query cleanup lost its owner"); + expectSame(fixture.failure, fixture.session::close); + expectSame(fixture.failure, fixture.session::close); + require(registry(fixture.engine, "sessions").contains(fixture.session), + "failed Session cleanup lost its owner"); + expectSame(fixture.failure, fixture.engine::close); + expectSame(fixture.failure, fixture.engine::close); + require(fixture.engine.isClosed(), "failed engine cleanup left admission open"); + require(fixture.plan.attempts.get() == 1, "partial physical-plan cleanup was replayed"); + } + + private static void unpublishedSessionFailure(SparkSession spark) throws Exception { + SparkJvmEngine healthyEngine = new SparkJvmEngine(spark); + SparkJvmEngine.Session healthy; + try (SparkJvmEngine.Task task = healthyEngine.newTask("unpublished clean session")) { + healthy = healthyEngine.openSession("unpublished-fixture", UUID.randomUUID().toString(), task); + } + // Reproduce the constructor's ownership state after newSession but before holder assignment. + field(healthy, "holder").set(healthy, null); + require(registry(healthyEngine, "sessions").remove(healthy), "clean fixture was not registered"); + RuntimeException construction = new IllegalArgumentException("injected SessionHolder failure"); + healthy.abandonUnpublished(construction); + require(construction.getSuppressed().length == 0, + "successful unpublished cleanup added a failure"); + require(!registry(healthyEngine, "sessions").contains(healthy), + "successfully cleaned unpublished session remained registered"); + require(field(healthy, "closeComplete").getBoolean(healthy), + "unpublished session cleanup did not finish"); + healthy.close(); + healthyEngine.close(); + + FailedQuery failed = new FailedQuery(spark); + field(failed.session, "holder").set(failed.session, null); + require(registry(failed.engine, "sessions").remove(failed.session), + "failed fixture was not registered"); + RuntimeException failedConstruction = + new IllegalArgumentException("injected SessionHolder and cleanup failure"); + failed.session.abandonUnpublished(failedConstruction); + require(failedConstruction.getSuppressed().length == 1 + && failedConstruction.getSuppressed()[0] == failed.failure, + "unpublished session lost its construction or cleanup error"); + require(registry(failed.engine, "sessions").contains(failed.session), + "failed unpublished cleanup lost its retained session"); + expectSame(failed.failure, failed.engine::close); + expectSame(failed.failure, failed.engine::close); + require(failed.plan.attempts.get() == 1, "unpublished partial cleanup was replayed"); + require(!spark.sparkContext().isStopped(), "unpublished cleanup stopped the shared SparkContext"); + } + + private static void nativeFactoryFailure(SparkSession spark, String library) throws Exception { + FailedQuery fixture = new FailedQuery(spark); + NativeEngineServer server = new NativeEngineServer(library, "127.0.0.1:0", fixture.engine); + String address = server.address(); + int port = Integer.parseInt(address.substring(address.lastIndexOf(':') + 1)); + long handle = field(server, "handle").getLong(server); + for (int attempt = 0; attempt < 2; attempt++) { + try { + server.close(); + throw new AssertionError("native stop accepted failed Java factory cleanup"); + } catch (IllegalStateException error) { + require(error.getMessage().contains("JNI cleanup is incomplete") + && error.getMessage().contains(fixture.failure.getMessage()), + "native stop lost the cleanup failure: " + error); + } + require(field(server, "handle").getLong(server) == handle, + "failed native stop discarded its retry handle"); + require(registry(fixture.engine, "sessions").contains(fixture.session), + "failed native factory cleanup discarded its Java owners"); + require(fixture.plan.attempts.get() == 1, "native retry replayed partial Java cleanup"); + } + try (ServerSocket rebound = new ServerSocket()) { + rebound.setReuseAddress(true); + rebound.bind(new InetSocketAddress("127.0.0.1", port)); + require(rebound.isBound(), "failed factory cleanup left its listener open"); + } + // server and its failed owners intentionally remain retained until this JVM exits. + } + + private static final class FailedQuery { + private final SparkJvmEngine engine; + private final SparkJvmEngine.Session session; + private final SparkJvmEngine.Query query; + private final RuntimeException failure = + new IllegalStateException("injected physical-plan cleanup failure"); + private final FailingCleanupPlan plan = new FailingCleanupPlan(failure); + + private FailedQuery(SparkSession spark) throws Exception { + engine = new SparkJvmEngine(spark); + try (SparkJvmEngine.Task task = engine.newTask("cleanup fixture open")) { + session = engine.openSession("cleanup-fixture", UUID.randomUUID().toString(), task); + } + SparkJvmEngine.Task task = engine.newTask("cleanup fixture prepare"); + query = session.prepare(Relation.newBuilder().setRange(Range.newBuilder() + .setStart(0).setEnd(1).setStep(1)).build().toByteArray(), task); + field(query, "plan").set(query, plan); + } + } + + private static final class FailingCleanupPlan extends SparkPlan { + private final RuntimeException failure; + private final AtomicInteger attempts = new AtomicInteger(); + + private FailingCleanupPlan(RuntimeException failure) { this.failure = failure; } + + @Override + public void cleanupResources() { + attempts.incrementAndGet(); + throw failure; + } + + @Override + public scala.collection.immutable.Seq output() { + return scala.collection.immutable.List$.MODULE$.empty(); + } + + @Override + public scala.collection.immutable.Seq children() { + return scala.collection.immutable.List$.MODULE$.empty(); + } + + @Override + public SparkPlan withNewChildrenInternal(scala.collection.immutable.IndexedSeq children) { + if (!children.isEmpty()) throw new IllegalArgumentException("leaf plan cannot have children"); + return this; + } + + @Override + public RDD doExecute() { throw new AssertionError("cleanup fixture executed"); } + + @Override + public String simpleStringWithNodeId() { return "FailingCleanupPlan"; } + + @Override + public String verboseString(int maxFields) { return "FailingCleanupPlan"; } + + @Override + public int productArity() { return 0; } + + @Override + public Object productElement(int index) { throw new IndexOutOfBoundsException(index); } + + @Override + public boolean canEqual(Object other) { return other instanceof FailingCleanupPlan; } + } + + private static Field field(Object owner, String name) throws ReflectiveOperationException { + Field field = owner.getClass().getDeclaredField(name); + field.setAccessible(true); + return field; + } + + private static Set registry(Object owner, String name) throws ReflectiveOperationException { + return (Set) field(owner, name).get(owner); + } + + private static void expectSame(RuntimeException expected, Runnable close) { + try { + close.run(); + throw new AssertionError("failed cleanup was accepted"); + } catch (RuntimeException actual) { + require(actual == expected, "cleanup replaced the original failure: " + actual); + } + } + + private static void require(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + } +} diff --git a/crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/EngineSmokeTest.java b/crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/EngineSmokeTest.java new file mode 100644 index 00000000..03b3d6f6 --- /dev/null +++ b/crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/EngineSmokeTest.java @@ -0,0 +1,1680 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.spark.sql.connect.nativebridge.engine; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.AbstractMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Properties; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.LockSupport; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.spark.SparkEnv; +import org.apache.spark.connect.proto.CommonInlineUserDefinedFunction; +import org.apache.spark.connect.proto.Deduplicate; +import org.apache.spark.connect.proto.Expression; +import org.apache.spark.connect.proto.LocalRelation; +import org.apache.spark.connect.proto.Project; +import org.apache.spark.connect.proto.Range; +import org.apache.spark.connect.proto.Read; +import org.apache.spark.connect.proto.Relation; +import org.apache.spark.connect.proto.SQL; +import org.apache.spark.connect.proto.ScalarScalaUDF; +import org.apache.spark.connect.proto.SubqueryAlias; +import org.apache.spark.connect.proto.TypedAggregateExpression; +import org.apache.spark.connect.proto.WithRelations; +import org.apache.spark.scheduler.SparkListener; +import org.apache.spark.scheduler.SparkListenerJobStart; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSessionExtensions; +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan; +import org.apache.spark.sql.catalyst.rules.Rule; +import org.apache.spark.sql.connect.service.SparkConnectService$; +import org.apache.spark.sql.connect.common.InvalidPlanInput; +import org.apache.spark.sql.util.ArrowUtils$; +import org.apache.spark.ml.linalg.VectorUDT; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.MetadataBuilder; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.sparkproject.connect.protobuf.Message; + +import scala.Option; +import scala.runtime.AbstractFunction1; +import scala.runtime.BoxedUnit; + +/** Direct Catalyst/Arrow tests. This program does not load the native library. */ +public final class EngineSmokeTest { + private static final AtomicReference ANALYSIS_GATE = new AtomicReference<>(); + private static final AtomicReference ANALYSIS_JOB = new AtomicReference<>(); + + private EngineSmokeTest() {} + + public static void main(String[] arguments) throws Exception { + shutdownHelpers(); + SparkSession spark = SparkSession.builder().appName("Typed JVM Engine Smoke") + .config("spark.sql.artifact.isolation.enabled", "true") + .config("spark.sql.artifact.isolation.alwaysApplyClassloader", "true") + .config("spark.redaction.regex", + "(?i)secret|password|token|access[.]?key|credential|native_hidden") + .withExtensions(new AbstractFunction1() { + @Override + public BoxedUnit apply(SparkSessionExtensions extensions) { + extensions.injectResolutionRule(new AbstractFunction1>() { + @Override + public Rule apply(SparkSession ruleSession) { + return new Rule() { + @Override + public LogicalPlan apply(LogicalPlan plan) { + AnalysisGate gate = ANALYSIS_GATE.get(); + if (gate != null) gate.enter(); + AnalysisJob job = ANALYSIS_JOB.get(); + if (job != null) job.enter(ruleSession); + return plan; + } + }; + } + }); + return BoxedUnit.UNIT; + } + }) + .getOrCreate(); + spark.sparkContext().setLogLevel("ERROR"); + try (SparkJvmEngine engine = new SparkJvmEngine(spark)) { + SparkJvmEngine.Session first = open(engine, "session-one"); + SparkJvmEngine.Session second = open(engine, "session-two"); + try { + metadataOnly(engine, first, second, spark); + failedControlCallsDoNotCancelJobs(engine, first, spark); + cancelActiveAnalysisJob(engine, first, spark); + rangeAndEmpty(engine, first); + duplicateFieldNameResults(engine, first); + namedTableRead(engine, first); + configAtomicityAndIsolation(engine, first, second); + configDependentRollback(engine, first); + configRedaction(engine, first); + configFieldCounts(engine, first); + plannerInputFailures(engine, first); + configSetterFailureRollsBackPartialWrite(engine, first); + failedTaskScopeCaptureCanRetry(engine, first); + configCancellationDoesNotWaitForMutation(engine, first); + cancelledBeforePrepare(engine, first); + commandsCannotMutateCatalog(engine, first); + earlyCloseKeepsContext(engine, first, second, spark); + cancelRunningFetch(engine, first, second, spark, true); + cancelRunningFetch(engine, first, second, spark, false); + releasedSessionPreparation(engine); + require(SparkConnectService$.MODULE$.server() == null, "Java gRPC listener was started"); + System.out.println("ENGINE_SMOKE_OK: metadata parity/lifecycle/context/cancellation without jobs; " + + "Catalyst range and SQL; schema and Arrow batches; " + + "concurrent metadata/config/preparation readers and exclusive mutations; " + + "config rollback/isolation; pre-cancel and running fetch cancellation; command/script/" + + "WITH_RELATIONS rejection; early close and shared-context survival; launcher teardown"); + } finally { + first.close(); + second.close(); + } + require(!spark.sparkContext().isStopped(), "session release stopped the shared SparkContext"); + } finally { + spark.stop(); + } + } + + private static SparkJvmEngine.Session open(SparkJvmEngine engine, String id) { + try (SparkJvmEngine.Task task = engine.newTask("open " + id)) { + return engine.openSession("smoke-user", id, task); + } + } + + private static Relation range(long count) { + return Relation.newBuilder().setRange(Range.newBuilder() + .setStart(0).setEnd(count).setStep(1).setNumPartitions(2)).build(); + } + + private static Relation sql(String text) { + return Relation.newBuilder().setSql(SQL.newBuilder().setQuery(text)).build(); + } + + private static SparkJvmEngine.Query query(SparkJvmEngine engine, SparkJvmEngine.Session session, + Relation relation) { + SparkJvmEngine.Task task = engine.newTask("prepare smoke query"); + try { + return session.prepare(relation.toByteArray(), task); + } catch (Throwable error) { + task.close(); + throw error; + } + } + + private static void rangeAndEmpty(SparkJvmEngine engine, SparkJvmEngine.Session session) + throws Exception { + try (SparkJvmEngine.Query query = query(engine, session, range(25))) { + require(!query.isStreaming(), "range became streaming"); + try (BufferAllocator allocator = allocator(); + ArrowStreamReader reader = new ArrowStreamReader( + new ByteArrayInputStream(query.schemaIpc()), allocator)) { + require(reader.getVectorSchemaRoot().getSchema().getFields().get(0).getName().equals("id"), + "prepared schema is missing the range id"); + require(reader.loadNextBatch(), "schema IPC did not include its empty record batch"); + require(reader.getVectorSchemaRoot().getRowCount() == 0, "prepare executed rows"); + } + long[] result = consume(query); + require(Arrays.equals(result, new long[] {25, 300}), "range Arrow values changed"); + require(query.nextIpcBatch() == null, "EOF was not stable"); + } + try (SparkJvmEngine.Query empty = query(engine, session, + sql("SELECT CAST(NULL AS BIGINT) AS missing WHERE false"))) { + require(empty.nextIpcBatch() == null, "empty SQL query returned rows"); + } + try (SparkJvmEngine.Query join = query(engine, session, + sql("SELECT sum(a.id + b.id) AS total FROM range(5) a JOIN range(5) b ON a.id = b.id"))) { + require(Arrays.equals(consume(join), new long[] {1, 20}), "Catalyst SQL join changed"); + } + try (SparkJvmEngine.Query expression = query(engine, session, + sql("SELECT id + 1 FROM range(5)"))) { + require(Arrays.equals(consume(expression), new long[] {5, 15}), + "internal alias metadata rejected an unaliased computed expression"); + } + try (SparkJvmEngine.Query aggregate = query(engine, session, + sql("SELECT sum(id) FROM range(5)"))) { + require(Arrays.equals(consume(aggregate), new long[] {1, 10}), + "internal alias metadata rejected an unaliased aggregate"); + } + } + + private static void namedTableRead(SparkJvmEngine engine, SparkJvmEngine.Session session) + throws Exception { + SparkSession sessionSpark = (SparkSession) privateField(session, "spark"); + String view = "named_read_" + UUID.randomUUID().toString().replace("-", ""); + sessionSpark.sql("SELECT CAST(7 AS BIGINT) AS id").createOrReplaceTempView(view); + Relation relation = Relation.newBuilder().setRead(Read.newBuilder() + .setNamedTable(Read.NamedTable.newBuilder().setUnparsedIdentifier(view))).build(); + try (SparkJvmEngine.Query prepared = query(engine, session, relation)) { + NativeIpcBatch batch = prepared.nextIpcBatch(); + require(batch != null && batch.rowCount() == 1, "named-table Read lost its row"); + try (BufferAllocator allocator = allocator(); + ArrowStreamReader reader = new ArrowStreamReader( + new ByteArrayInputStream(batch.data()), allocator)) { + require(reader.loadNextBatch(), "named-table Read IPC has no record batch"); + var root = reader.getVectorSchemaRoot(); + require(root.getRowCount() == 1 && root.getFieldVectors().size() == 1 + && Long.valueOf(7).equals(root.getVector(0).getObject(0)), + "named-table Read returned different rows"); + require(!reader.loadNextBatch(), "named-table Read IPC contains extra batches"); + } + require(prepared.nextIpcBatch() == null, "named-table Read returned extra rows"); + } finally { + sessionSpark.catalog().dropTempView(view); + } + } + + private static void failedControlCallsDoNotCancelJobs(SparkJvmEngine engine, + SparkJvmEngine.Session session, SparkSession spark) throws Exception { + for (int mode = 0; mode < 3; mode++) { + SparkJvmEngine.Task task = engine.newTask("failed control cleanup"); + String group = (String) privateField(task, "jobGroup"); + try { + if (mode == 0) { + expectCode(3, () -> session.prepare(sql("SELECT FROM").toByteArray(), task)); + } else if (mode == 1) { + expectCode(3, () -> session.analyze(0, sql("SELECT FROM").toByteArray(), task)); + } else { + byte[] invalid = encode("spark.sql.shuffle.partitions", "invalid-integer"); + expectCode(3, () -> session.config(6, invalid, task)); + } + require(!flag(task, "executionStarted"), "failed control call entered result execution"); + } finally { + task.close(); + } + // cancelJobGroupAndFutureJobs permanently marks a group in the real DAGScheduler. + // Reusing this otherwise unused group proves cleanup never submitted that cancellation. + Properties before = spark.sparkContext().getLocalProperties(); + try { + spark.sparkContext().setLocalProperties(new Properties()); + spark.sparkContext().setJobGroup(group, "failed control cancellation probe", false); + require(spark.range(1).repartition(1).collectAsList().size() == 1, + "failed control cleanup cancelled an unused Spark job group"); + } finally { + spark.sparkContext().setLocalProperties(before); + } + } + } + + private static final class AnalysisJob { + private final AtomicBoolean entered = new AtomicBoolean(); + + private void enter(SparkSession spark) { + if (entered.compareAndSet(false, true)) { + // A real analyzer extension can submit Spark work before result fetching begins. + // The atomic guard prevents this test's nested query from recursively submitting jobs. + spark.sql("SELECT sum(id) AS total FROM range(1000000000000)").collectAsList(); + } + } + } + + private static void cancelActiveAnalysisJob(SparkJvmEngine engine, + SparkJvmEngine.Session session, SparkSession spark) throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (SparkJvmEngine.Task task = engine.newTask("analysis job cancellation")) { + String group = (String) privateField(task, "jobGroup"); + AnalysisJob job = new AnalysisJob(); + try { + require(ANALYSIS_JOB.compareAndSet(null, job), "another analysis job probe is active"); + Future analysis = executor.submit( + () -> session.analyze(0, range(1).toByteArray(), task)); + long limit = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (spark.sparkContext().statusTracker().getActiveJobIds().length == 0 + && !analysis.isDone() && System.nanoTime() < limit) Thread.sleep(10); + require(!analysis.isDone() && spark.sparkContext().statusTracker().getActiveJobIds().length > 0, + "analysis probe did not submit a real Spark job"); + require(!flag(task, "executionStarted"), "analysis job probe unexpectedly began fetching"); + task.cancel(); + try { + analysis.get(10, TimeUnit.SECONDS); + throw new AssertionError("analysis Spark job survived cancellation"); + } catch (ExecutionException expected) { + require(expected.getCause() instanceof SparkJvmEngine.EngineException + && ((SparkJvmEngine.EngineException) expected.getCause()).code() == 1, + "analysis job cancellation returned the wrong status: " + expected.getCause()); + } + limit = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (spark.sparkContext().statusTracker().getActiveJobIds().length != 0 + && System.nanoTime() < limit) Thread.sleep(10); + require(spark.sparkContext().statusTracker().getActiveJobIds().length == 0, + "analysis cancellation left a Spark job running"); + Properties before = spark.sparkContext().getLocalProperties(); + try { + spark.sparkContext().setLocalProperties(new Properties()); + spark.sparkContext().setJobGroup(group, "cancelled analysis group probe", false); + try { + spark.range(1).repartition(1).collectAsList(); + throw new AssertionError("analysis cancellation did not cancel future group submissions"); + } catch (Exception expected) { + require(expected.getMessage() != null + && expected.getMessage().toLowerCase(java.util.Locale.ROOT).contains("cancel"), + "analysis group probe failed for another reason: " + expected); + } + } finally { + spark.sparkContext().setLocalProperties(before); + } + } finally { + ANALYSIS_JOB.compareAndSet(job, null); + // Ensure a failed assertion cannot leave this deliberately large test job running. + spark.sparkContext().cancelJobGroupAndFutureJobs(group); + } + } finally { + executor.shutdownNow(); + require(executor.awaitTermination(10, TimeUnit.SECONDS), "analysis job worker did not stop"); + } + } + + private static void metadataOnly(SparkJvmEngine engine, SparkJvmEngine.Session first, + SparkJvmEngine.Session second, SparkSession spark) throws Exception { + AtomicInteger jobs = new AtomicInteger(); + SparkListener listener = new SparkListener() { + @Override + public void onJobStart(SparkListenerJobStart event) { jobs.incrementAndGet(); } + }; + spark.sparkContext().listenerBus().waitUntilEmpty(10000); + spark.sparkContext().addSparkListener(listener); + try { + metadataParity(engine, first); + metadataFailures(engine, first); + schemaInformationCannotBeLost(engine, first); + shuffleCleanupIsRejected(engine, first); + metadataInvalidModeBoundary(engine, first); + metadataConfigGetterFailure(engine, first); + metadataConcurrentReadersAndWriters(engine, first); + metadataCallerContext(engine, first, second, spark); + metadataBlocked(engine, first, second, spark, false); + try (SparkJvmEngine.Session closing = open(engine, "metadata-closing")) { + metadataBlocked(engine, closing, second, spark, true); + for (int opcode = 0; opcode < 3; opcode++) { + int mode = opcode; + try (SparkJvmEngine.Task task = engine.newTask("closed metadata session")) { + expectCode(9, () -> closing.analyze(mode, range(1).toByteArray(), task)); + } + } + } + require(registry(first, "queries").isEmpty(), "metadata left a Query registered"); + require(registry(first, "activeTasks").isEmpty(), "metadata left an active Task"); + require(registry(engine, "tasks").isEmpty(), "metadata left a Task registered"); + } finally { + try { + spark.sparkContext().listenerBus().waitUntilEmpty(10000); + } finally { + spark.sparkContext().removeSparkListener(listener); + } + } + require(jobs.get() == 0, "metadata submitted Spark jobs: " + jobs.get()); + } + + private static void metadataParity(SparkJvmEngine engine, SparkJvmEngine.Session session) + throws Exception { + Relation[] relations = { + range(5), + Relation.newBuilder().setLocalRelation(LocalRelation.newBuilder().setSchema("id INT")).build(), + sql("SELECT true AS flag, CAST(1 AS TINYINT) AS tiny, CAST(2 AS SMALLINT) AS small, " + + "3 AS number, CAST(4 AS BIGINT) AS wide, CAST(1.5 AS FLOAT) AS single, " + + "CAST(2.5 AS DOUBLE) AS double_value, unhex('00ff') AS binary_value"), + sql("SELECT 'hello 😀' AS text, CAST(NULL AS STRING) AS missing, NULL AS untyped, " + + "array(1, CAST(NULL AS INT), 3) AS items, map('x', 7) AS mapping, " + + "named_struct('a', 1, 'b', 'nested') AS nested"), + sql("SELECT CAST('12.34' AS DECIMAL(10,2)) AS amount, DATE '2026-01-02' AS day, " + + "TIMESTAMP '2026-01-02 03:04:05' AS instant, " + + "CAST('2026-01-02 03:04:05' AS TIMESTAMP_NTZ) AS local_time"), + sql("SELECT INTERVAL '1-2' YEAR TO MONTH AS months, " + + "INTERVAL '1 02:03:04' DAY TO SECOND AS duration, make_interval(1) AS calendar"), + sql("SELECT id FROM range(5) WHERE id < 0") + }; + boolean[] expectedLocal = {false, true, false, false, false, false, false}; + require(relations.length == expectedLocal.length, "metadata parity expectations are incomplete"); + boolean sawLocal = false; + boolean sawNonLocal = false; + for (int index = 0; index < relations.length; index++) { + Relation relation = relations[index]; + try (SparkJvmEngine.Query prepared = query(engine, session, relation)) { + boolean local = prepared.isLocal(); + require(local == expectedLocal[index], "prepare IsLocal differs from expected for case " + index); + sawLocal |= local; + sawNonLocal |= !local; + byte[] expected = prepared.schemaIpc(); + for (int opcode = 0; opcode < 3; opcode++) { + byte[] actual = metadata(engine, session, opcode, relation); + require(actual[0] == (opcode == 1 && expectedLocal[index] ? 1 : 0), + "metadata flag differs from expected for case " + index + " mode " + opcode); + require(Arrays.equals(expected, Arrays.copyOfRange(actual, 1, actual.length)), + "metadata changed prepare's schema IPC for case " + index + " mode " + opcode); + } + } + } + require(sawLocal && sawNonLocal, "metadata parity did not cover both IsLocal values"); + } + + private static byte[] metadata(SparkJvmEngine engine, SparkJvmEngine.Session session, + int opcode, Relation relation) throws Exception { + int tasksBefore = registry(engine, "tasks").size(); + int queriesBefore = registry(session, "queries").size(); + byte[] result; + SparkJvmEngine.Task task = engine.newTask("metadata smoke"); + try { + result = session.analyze(opcode, relation.toByteArray(), task); + require(result.length > 1 && (result[0] == 0 || result[0] == 1), + "metadata did not return a strict boolean followed by IPC"); + require(flag(task, "completed"), "successful metadata Task is incomplete"); + task.cancel(); + require(!flag(task, "cancelled"), "completed metadata Task was cancelled"); + require(registry(session, "queries").size() == queriesBefore, + "metadata registered a Query"); + require(registry(session, "activeTasks").isEmpty(), "metadata Task remained active"); + } finally { + task.close(); + } + require(!flag(task, "cancelled"), "closing completed metadata cancelled its job group"); + require(registry(engine, "tasks").size() == tasksBefore, "metadata retained a Task"); + return result; + } + + private static void schemaInformationCannotBeLost(SparkJvmEngine engine, + SparkJvmEngine.Session session) throws Exception { + Metadata tagged = new MetadataBuilder().putString("comment", "must not disappear").build(); + Metadata rawChar = new MetadataBuilder().putString("__autoGeneratedAlias", "true") + .putString("__CHAR_VARCHAR_TYPE_STRING", "char(4)").build(); + Metadata internalAndUser = new MetadataBuilder().withMetadata(tagged) + .putString("__autoGeneratedAlias", "true").build(); + DataType[] unsupported = { + DataTypes.createCharType(4), DataTypes.createVarcharType(4), new VectorUDT(), + DataTypes.createArrayType(DataTypes.createVarcharType(4)), + DataTypes.createMapType(DataTypes.StringType, new VectorUDT()), + new StructType(new StructField[] { + DataTypes.createStructField("annotated", DataTypes.LongType, true, tagged) + }), + DataTypes.createArrayType(new StructType(new StructField[] { + DataTypes.createStructField("nested", DataTypes.StringType, true, tagged) + })), + new StructType(new StructField[] { + DataTypes.createStructField("raw_char", DataTypes.StringType, true, rawChar) + }), + DataTypes.createArrayType(new StructType(new StructField[] { + DataTypes.createStructField("mixed", DataTypes.StringType, true, internalAndUser) + })) + }; + for (DataType type : unsupported) { + expectCode(12, () -> SparkJvmEngine.validateResultType(type)); + } + SparkJvmEngine.validateResultType(new StructType(new StructField[] { + DataTypes.createStructField("plain", DataTypes.createArrayType( + DataTypes.createMapType(DataTypes.StringType, DataTypes.LongType)), true, Metadata.empty()) + })); + for (String query : new String[] { + "SELECT 'a' COLLATE UTF8_LCASE AS value", + "SELECT array('a' COLLATE UTF8_LCASE) AS value", + "SELECT map('key', 'a' COLLATE UTF8_LCASE) AS value"}) { + matchingMetadataFailure(engine, session, sql(query).toByteArray(), 12); + } + for (String expression : new String[] { + "named_struct('a', 1, 'a', 2)", + "array(named_struct('a', 1, 'a', 2))", + "map('key', named_struct('a', 1, 'a', 2))", + "map(named_struct('a', 1, 'a', 2), 'value')"}) { + RuntimeException failure = matchingMetadataFailure(engine, session, + sql("SELECT " + expression + " AS s").toByteArray(), 12); + require(failure.getMessage().contains("duplicate nested field names"), + "duplicate fields failed outside the schema-preservation guard"); + } + Relation alias = Relation.newBuilder().setProject(Project.newBuilder().setInput(range(1)) + .addExpressions(Expression.newBuilder().setAlias(Expression.Alias.newBuilder() + .setExpr(Expression.newBuilder().setUnresolvedAttribute( + Expression.UnresolvedAttribute.newBuilder().setUnparsedIdentifier("id"))) + .addName("annotated").setMetadata(tagged.json())))).build(); + matchingMetadataFailure(engine, session, alias.toByteArray(), 12); + SparkSession sessionSpark = (SparkSession) privateField(session, "spark"); + Metadata generated = sessionSpark.sql("SELECT sum(id) FROM range(1)") + .schema().fields()[0].metadata(); + require(generated.contains("__autoGeneratedAlias") + && generated.getString("__autoGeneratedAlias").equals("true"), + "unaliased aggregate did not exercise Spark's generated-alias metadata"); + try (SparkJvmEngine.Query unaliased = query(engine, session, + sql("SELECT sum(id) FROM range(1)"))) { + require(unaliased.schemaIpc().length > 0, "internal alias metadata rejected preparation"); + } + SparkJvmEngine.validateResultType(DataTypes.createArrayType(new StructType(new StructField[] { + DataTypes.createStructField("internal", DataTypes.StringType, true, generated) + }))); + require(sessionSpark.sql("SELECT sum(id) AS total FROM range(1)") + .schema().fields()[0].metadata().isEmpty(), + "explicit aggregate alias retained unsupported generated metadata"); + String view = "metadata_udt_" + UUID.randomUUID().toString().replace("-", ""); + StructType udtSchema = new StructType(new StructField[] { + DataTypes.createStructField("features", new VectorUDT(), true) + }); + sessionSpark.createDataFrame(java.util.Collections.emptyList(), udtSchema) + .createOrReplaceTempView(view); + try { + matchingMetadataFailure(engine, session, sql("SELECT * FROM " + view).toByteArray(), 12); + } finally { + sessionSpark.catalog().dropTempView(view); + } + } + + private static void duplicateFieldNameResults(SparkJvmEngine engine, + SparkJvmEngine.Session session) throws Exception { + for (boolean nested : new boolean[] {false, true}) { + String statement = nested ? "SELECT named_struct('a', 1, 'A', 2) AS s" + : "SELECT 1 AS a, 2 AS a"; + try (SparkJvmEngine.Query query = query(engine, session, sql(statement)); + BufferAllocator allocator = allocator(); + ArrowStreamReader schema = new ArrowStreamReader( + new ByteArrayInputStream(query.schemaIpc()), allocator)) { + var fields = schema.getVectorSchemaRoot().getSchema().getFields(); + if (nested) fields = fields.get(0).getChildren(); + require(fields.size() == 2 && fields.get(0).getName().equals("a") + && fields.get(1).getName().equals(nested ? "A" : "a"), + "supported duplicate or case-distinct field names changed"); + for (int opcode = 0; opcode < 3; opcode++) { + byte[] analyzed = metadata(engine, session, opcode, sql(statement)); + try (ArrowStreamReader reader = new ArrowStreamReader( + new ByteArrayInputStream(Arrays.copyOfRange(analyzed, 1, analyzed.length)), allocator)) { + require(reader.getVectorSchemaRoot().getSchema().equals( + schema.getVectorSchemaRoot().getSchema()), "field-name analysis changed the schema"); + } + } + NativeIpcBatch batch = query.nextIpcBatch(); + require(batch != null && batch.rowCount() == 1, "field-name control lost its row"); + try (ArrowStreamReader reader = new ArrowStreamReader( + new ByteArrayInputStream(batch.data()), allocator)) { + require(reader.loadNextBatch(), "field-name control has no Arrow batch"); + var root = reader.getVectorSchemaRoot(); + require(root.getSchema().equals(schema.getVectorSchemaRoot().getSchema()), + "field-name execution changed the declared schema"); + Object first = root.getVector(0).getObject(0); + Object second = nested ? ((Map) first).get("A") : root.getVector(1).getObject(0); + if (nested) first = ((Map) first).get("a"); + require(Integer.valueOf(1).equals(first) && Integer.valueOf(2).equals(second), + "field-name control changed values"); + require(!reader.loadNextBatch(), "field-name control has extra Arrow batches"); + } + require(query.nextIpcBatch() == null, "field-name control has extra IPC batches"); + } + } + } + + private static void shuffleCleanupIsRejected(SparkJvmEngine engine, + SparkJvmEngine.Session session) throws Exception { + String key = "spark.sql.classic.shuffleDependency.fileCleanup.enabled"; + String before = config(engine, session, 4).get(key); + config(engine, session, 6, key, "true"); + try { + matchingMetadataFailure(engine, session, range(1).toByteArray(), 12); + } finally { + config(engine, session, before == null ? 7 : 6, key, before); + } + try (SparkJvmEngine.Query recovered = query(engine, session, range(1))) { + require(recovered.schemaIpc().length > 0, "shuffle-cleanup rejection poisoned the session"); + } + } + + private static void releasedSessionPreparation(SparkJvmEngine engine) { + SparkJvmEngine.Session released = open(engine, "released-prepare"); + released.close(); + try (SparkJvmEngine.Task task = engine.newTask("released preparation")) { + expectCode(9, () -> released.prepare(range(1).toByteArray(), task)); + } + } + + private static void metadataFailures(SparkJvmEngine engine, SparkJvmEngine.Session session) + throws Exception { + Expression udf = Expression.newBuilder().setCommonInlineUserDefinedFunction( + CommonInlineUserDefinedFunction.newBuilder().setFunctionName("forbidden_metadata_udf")) + .build(); + Expression typedUdf = Expression.newBuilder().setTypedAggregateExpression( + TypedAggregateExpression.getDefaultInstance()).build(); + Method guard = SparkJvmEngine.class.getDeclaredMethod("validateRelations", Message.class); + guard.setAccessible(true); + // Empty messages exercise the guard only; no function payload is deserialized or run. + for (Message function : new Message[] { + TypedAggregateExpression.getDefaultInstance(), ScalarScalaUDF.getDefaultInstance()}) { + try { + guard.invoke(null, function); + throw new AssertionError("typed UDF message passed the supported-operation guard"); + } catch (InvocationTargetException expected) { + require(expected.getCause() instanceof SparkJvmEngine.EngineException + && ((SparkJvmEngine.EngineException) expected.getCause()).code() == 12, + "typed UDF guard failed for another reason: " + expected.getCause()); + } + } + Relation nestedUdf = Relation.newBuilder().setProject(Project.newBuilder().setInput(range(1)) + .addExpressions(Expression.newBuilder().setUnresolvedFunction( + Expression.UnresolvedFunction.newBuilder().setFunctionName("coalesce") + .addArguments(udf)))).build(); + Relation references = Relation.newBuilder().setWithRelations(WithRelations.newBuilder() + .setRoot(sql("CREATE TEMP VIEW metadata_forbidden_refs AS SELECT 1 AS id")) + .addReferences(Relation.newBuilder().setSubqueryAlias(SubqueryAlias.newBuilder() + .setAlias("unused_metadata_ref").setInput(range(1))))).build(); + for (Relation relation : new Relation[] { + sql("CREATE TEMP VIEW metadata_forbidden_command AS SELECT 1 AS id"), + sql("BEGIN CREATE TEMP VIEW metadata_forbidden_script AS SELECT 1 AS id; END"), + references, nestedUdf, + Relation.newBuilder().setProject(Project.newBuilder().setInput(range(1)) + .addExpressions(typedUdf)).build()}) { + matchingMetadataFailure(engine, session, relation.toByteArray(), 12); + } + missing(engine, session, "metadata_forbidden_command"); + missing(engine, session, "metadata_forbidden_script"); + missing(engine, session, "metadata_forbidden_refs"); + matchingMetadataFailure(engine, session, sql("SELECT missing FROM range(1)").toByteArray(), 3); + matchingMetadataFailure(engine, session, sql("SELECT FROM").toByteArray(), 3); + matchingMetadataFailure(engine, session, new byte[] {0}, 3); + Path source = Files.createTempDirectory("native-unsupported-read-"); + try { + // If the guard regresses, planning reaches this missing local path instead of returning 12. + Relation dataSource = Relation.newBuilder().setRead(Read.newBuilder() + .setDataSource(Read.DataSource.newBuilder().setFormat("parquet") + .addPaths(source.resolve("missing.parquet").toUri().toString()))).build(); + RuntimeException failure = + matchingMetadataFailure(engine, session, dataSource.toByteArray(), 12); + require(failure.getMessage().contains("only named-table reads"), + "DataSource Read failed outside the named-table guard"); + } finally { + Files.delete(source); + } + SparkSession sessionSpark = (SparkSession) privateField(session, "spark"); + sessionSpark.readStream().format("rate").load().createOrReplaceTempView("metadata_stream"); + try { + matchingMetadataFailure(engine, session, sql("SELECT * FROM metadata_stream").toByteArray(), 12); + } finally { + sessionSpark.catalog().dropTempView("metadata_stream"); + } + for (int opcode = 0; opcode < 3; opcode++) { + int mode = opcode; + try (SparkJvmEngine.Task task = engine.newTask("pre-cancel metadata")) { + task.cancel(); + expectCode(1, () -> session.analyze(mode, range(1).toByteArray(), task)); + require(!flag(task, "completed"), "pre-cancelled metadata completed"); + } + } + for (int opcode : new int[] {-1, 3}) { + try (SparkJvmEngine.Task task = engine.newTask("invalid metadata mode")) { + expectCode(12, () -> session.analyze(opcode, range(1).toByteArray(), task)); + } + } + } + + private static RuntimeException matchingMetadataFailure(SparkJvmEngine engine, + SparkJvmEngine.Session session, byte[] relation, int code) throws Exception { + RuntimeException expected; + try (SparkJvmEngine.Task task = engine.newTask("prepare expected failure")) { + try (SparkJvmEngine.Query ignored = session.prepare(relation, task)) { + throw new AssertionError("prepare unexpectedly accepted invalid metadata input"); + } catch (RuntimeException error) { + expected = error; + } + } + if (code >= 0) { + require(expected instanceof SparkJvmEngine.EngineException + && ((SparkJvmEngine.EngineException) expected).code() == code, + "prepare produced an unexpected failure: " + expected); + } + for (int opcode = 0; opcode < 3; opcode++) { + try (SparkJvmEngine.Task task = engine.newTask("metadata expected failure")) { + try { + session.analyze(opcode, relation, task); + throw new AssertionError("metadata accepted a plan rejected by prepare"); + } catch (RuntimeException error) { + require(error.getClass() == expected.getClass(), "metadata changed the failure type"); + if (expected instanceof SparkJvmEngine.EngineException) { + require(((SparkJvmEngine.EngineException) error).code() + == ((SparkJvmEngine.EngineException) expected).code(), + "metadata changed the engine failure code"); + } + } + require(!flag(task, "completed"), "failed metadata Task was marked complete"); + require(!flag(task, "executionStarted"), "failed metadata entered query execution"); + } + } + require(registry(session, "queries").isEmpty(), "failed metadata retained a Query"); + require(registry(session, "activeTasks").isEmpty(), "failed metadata retained an active Task"); + return expected; + } + + @SuppressWarnings("unchecked") + private static void metadataConfigGetterFailure(SparkJvmEngine engine, + SparkJvmEngine.Session session) throws Exception { + org.apache.spark.sql.classic.SparkSession sessionSpark = + (org.apache.spark.sql.classic.SparkSession) privateField(session, "spark"); + Map settings = (Map) privateField( + sessionSpark.sessionState().conf(), "settings"); + String key = "spark.sql.execution.arrow.maxRecordsPerBatch"; + // Bypass setter validation only in this test to exercise the shared typed getter boundary. + String previous = settings.put(key, "invalid-metadata-batch-size"); + try { + matchingMetadataFailure(engine, session, range(1).toByteArray(), -1); + } finally { + if (previous == null) settings.remove(key); + else settings.put(key, previous); + } + } + + private static void metadataCallerContext(SparkJvmEngine engine, SparkJvmEngine.Session first, + SparkJvmEngine.Session second, SparkSession spark) throws Exception { + String key = "spark.sql.timestampType"; + String firstBefore = config(engine, first, 4).get(key); + String secondBefore = config(engine, second, 4).get(key); + try { + config(engine, first, 6, key, "TIMESTAMP_LTZ"); + config(engine, second, 6, key, "TIMESTAMP_NTZ"); + Relation relation = sql("SELECT CAST('2026-01-02 03:04:05' AS TIMESTAMP) AS instant"); + byte[] expected; + try (SparkJvmEngine.Query prepared = query(engine, first, relation)) { + expected = prepared.schemaIpc(); + } + SparkSession caller = (SparkSession) privateField(second, "spark"); + for (boolean interrupted : new boolean[] {false, true}) { + try (SparkJvmEngine.Task task = engine.newTask("metadata caller context")) { + callerContext(spark, caller, interrupted, () -> { + byte[] result = first.analyze(0, relation.toByteArray(), task); + require(Arrays.equals(expected, Arrays.copyOfRange(result, 1, result.length)), + "metadata used the caller's SQL configuration"); + }); + } + try (SparkJvmEngine.Task task = engine.newTask("failed metadata caller context")) { + callerContext(spark, caller, interrupted, () -> expectCode(12, () -> first.analyze(0, + sql("CREATE TEMP VIEW metadata_context_forbidden AS SELECT 1").toByteArray(), task))); + } + } + } finally { + config(engine, first, firstBefore == null ? 7 : 6, key, firstBefore); + config(engine, second, secondBefore == null ? 7 : 6, key, secondBefore); + } + } + + private static void callerContext(SparkSession spark, SparkSession caller, + boolean interrupted, Runnable action) throws Exception { + Thread thread = Thread.currentThread(); + boolean previousInterrupted = Thread.interrupted(); + ClassLoader previousLoader = thread.getContextClassLoader(); + Properties previousProperties = spark.sparkContext().getLocalProperties(); + SparkEnv previousEnv = SparkEnv.get(); + Option previousSession = SparkSession.getActiveSession(); + ClassLoader callerLoader = new ClassLoader(previousLoader) {}; + Properties callerProperties = new Properties(); + callerProperties.setProperty("metadata.caller.context", "preserved"); + try (AutoCloseable restore = () -> { + Thread.interrupted(); + spark.sparkContext().setLocalProperties(previousProperties); + thread.setContextClassLoader(previousLoader); + if (previousSession.isDefined()) SparkSession.setActiveSession(previousSession.get()); + else SparkSession.clearActiveSession(); + SparkEnv.set(previousEnv); + if (previousInterrupted) thread.interrupt(); + }) { + thread.setContextClassLoader(callerLoader); + spark.sparkContext().setLocalProperties(callerProperties); + SparkSession.setActiveSession(caller); + if (interrupted) thread.interrupt(); + try (AutoCloseable verify = () -> { + require(thread.getContextClassLoader() == callerLoader, "metadata changed caller classloader"); + require(spark.sparkContext().getLocalProperties().equals(callerProperties), + "metadata changed caller local properties"); + Option active = SparkSession.getActiveSession(); + require(active.isDefined() && active.get() == caller, "metadata changed caller active session"); + // SparkEnv is process-global in the pinned Spark assembly. This checks stability of the + // current global environment, not restoration of a distinct caller environment. Replacing + // it with a sentinel while Spark background threads run would affect those threads too. + require(SparkEnv.get() == previousEnv, "metadata changed the current global SparkEnv"); + require(thread.isInterrupted() == interrupted, "metadata changed caller interrupt state"); + }) { + action.run(); + } + } + } + + private static void metadataInvalidModeBoundary(SparkJvmEngine engine, + SparkJvmEngine.Session session) throws Exception { + ReentrantReadWriteLock catalog = (ReentrantReadWriteLock) privateField(session, "catalog"); + ExecutorService executor = Executors.newSingleThreadExecutor(); + catalog.writeLock().lock(); + try { + Future invalid = executor.submit(() -> { + try (SparkJvmEngine.Task task = engine.newTask("invalid metadata mode")) { + expectCode(12, () -> session.analyze(3, range(1).toByteArray(), task)); + } + return null; + }); + invalid.get(10, TimeUnit.SECONDS); + try (SparkJvmEngine.Task task = engine.newTask("cancelled invalid metadata mode")) { + task.cancel(); + expectCode(1, () -> session.analyze(3, range(1).toByteArray(), task)); + } + } finally { + catalog.writeLock().unlock(); + executor.shutdownNow(); + require(executor.awaitTermination(10, TimeUnit.SECONDS), "invalid metadata worker did not stop"); + } + } + + /** Test-only rule barrier: each registered reader waits on its first analyzer invocation. */ + private static final class AnalysisGate implements AutoCloseable { + private final CountDownLatch entered; + private final CountDownLatch proceed = new CountDownLatch(1); + private final Set readers = ConcurrentHashMap.newKeySet(); + private final Set threads = ConcurrentHashMap.newKeySet(); + + private AnalysisGate(int readers) { + entered = new CountDownLatch(readers); + require(ANALYSIS_GATE.compareAndSet(null, this), "another analysis gate is active"); + } + + private void enter() { + Thread thread = Thread.currentThread(); + if (!readers.contains(thread) || !threads.add(thread)) return; + entered.countDown(); + try { + require(proceed.await(30, TimeUnit.SECONDS), "analysis gate was not released"); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("analysis gate interrupted", error); + } + } + + @Override + public void close() { + ANALYSIS_GATE.compareAndSet(this, null); + proceed.countDown(); + } + } + + private static byte[] concurrentMetadata(SparkJvmEngine engine, SparkJvmEngine.Session session, + Relation relation) { + try (SparkJvmEngine.Task task = engine.newTask("concurrent metadata")) { + return session.analyze(0, relation.toByteArray(), task); + } + } + + private static void awaitCatalogWaiter(ReentrantReadWriteLock catalog, + AtomicReference worker, Future result, String operation) { + long limit = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while ((worker.get() == null || !catalog.hasQueuedThread(worker.get())) + && !result.isDone() && System.nanoTime() < limit) { + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1)); + } + require(worker.get() != null && catalog.hasQueuedThread(worker.get()) && !result.isDone(), + operation + " did not wait for active metadata readers"); + } + + private static void metadataConcurrentReadersAndWriters(SparkJvmEngine engine, + SparkJvmEngine.Session session) throws Exception { + String key = "spark-connect.concurrent.metadata"; + config(engine, session, 6, key, "before"); + Relation relation = sql("SELECT id FROM range(5)"); + byte[] expected = metadata(engine, session, 0, relation); + ReentrantReadWriteLock catalog = (ReentrantReadWriteLock) privateField(session, "catalog"); + ExecutorService executor = Executors.newFixedThreadPool(4); + try (AnalysisGate gate = new AnalysisGate(3)) { + Future first = executor.submit(() -> { + gate.readers.add(Thread.currentThread()); + return concurrentMetadata(engine, session, relation); + }); + Future second = executor.submit(() -> { + gate.readers.add(Thread.currentThread()); + return concurrentMetadata(engine, session, relation); + }); + Future prepare = executor.submit(() -> { + gate.readers.add(Thread.currentThread()); + try (SparkJvmEngine.Query prepared = query(engine, session, relation)) { + require(prepared.schemaIpc().length > 0, "concurrent prepare lost its schema"); + } + return null; + }); + require(gate.entered.await(10, TimeUnit.SECONDS), + "same-session metadata and preparation did not enter Spark concurrently"); + + Future> read = executor.submit( + () -> config(engine, session, 1, key, null)); + require("before".equals(read.get(10, TimeUnit.SECONDS).get(key)), + "Config read did not overlap metadata or observed a premature mutation"); + + AtomicReference mutationThread = new AtomicReference<>(); + Future mutation = executor.submit(() -> { + mutationThread.set(Thread.currentThread()); + config(engine, session, 6, key, "after"); + return null; + }); + awaitCatalogWaiter(catalog, mutationThread, mutation, "Config mutation"); + + gate.close(); + require(Arrays.equals(expected, first.get(10, TimeUnit.SECONDS)), + "first concurrent metadata changed its schema"); + require(Arrays.equals(expected, second.get(10, TimeUnit.SECONDS)), + "second concurrent metadata changed its schema"); + mutation.get(10, TimeUnit.SECONDS); + prepare.get(10, TimeUnit.SECONDS); + require("after".equals(config(engine, session, 1, key, null).get(key)), + "Config mutation did not commit after metadata readers drained"); + } finally { + executor.shutdownNow(); + require(executor.awaitTermination(10, TimeUnit.SECONDS), + "concurrent metadata workers did not stop"); + config(engine, session, 7, key, null); + } + require(registry(session, "activeTasks").isEmpty(), "concurrent metadata retained active Tasks"); + require(registry(session, "queries").isEmpty(), "concurrent prepare retained a Query"); + } + + private static void metadataBlocked(SparkJvmEngine engine, SparkJvmEngine.Session session, + SparkJvmEngine.Session callerSession, SparkSession spark, boolean closeSession) throws Exception { + ReentrantReadWriteLock catalog = (ReentrantReadWriteLock) privateField(session, "catalog"); + Set active = registry(session, "activeTasks"); + AtomicBoolean released = (AtomicBoolean) privateField(session, "released"); + SparkSession caller = (SparkSession) privateField(callerSession, "spark"); + AtomicReference worker = new AtomicReference<>(); + ExecutorService executor = Executors.newFixedThreadPool(2); + AtomicReference> close = new AtomicReference<>(); + try (AutoCloseable shutdownExecutor = () -> { + executor.shutdownNow(); + require(executor.awaitTermination(10, TimeUnit.SECONDS), "metadata workers did not stop"); + }; + AutoCloseable awaitClose = () -> { + Future closing = close.get(); + if (closing != null) closing.get(10, TimeUnit.SECONDS); + }; + SparkJvmEngine.Task task = engine.newTask("blocked metadata")) { + catalog.writeLock().lock(); + try (AutoCloseable unlock = catalog.writeLock()::unlock) { + Future analysis = executor.submit(() -> { + worker.set(Thread.currentThread()); + callerContext(spark, caller, false, + () -> expectCode(1, () -> session.analyze(0, range(1).toByteArray(), task))); + return null; + }); + long limit = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while ((worker.get() == null || !active.contains(task) + || !catalog.hasQueuedThread(worker.get())) && !analysis.isDone() + && System.nanoTime() < limit) Thread.sleep(5); + require(worker.get() != null && active.contains(task) && catalog.hasQueuedThread(worker.get()), + "metadata did not wait on the catalog lock"); + if (closeSession) { + Future closing = executor.submit(session::close); + close.set(closing); + limit = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!released.get() && !closing.isDone() && System.nanoTime() < limit) Thread.sleep(5); + require(released.get(), "session close did not publish release"); + } else { + task.cancel(); + } + analysis.get(10, TimeUnit.SECONDS); + require(!flag(task, "completed"), "blocked cancelled metadata completed"); + require(active.isEmpty(), "blocked metadata retained an active Task"); + } + } + } + + private static Object privateField(Object object, String name) throws ReflectiveOperationException { + for (Class type = object.getClass(); type != null; type = type.getSuperclass()) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field.get(object); + } catch (NoSuchFieldException missing) { + // Scala implementation fields can live on a superclass. + } + } + throw new NoSuchFieldException(name); + } + + private static Set registry(Object object, String name) throws ReflectiveOperationException { + return (Set) privateField(object, name); + } + + private static boolean flag(Object object, String name) throws ReflectiveOperationException { + return (Boolean) privateField(object, name); + } + + private static long[] consume(SparkJvmEngine.Query query) throws Exception { + long rows = 0; + long sum = 0; + NativeIpcBatch batch; + while ((batch = query.nextIpcBatch()) != null) { + try (BufferAllocator allocator = allocator(); + ArrowStreamReader reader = new ArrowStreamReader( + new ByteArrayInputStream(batch.data()), allocator)) { + require(reader.loadNextBatch(), "result IPC has no record batch"); + var root = reader.getVectorSchemaRoot(); + require(root.getRowCount() == batch.rowCount(), "IPC row count differs from its header"); + for (int row = 0; row < root.getRowCount(); row++) { + rows++; + Object value = root.getVector(0).getObject(row); + if (value != null) sum += ((Number) value).longValue(); + } + require(!reader.loadNextBatch(), "IPC contains multiple record batches"); + } + } + return new long[] {rows, sum}; + } + + private static BufferAllocator allocator() { + return ArrowUtils$.MODULE$.rootAllocator().newChildAllocator("engine-smoke", 0, Long.MAX_VALUE); + } + + private static Map config(SparkJvmEngine engine, SparkJvmEngine.Session session, + int opcode, String... alternating) throws Exception { + try (SparkJvmEngine.Task task = engine.newTask("config smoke")) { + return decode(session.config(opcode, encode(alternating), task)); + } + } + + private static void configRedaction(SparkJvmEngine engine, SparkJvmEngine.Session session) + throws Exception { + org.apache.spark.sql.classic.SparkSession sessionSpark = + (org.apache.spark.sql.classic.SparkSession) privateField(session, "spark"); + String patternKey = "spark.redaction.regex"; + String legacyKey = "spark.sql.legacy.setCommandRejectsSparkCoreConfs"; + String previousPattern = sessionSpark.conf().getAllAsJava().get(patternKey); + String previousLegacy = sessionSpark.conf().getAllAsJava().get(legacyKey); + String suffix = UUID.randomUUID().toString(); + String hiddenPrefix = "spark.connect.native_hidden." + suffix + "."; + String hiddenKey = hiddenPrefix + "value"; + String visiblePrefix = "spark.connect.redaction." + suffix + "."; + String valueKey = visiblePrefix + "value"; + String controlKey = visiblePrefix + "control"; + // Match Spark Connect's own regression: clients can enable legacy core-conf writes. + config(engine, session, 6, legacyKey, "false"); + try { + Map missingErrors = new LinkedHashMap<>(); + for (String key : new String[] {hiddenKey, valueKey}) { + byte[] fields = encode(key, null); + try (SparkJvmEngine.Task task = engine.newTask("missing config get")) { + missingErrors.put(key, expectCode(5, () -> session.config(1, fields, task)).getMessage()); + } + } + config(engine, session, 6, hiddenKey, "fake-key-marker", valueKey, + "native_hidden_fake_value", controlKey, "visible-marker", patternKey, "a^"); + // A session-level override cannot disable the hosting SparkContext's pattern. + require("a^".equals(sessionSpark.conf().get(patternKey)), "session pattern was not changed"); + for (String key : new String[] {hiddenKey, valueKey}) { + byte[] fields = encode(key, null); + try (SparkJvmEngine.Task task = engine.newTask("redacted config get")) { + SparkJvmEngine.EngineException redacted = + expectCode(5, () -> session.config(1, fields, task)); + require(missingErrors.get(key).equals(redacted.getMessage()), + "redacted Get was distinguishable from the same unset key"); + } + Map optional = config(engine, session, 2, key, null); + require(optional.containsKey(key) && optional.get(key) == null, + "GetOption exposed a redacted fake marker"); + require("caller-default".equals(config(engine, session, 3, key, "caller-default").get(key)), + "GetWithDefault did not return the caller's fallback"); + require(config(engine, session, 3, key, null).get(key) == null, + "GetWithDefault did not preserve an absent fallback"); + } + Map all = config(engine, session, 4); + require(!all.containsKey(hiddenKey) && !all.containsKey(valueKey) + && "visible-marker".equals(all.get(controlKey)), + "GetAll did not filter full keys and values before returning configuration"); + require(config(engine, session, 4, hiddenPrefix, null).isEmpty(), + "GetAll stripped the matching prefix before redacting"); + Map prefixed = config(engine, session, 4, visiblePrefix, null); + require(prefixed.size() == 1 && "visible-marker".equals(prefixed.get("control")), + "prefixed GetAll exposed a fake value marker or changed visible keys"); + } finally { + try { + config(engine, session, 7, hiddenKey, null, valueKey, null, controlKey, null); + } finally { + try { + config(engine, session, previousPattern == null ? 7 : 6, patternKey, previousPattern); + } finally { + config(engine, session, previousLegacy == null ? 7 : 6, legacyKey, previousLegacy); + } + } + } + } + + private static void configFieldCounts(SparkJvmEngine engine, SparkJvmEngine.Session session) + throws Exception { + String[] fields = new String[10001 * 2]; + for (int index = 0; index < fields.length; index += 2) { + fields[index] = "spark.connect.config-count." + index; + } + Map values = config(engine, session, 2, fields); + require(values.size() == 10001 && values.values().stream().allMatch(value -> value == null), + "valid config request above 10000 fields was rejected or changed"); + for (byte[] malformed : new byte[][] {{-1, -1, -1, -1}, {0, 0, 0, 1}}) { + try (SparkJvmEngine.Task task = engine.newTask("malformed config count")) { + expectCode(3, () -> session.config(2, malformed, task)); + } + } + } + + private static void configAtomicityAndIsolation(SparkJvmEngine engine, + SparkJvmEngine.Session first, SparkJvmEngine.Session second) throws Exception { + config(engine, first, 6, "spark-connect.smoke", "hello 😀"); + require(config(engine, first, 1, "spark-connect.smoke", null).get("spark-connect.smoke").equals("hello 😀"), + "config UTF-8 changed"); + require(config(engine, second, 2, "spark-connect.smoke", null).get("spark-connect.smoke") == null, + "configuration leaked between sessions"); + require(config(engine, first, 3, "spark-connect.default", "fallback").get("spark-connect.default").equals("fallback"), + "config default was ignored"); + String nullableDefault = "spark-connect.null-default." + UUID.randomUUID(); + Map nullable = config(engine, first, 3, nullableDefault, null); + require(nullable.containsKey(nullableDefault) && nullable.get(nullableDefault) == null, + "missing config with null default did not return null"); + require("hello 😀".equals(config(engine, first, 3, "spark-connect.smoke", null).get("spark-connect.smoke")), + "null default hid an existing config value"); + String prefix = "spark-connect.prefix." + UUID.randomUUID() + "."; + String neighboring = prefix.substring(0, prefix.length() - 1) + "-neighbor"; + config(engine, first, 6, prefix + "z", "last", prefix + "a", "first", neighboring, "excluded"); + try { + Map prefixed = config(engine, first, 4, prefix, null); + require(prefixed.equals(Map.of("a", "first", "z", "last")), + "GetAll prefix did not filter and strip keys: " + prefixed); + require(new java.util.ArrayList<>(prefixed.keySet()).equals(java.util.List.of("a", "z")), + "GetAll prefix keys were not sorted: " + prefixed.keySet()); + } finally { + config(engine, first, 7, prefix + "z", null, prefix + "a", null, neighboring, null); + } + require(config(engine, first, 5, "spark.sql.shuffle.partitions", null) + .get("spark.sql.shuffle.partitions").equals("true"), "modifiable config is marked static"); + String custom = "spark.connect.native.smoke." + UUID.randomUUID(); + require("false".equals(config(engine, first, 5, custom, null).get(custom)), + "unregistered key unexpectedly reports a registered SQL config"); + config(engine, first, 6, custom, "unregistered"); + require("unregistered".equals(config(engine, first, 1, custom, null).get(custom)), + "RuntimeConfig rejected an unregistered spark.* key"); + require(config(engine, second, 2, custom, null).get(custom) == null, + "unregistered spark.* configuration leaked between sessions"); + Map beforeStatic = config(engine, first, 4); + for (int opcode : new int[] {6, 7}) { + try { + config(engine, first, opcode, custom, opcode == 6 ? "changed" : null, + "spark.sql.warehouse.dir", opcode == 6 ? "/unused-native-smoke" : null); + throw new AssertionError("static config mutation was accepted"); + } catch (SparkJvmEngine.EngineException expected) { + require(expected.code() == 3, "static config did not return INVALID_ARGUMENT"); + for (Throwable error = expected; error != null; error = error.getCause()) { + require(error.getSuppressed().length == 0, + "static config rejection added a redundant rollback failure: " + error); + } + require(config(engine, first, 4).equals(beforeStatic), + "failed RuntimeConfig set/unset did not restore an unregistered spark.* key"); + } + } + config(engine, first, 7, custom, null); + require(config(engine, first, 2, custom, null).get(custom) == null, + "RuntimeConfig could not unset an unregistered spark.* key"); + Map before = config(engine, first, 4); + try { + config(engine, first, 6, "spark-connect.smoke", "mutated", "spark-connect.rollback.new", "new", + "spark.sql.shuffle.partitions", "not-a-number"); + throw new AssertionError("invalid config batch was accepted"); + } catch (RuntimeException expected) { + require(config(engine, first, 4).equals(before), "failed config batch was not rolled back"); + } + config(engine, first, 7, "spark-connect.smoke", null); + require(config(engine, first, 2, "spark-connect.smoke", null).get("spark-connect.smoke") == null, + "config unset failed"); + try { + config(engine, first, 1, "spark-connect.missing." + UUID.randomUUID(), null); + throw new AssertionError("missing config key was accepted"); + } catch (SparkJvmEngine.EngineException expected) { + require(expected.code() == 5, "missing config key did not return NOT_FOUND"); + } + try { + config(engine, first, 6, "spark.sql.shuffle.partitions", "invalid-integer"); + throw new AssertionError("invalid config value was accepted"); + } catch (SparkJvmEngine.EngineException expected) { + require(expected.code() == 3, "invalid config did not return INVALID_ARGUMENT"); + } + } + + private static void configDependentRollback(SparkJvmEngine engine, + SparkJvmEngine.Session session) throws Exception { + String legacy = "spark.sql.legacy.setCommandRejectsSparkCoreConfs"; + String core = "spark.driver.maxResultSize"; + Map initial = config(engine, session, 4); + try { + for (String originalCore : new String[] {null, "512m"}) { + for (String originalLegacy : new String[] {null, "false", "true"}) { + config(engine, session, 6, legacy, "false"); + config(engine, session, originalCore == null ? 7 : 6, core, originalCore); + config(engine, session, originalLegacy == null ? 7 : 6, legacy, originalLegacy); + Map before = config(engine, session, 4); + String[] changes = "false".equals(originalLegacy) + ? new String[] {core, "2g", legacy, "true", + "spark.sql.shuffle.partitions", "invalid-integer"} + : new String[] {legacy, "false", core, "2g", legacy, "true", + "spark.sql.shuffle.partitions", "invalid-integer"}; + try { + config(engine, session, 6, changes); + throw new AssertionError("dependent config batch was accepted"); + } catch (SparkJvmEngine.EngineException expected) { + require(expected.code() == 3 + && expected.getMessage().contains("spark.sql.shuffle.partitions") + && expected.getMessage().contains("invalid-integer"), + "dependent config failed before the invalid shuffle setting"); + require(expected.getSuppressed().length == 0 + && expected.getCause().getSuppressed().length == 0, + "dependent config rollback added cleanup failures"); + } + require(before.equals(config(engine, session, 4)), + "dependent config batch did not restore explicit values and unset keys"); + } + } + config(engine, session, 6, legacy, "false", core, "512m"); + Map beforeUnset = config(engine, session, 4); + try { + config(engine, session, 7, core, null, legacy, null, "spark.sql.warehouse.dir", null); + throw new AssertionError("dependent config unset was accepted"); + } catch (SparkJvmEngine.EngineException expected) { + require(expected.code() == 3 && expected.getMessage().contains("spark.sql.warehouse.dir"), + "dependent unset failed before the static setting"); + require(expected.getSuppressed().length == 0 + && expected.getCause().getSuppressed().length == 0, + "dependent unset rollback added cleanup failures"); + } + require(beforeUnset.equals(config(engine, session, 4)), + "dependent config unset did not restore the original settings"); + } finally { + config(engine, session, 6, legacy, "false"); + config(engine, session, initial.get(core) == null ? 7 : 6, core, initial.get(core)); + config(engine, session, initial.get(legacy) == null ? 7 : 6, legacy, initial.get(legacy)); + } + require(initial.equals(config(engine, session, 4)), "dependent config test leaked settings"); + } + + private static void cancelledBeforePrepare(SparkJvmEngine engine, SparkJvmEngine.Session session) { + try (SparkJvmEngine.Task task = engine.newTask("pre-cancel")) { + task.cancel(); + expectCode(1, () -> session.prepare(range(1).toByteArray(), task)); + } + } + + private static void failedTaskScopeCaptureCanRetry(SparkJvmEngine engine, + SparkJvmEngine.Session session) throws Exception { + byte[] empty = encode(); + for (RuntimeException failure : new RuntimeException[] { + new IllegalArgumentException("expected scope failure"), + new NoSuchElementException("expected scope failure")}) { + AtomicReference unexpected = new AtomicReference<>(); + try (SparkJvmEngine.Task task = engine.newTask("scope capture recovery")) { + Thread caller = new Thread(() -> { + try { + Thread.currentThread().interrupt(); + expectCode(13, () -> session.config(4, empty, task)); + require(Thread.interrupted(), "failed scope capture lost the caller interrupt"); + require(privateField(task, "running") == null, "failed scope capture retained running"); + require(!flag(task, "completed"), "failed scope capture completed the task"); + session.config(4, empty, task); + require(flag(task, "completed"), "scope capture failure prevented task reuse"); + } catch (Throwable error) { + unexpected.set(error); + } + }, "scope-capture-recovery") { + private boolean fail = true; + @Override + public ClassLoader getContextClassLoader() { + if (fail) { + fail = false; + throw failure; + } + return super.getContextClassLoader(); + } + }; + caller.start(); + caller.join(10000); + require(!caller.isAlive(), "scope capture recovery did not finish"); + if (unexpected.get() != null) throw new AssertionError("scope capture recovery", unexpected.get()); + } + } + } + + private static void plannerInputFailures(SparkJvmEngine engine, + SparkJvmEngine.Session session) throws Exception { + Relation invalid = Relation.newBuilder().setDeduplicate(Deduplicate.newBuilder() + .setInput(range(1)).setAllColumnsAsKeys(true).addColumnNames("id")).build(); + try (SparkJvmEngine.Task task = engine.newTask("invalid planner input")) { + try (SparkJvmEngine.Query ignored = session.prepare(invalid.toByteArray(), task)) { + throw new AssertionError("planner accepted incompatible deduplication fields"); + } catch (SparkJvmEngine.EngineException error) { + require(error.code() == 3 && error.getCause() instanceof InvalidPlanInput, + "malformed allowlisted plan did not preserve planner validation: " + error); + require(error.getCause().getMessage().contains("both all columns and a subset"), + "invalid plan failed before the intended planner validation: " + error); + } + } + matchingMetadataFailure(engine, session, invalid.toByteArray(), 3); + } + + @SuppressWarnings("unchecked") + private static void configSetterFailureRollsBackPartialWrite(SparkJvmEngine engine, + SparkJvmEngine.Session session) throws Exception { + org.apache.spark.sql.classic.SparkSession sessionSpark = + (org.apache.spark.sql.classic.SparkSession) privateField(session, "spark"); + Object conf = sessionSpark.sessionState().conf(); + Field settingsField = conf.getClass().getDeclaredField("settings"); + settingsField.setAccessible(true); + Map settings = (Map) settingsField.get(conf); + String key = "spark-connect.partial-config." + UUID.randomUUID(); + String failedKey = key + ".throws"; + String unvisitedKey = key + ".unvisited"; + RuntimeException failure = new IllegalStateException("injected failure after config write"); + RuntimeException rollbackFailure = new IllegalStateException("injected config rollback failure"); + AtomicBoolean fail = new AtomicBoolean(true); + AtomicBoolean interruptSetter = new AtomicBoolean(); + AtomicBoolean failRollback = new AtomicBoolean(); + AtomicInteger unvisitedWrites = new AtomicInteger(); + settings.put(key, "before"); + Map failing = new AbstractMap<>() { + @Override + public Set> entrySet() { return settings.entrySet(); } + @Override + public String get(Object name) { return settings.get(name); } + @Override + public String remove(Object name) { + if (unvisitedKey.equals(name)) unvisitedWrites.incrementAndGet(); + if (failedKey.equals(name) && failRollback.get()) throw rollbackFailure; + return settings.remove(name); + } + @Override + public String put(String name, String value) { + if (unvisitedKey.equals(name)) unvisitedWrites.incrementAndGet(); + String previous = settings.put(name, value); + if (failedKey.equals(name) && fail.compareAndSet(true, false)) { + if (interruptSetter.get()) Thread.currentThread().interrupt(); + throw failure; + } + return previous; + } + }; + try { + settingsField.set(conf, failing); + try { + config(engine, session, 6, key, "changed", failedKey, "changed", unvisitedKey, "unused"); + throw new AssertionError("injected config setter failure was accepted"); + } catch (SparkJvmEngine.EngineException error) { + require(error.code() == 13 && error.getCause() == failure, + "config setter failure changed the original error"); + require(failure.getSuppressed().length == 0, + "config rollback added unexpected cleanup errors"); + } + require("before".equals(settings.get(key)) && !settings.containsKey(failedKey), + "config rollback missed a setter that wrote before throwing"); + require(unvisitedWrites.get() == 0 && !settings.containsKey(unvisitedKey), + "config rollback mutated an unvisited key"); + fail.set(true); + Map warnings = config(engine, session, 8, failedKey, "fake-rejected-value"); + require(warnings.size() == 1 + && IllegalStateException.class.getName().equals(warnings.get(failedKey)), + "silent setter failure lost its provenance or included its rejected value/message"); + require(!settings.containsKey(failedKey), "silent setter failure did not roll back its write"); + require(config(engine, session, 8, unvisitedKey, "after-warning").isEmpty() + && "after-warning".equals(settings.get(unvisitedKey)), + "a silent setter warning prevented the following key from being applied"); + fail.set(true); + interruptSetter.set(true); + try { + config(engine, session, 8, failedKey, "interrupted-value"); + throw new AssertionError("interrupted setter became a silent warning"); + } catch (SparkJvmEngine.EngineException expected) { + require(expected.code() == 1, "interrupted setter failed with another code"); + } finally { + interruptSetter.set(false); + } + require(!settings.containsKey(failedKey), "interrupted setter did not roll back its write"); + fail.set(true); + failRollback.set(true); + try { + config(engine, session, 8, failedKey, "unrestored-value"); + throw new AssertionError("failed rollback became a silent warning"); + } catch (SparkJvmEngine.EngineException expected) { + require(expected.code() == 13 && expected.getCause() == failure + && Arrays.asList(failure.getSuppressed()).contains(rollbackFailure), + "failed silent rollback did not retain the operation failure"); + } + } finally { + settingsField.set(conf, settings); + settings.remove(key); + settings.remove(failedKey); + settings.remove(unvisitedKey); + } + } + + private static void configCancellationDoesNotWaitForMutation(SparkJvmEngine engine, + SparkJvmEngine.Session session) throws Exception { + for (int opcode : new int[] {6, 8}) { + configCancellationDoesNotWaitForMutation(engine, session, opcode); + } + } + + @SuppressWarnings("unchecked") + private static void configCancellationDoesNotWaitForMutation(SparkJvmEngine engine, + SparkJvmEngine.Session session, int opcode) throws Exception { + org.apache.spark.sql.classic.SparkSession sessionSpark = + (org.apache.spark.sql.classic.SparkSession) privateField(session, "spark"); + Object conf = sessionSpark.sessionState().conf(); + Field settingsField = conf.getClass().getDeclaredField("settings"); + settingsField.setAccessible(true); + Map settings = (Map) settingsField.get(conf); + String key = "spark-connect.cancel-config." + UUID.randomUUID(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicBoolean block = new AtomicBoolean(true); + Map gated = new AbstractMap<>() { + @Override + public Set> entrySet() { return settings.entrySet(); } + @Override + public String get(Object name) { return settings.get(name); } + @Override + public String remove(Object name) { return settings.remove(name); } + @Override + public String put(String name, String value) { + String previous = settings.put(name, value); + if (key.equals(name) && block.compareAndSet(true, false)) { + entered.countDown(); + boolean interrupted = false; + while (release.getCount() != 0) { + try { release.await(); } catch (InterruptedException ignored) { interrupted = true; } + } + if (interrupted) Thread.currentThread().interrupt(); + } + return previous; + } + }; + ExecutorService callers = Executors.newFixedThreadPool(2); + try (SparkJvmEngine.Task task = engine.newTask("cancel blocked config mutation")) { + byte[] fields = opcode == 8 ? encode(key, "changed") + : encode(key, "changed", key + ".later", "must not apply"); + settingsField.set(conf, gated); + Future write = callers.submit(() -> expectCode(1, () -> session.config(opcode, fields, task))); + try { + require(entered.await(10, TimeUnit.SECONDS), "config mutation did not reach its gate"); + Future cancel = callers.submit(task::cancel); + cancel.get(5, TimeUnit.SECONDS); + require(!write.isDone(), "mutation gate did not remain blocked during cancellation"); + } finally { + release.countDown(); + } + write.get(10, TimeUnit.SECONDS); + require(!settings.containsKey(key) && !settings.containsKey(key + ".later"), + "cancelled config mutation did not roll back its partial write"); + require(!flag(task, "completed"), "cancelled config mutation completed"); + } finally { + release.countDown(); + callers.shutdownNow(); + require(callers.awaitTermination(10, TimeUnit.SECONDS), "config cancellation callers remained"); + settingsField.set(conf, settings); + settings.remove(key); + settings.remove(key + ".later"); + } + } + + private static void commandsCannotMutateCatalog(SparkJvmEngine engine, + SparkJvmEngine.Session session) throws Exception { + reject(engine, session, sql("CREATE TEMP VIEW engine_forbidden_command AS SELECT 1 AS id")); + missing(engine, session, "engine_forbidden_command"); + reject(engine, session, sql( + "BEGIN CREATE TEMP VIEW engine_forbidden_script AS SELECT 1 AS id; END")); + missing(engine, session, "engine_forbidden_script"); + reject(engine, session, Relation.newBuilder().setWithRelations(WithRelations.newBuilder() + .setRoot(sql("CREATE TEMP VIEW engine_forbidden_refs AS SELECT 1 AS id")) + .addReferences(Relation.newBuilder().setSubqueryAlias(SubqueryAlias.newBuilder() + .setAlias("unused_ref").setInput(range(1))))).build()); + missing(engine, session, "engine_forbidden_refs"); + + SparkSession sessionSpark = (SparkSession) privateField(session, "spark"); + String table = "engine_write_guard_" + UUID.randomUUID().toString().replace("-", ""); + Path directory = Files.createTempDirectory("engine-write-guard-quote'-"); + String location = directory.toUri().toString().replace("\\", "\\\\").replace("'", "\\'"); + try { + require(sessionSpark.sql("SELECT '" + location + "' AS location").head().getString(0) + .equals(directory.toUri().toString()), "quoted temp URI did not round-trip through SQL"); + sessionSpark.sql("CREATE TABLE " + table + " (id BIGINT) USING parquet LOCATION '" + + location + "'").collect(); + // Prove the target accepts real writes before testing rejection by the adapter. + sessionSpark.sql("INSERT INTO " + table + " VALUES (7)").collect(); + require(Arrays.equals(tableValues(sessionSpark, table), new long[] {1, 7}), + "positive INSERT control did not change the target"); + for (String statement : new String[] { + "INSERT INTO " + table + " VALUES (99)", + "INSERT OVERWRITE TABLE " + table + " VALUES (99)", + "WITH source AS (SELECT 99 AS id) INSERT INTO " + table + " SELECT * FROM source", + "EXECUTE IMMEDIATE 'INSERT INTO " + table + " VALUES (99)'", + "INSERT OVERWRITE DIRECTORY '" + location + "' USING parquet SELECT 99 AS id", + }) { + matchingMetadataFailure(engine, session, sql(statement).toByteArray(), 12); + // matchingMetadataFailure exercises prepare and every Analyze opcode; opcode zero + // is also the typed validation used by the native frontend's SqlCommand path. + require(Arrays.equals(tableValues(sessionSpark, table), new long[] {1, 7}), + "rejected SQL mutated its target: " + statement); + } + System.out.println("ENGINE_WRITE_GUARD_OK: valid INSERT positive control; prepare, schema, " + + "isLocal, isStreaming and SqlCommand validation leave INSERT/OVERWRITE/dynamic SQL targets unchanged"); + } finally { + try { + sessionSpark.sql("DROP TABLE IF EXISTS " + table).collect(); + } finally { + try (var paths = Files.walk(directory)) { + for (Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) { + Files.deleteIfExists(path); + } + } + } + } + } + + private static long[] tableValues(SparkSession spark, String table) { + Row value = spark.sql("SELECT count(*) AS rows, coalesce(sum(id), 0) AS total FROM " + + table).collectAsList().get(0); + return new long[] {value.getLong(0), value.getLong(1)}; + } + + private static void reject(SparkJvmEngine engine, SparkJvmEngine.Session session, Relation relation) { + try (SparkJvmEngine.Task task = engine.newTask("reject side effects")) { + expectCode(12, () -> session.prepare(relation.toByteArray(), task)); + } + } + + private static void missing(SparkJvmEngine engine, SparkJvmEngine.Session session, String table) { + try (SparkJvmEngine.Task task = engine.newTask("assert missing view")) { + try (SparkJvmEngine.Query ignored = session.prepare( + sql("SELECT * FROM " + table).toByteArray(), task)) { + throw new AssertionError("rejected SQL created view " + table); + } catch (RuntimeException expected) { + require(expected.getMessage().contains(table), "unexpected missing-view failure"); + } + } + } + + private static void earlyCloseKeepsContext(SparkJvmEngine engine, + SparkJvmEngine.Session first, SparkJvmEngine.Session second, SparkSession spark) + throws Exception { + try (SparkJvmEngine.Query partial = query(engine, first, range(100000))) { + require(partial.nextIpcBatch() != null, "early-close query returned no batch"); + } + require(!spark.sparkContext().isStopped(), "cursor close stopped the shared SparkContext"); + try (SparkJvmEngine.Query other = query(engine, second, sql("SELECT 7 AS value"))) { + require(Arrays.equals(consume(other), new long[] {1, 7}), "another session broke after close"); + } + } + + private static void cancelRunningFetch(SparkJvmEngine engine, SparkJvmEngine.Session first, + SparkJvmEngine.Session second, SparkSession spark, boolean interrupt) throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (SparkJvmEngine.Task task = engine.newTask("running cancellation"); + SparkJvmEngine.Query query = first.prepare( + sql("SELECT sum(id) AS total FROM range(1000000000000)").toByteArray(), task)) { + Future fetch = executor.submit(query::nextIpcBatch); + long limit = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (spark.sparkContext().statusTracker().getActiveJobIds().length == 0 + && !fetch.isDone() && System.nanoTime() < limit) Thread.sleep(10); + require(!fetch.isDone(), "long query finished before cancellation"); + require(spark.sparkContext().statusTracker().getActiveJobIds().length != 0, + "long query did not submit a Spark job"); + if (interrupt) { + task.cancel(); + } else { + // Exercise Spark's job-cancellation exception path without a task/thread interrupt. + // The production cancel() always interrupts too, which would hide this status race. + Field cancelled = task.getClass().getDeclaredField("cancelled"); + cancelled.setAccessible(true); + synchronized (task) { cancelled.setBoolean(task, true); } + spark.sparkContext().cancelJobGroupAndFutureJobs((String) privateField(task, "jobGroup")); + } + try { + fetch.get(10, TimeUnit.SECONDS); + throw new AssertionError("running fetch survived cancellation"); + } catch (ExecutionException error) { + require(error.getCause() instanceof SparkJvmEngine.EngineException + && ((SparkJvmEngine.EngineException) error.getCause()).code() == 1, + "running cancellation returned the wrong error: " + error.getCause()); + } + } finally { + executor.shutdownNow(); + require(executor.awaitTermination(10, TimeUnit.SECONDS), "fetch worker did not stop"); + } + try (SparkJvmEngine.Query other = query(engine, second, sql("SELECT 3 AS value"))) { + require(Arrays.equals(consume(other), new long[] {1, 3}), "cancellation broke another session"); + } + } + + private static void shutdownHelpers() { + AtomicInteger cleaned = new AtomicInteger(); + CountDownLatch failed = new CountDownLatch(1); + require(!EmbeddedSparkEngine.shutdown(() -> { throw new IllegalStateException("expected stop failure"); }, + cleaned::incrementAndGet, cleaned::incrementAndGet, failed, false), + "failed native shutdown was accepted"); + require(failed.getCount() == 0 && cleaned.get() == 0, + "native failure did not retain resources or release its waiter"); + CountDownLatch startup = new CountDownLatch(1); + require(EmbeddedSparkEngine.shutdown(() -> {}, cleaned::incrementAndGet, + cleaned::incrementAndGet, startup, true), "startup cleanup failed"); + require(startup.getCount() == 0 && cleaned.get() == 2, "startup cleanup did not finish"); + } + + private static byte[] encode(String... fields) throws Exception { + require(fields.length % 2 == 0, "config fields must be pairs"); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + DataOutputStream output = new DataOutputStream(bytes); + output.writeInt(fields.length / 2); + for (String value : fields) { + if (value == null) output.writeInt(-1); + else { + byte[] data = value.getBytes(StandardCharsets.UTF_8); + output.writeInt(data.length); + output.write(data); + } + } + return bytes.toByteArray(); + } + + private static Map decode(byte[] bytes) throws Exception { + DataInputStream input = new DataInputStream(new ByteArrayInputStream(bytes)); + int count = input.readInt(); + Map result = new LinkedHashMap<>(); + for (int i = 0; i < count; i++) result.put(readString(input), readString(input)); + require(input.available() == 0, "trailing config bytes"); + return result; + } + + private static String readString(DataInputStream input) throws Exception { + int len = input.readInt(); + return len == -1 ? null : new String(input.readNBytes(len), StandardCharsets.UTF_8); + } + + private static SparkJvmEngine.EngineException expectCode(int code, Runnable action) { + try { + action.run(); + throw new AssertionError("operation did not fail with code " + code); + } catch (SparkJvmEngine.EngineException error) { + require(error.code() == code, "unexpected engine error: " + error); + return error; + } + } + + private static void require(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + } +} diff --git a/crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/IpcBatchSmokeTest.java b/crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/IpcBatchSmokeTest.java new file mode 100644 index 00000000..c1aacac4 --- /dev/null +++ b/crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/IpcBatchSmokeTest.java @@ -0,0 +1,436 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.spark.sql.connect.nativebridge.engine; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.lang.reflect.Method; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.spark.SparkException; +import org.apache.spark.connect.proto.Range; +import org.apache.spark.connect.proto.Relation; +import org.apache.spark.connect.proto.SQL; +import org.apache.spark.scheduler.SparkListener; +import org.apache.spark.scheduler.SparkListenerJobStart; +import org.apache.spark.scheduler.SparkListenerTaskStart; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.util.ArrowUtils$; + +/** Spark IPC interoperability, partition order, errors, cancellation and empty nested values. */ +public final class IpcBatchSmokeTest { + private IpcBatchSmokeTest() {} + + public static void main(String[] arguments) throws Exception { + require(arguments.length == 0 || (arguments.length == 1 + && arguments[0].equals("--result-size-only")), "unknown IPC test arguments"); + SparkSession spark = SparkSession.builder().appName("Native Arrow IPC Smoke") + .config("spark.sql.session.timeZone", "UTC") + .config("spark.sql.artifact.isolation.enabled", "true") + .config("spark.sql.artifact.isolation.alwaysApplyClassloader", "true") + .getOrCreate(); + spark.sparkContext().setLogLevel("ERROR"); + try (SparkJvmEngine engine = new SparkJvmEngine(spark)) { + if (arguments.length == 1) { + partitionResultSizeFailure(engine); + System.out.println("IPC_RESULT_SIZE_SMOKE_OK: result-size abort and session recovery"); + return; + } + resourceErrorMapping(); + allPartitions(engine); + wideRowsUseByteEstimate(engine); + laterPartitionFailure(engine); + nestedAndEmpty(engine); + emptyVariableWidthValues(engine, false); + emptyVariableWidthValues(engine, true); + cancelSparkWorker(engine, spark); + require(!spark.sparkContext().isStopped(), "session close stopped shared SparkContext"); + System.out.println("IPC_BATCH_SMOKE_OK: self-contained IPC, partition order, retained bytes, " + + "nested/decimal/empty/wide values, resource errors, cancellation and session recovery"); + } finally { + spark.stop(); + } + } + + private static SparkJvmEngine.Session open(SparkJvmEngine engine) { + try (SparkJvmEngine.Task task = engine.newTask("IPC smoke open")) { + return engine.openSession("ipc-smoke", UUID.randomUUID().toString(), task); + } + } + + private static SparkJvmEngine.Query prepare(SparkJvmEngine engine, + SparkJvmEngine.Session session, Relation relation) { + SparkJvmEngine.Task task = engine.newTask("IPC smoke prepare"); + try { + return session.prepare(relation.toByteArray(), task); + } catch (Throwable failure) { + task.close(); + throw failure; + } + } + + private static Relation sql(String value) { + return Relation.newBuilder().setSql(SQL.newBuilder().setQuery(value)).build(); + } + + private static void config(SparkJvmEngine engine, SparkJvmEngine.Session session, + String key, String value) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(bytes); + out.writeInt(1); + for (String field : new String[] {key, value}) { + byte[] encoded = field.getBytes(StandardCharsets.UTF_8); + out.writeInt(encoded.length); + out.write(encoded); + } + try (SparkJvmEngine.Task task = engine.newTask("IPC smoke configuration")) { + session.config(6, bytes.toByteArray(), task); + } + } + + @FunctionalInterface + private interface BatchCheck { void run(VectorSchemaRoot root) throws Exception; } + + private static void checkBatch(NativeIpcBatch batch, BatchCheck check) throws Exception { + require(batch != null, "missing IPC batch"); + try (BufferAllocator allocator = ArrowUtils$.MODULE$.rootAllocator() + .newChildAllocator("ipc-smoke", 0, Long.MAX_VALUE); + ArrowStreamReader reader = new ArrowStreamReader( + new ByteArrayInputStream(batch.data()), allocator)) { + require(reader.loadNextBatch(), "IPC stream contains no record batch"); + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + require(root.getRowCount() == batch.rowCount(), "IPC row count differs from its header"); + check.run(root); + require(!reader.loadNextBatch(), "IPC stream contains multiple record batches"); + } + } + + private static void assertSingle(SparkJvmEngine engine, SparkJvmEngine.Session session, + int expected) throws Exception { + try (SparkJvmEngine.Query query = prepare(engine, session, + sql("SELECT " + expected + " AS v"))) { + checkBatch(query.nextIpcBatch(), root -> require(root.getRowCount() == 1 + && ((Number) root.getVector(0).getObject(0)).intValue() == expected, + "session recovery result changed")); + require(query.nextIpcBatch() == null, "recovery query EOF changed"); + } + } + + private static void resourceErrorMapping() throws Exception { + Method engineError = SparkJvmEngine.class.getDeclaredMethod("engineError", Exception.class); + engineError.setAccessible(true); + SparkJvmEngine.EngineException original = + new SparkJvmEngine.EngineException(8, "executor IPC exceeds its size limit"); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream output = new ObjectOutputStream(bytes)) { + output.writeObject(new SparkException("executor failed", original)); + } + SparkException transported; + try (ObjectInputStream input = new ObjectInputStream( + new ByteArrayInputStream(bytes.toByteArray()))) { + transported = (SparkException) input.readObject(); + } + SparkJvmEngine.EngineException mapped = + (SparkJvmEngine.EngineException) engineError.invoke(null, transported); + require(mapped.code() == 8 && original.getMessage().equals(mapped.getMessage()), + "executor serialization or Spark wrapper lost RESOURCE_EXHAUSTED"); + OutOfMemoryException allocation = new OutOfMemoryException("Arrow allocation limit"); + mapped = (SparkJvmEngine.EngineException) engineError.invoke(null, allocation); + require(mapped.code() == 8 && mapped.getCause() == allocation, + "driver Arrow allocation failure lost RESOURCE_EXHAUSTED"); + try { + SparkJvmEngine.checkedIpc(new byte[SparkJvmEngine.MAX_IPC_BYTES + 1]); + throw new AssertionError("executor IPC check accepted a batch over 64 MiB"); + } catch (SparkJvmEngine.EngineException error) { + require(error.code() == 8, "oversized IPC batch lost RESOURCE_EXHAUSTED"); + } + } + + private static void wideRowsUseByteEstimate(SparkJvmEngine engine) throws Exception { + try (SparkJvmEngine.Session session = open(engine)) { + config(engine, session, "spark.sql.execution.arrow.maxRecordsPerBatch", "100000"); + String expected = "x".repeat(500000); + long rows = 0; + int batches = 0; + try (SparkJvmEngine.Query query = prepare(engine, session, + sql("SELECT repeat('x', 500000) AS value FROM range(20)"))) { + NativeIpcBatch batch; + while ((batch = query.nextIpcBatch()) != null) { + // The producer's 2 MiB estimate can overshoot by one wide row. This + // fixture's 3 MiB allowance is independent of Connect transport chunking. + require(batch.data().length < 3 * 1024 * 1024, + "wide-row producer IPC exceeded this fixture's size-estimate allowance"); + checkBatch(batch, root -> { + for (int row = 0; row < root.getRowCount(); row++) { + require(expected.equals(root.getVector(0).getObject(row).toString()), + "wide string value changed"); + } + }); + rows += batch.rowCount(); + batches++; + } + } + require(rows == 20 && batches > 1, "byte estimate did not split complete wide-row output"); + } + } + + private static void allPartitions(SparkJvmEngine engine) throws Exception { + NativeIpcBatch retained; + byte[] expected; + try (SparkJvmEngine.Session session = open(engine)) { + config(engine, session, "spark.sql.execution.arrow.maxRecordsPerBatch", "3"); + Relation range = Relation.newBuilder().setRange(Range.newBuilder() + .setStart(0).setEnd(29).setStep(1).setNumPartitions(3)).build(); + long[] rowsAndSum = {0, 0}; + int batches = 0; + try (SparkJvmEngine.Query query = prepare(engine, session, range)) { + retained = query.nextIpcBatch(); + require(retained != null, "range returned no batches"); + expected = retained.data().clone(); + NativeIpcBatch batch = retained; + while (batch != null) { + require(batch.rowCount() > 0 && batch.rowCount() <= 3, "configured row limit changed"); + checkBatch(batch, root -> { + for (int row = 0; row < root.getRowCount(); row++) { + long value = ((Number) root.getVector(0).getObject(row)).longValue(); + require(value == rowsAndSum[0], "partition cursor reordered or lost rows"); + rowsAndSum[0]++; + rowsAndSum[1] += value; + } + }); + batches++; + require(Arrays.equals(retained.data(), expected), "fetch overwrote prior IPC bytes"); + batch = query.nextIpcBatch(); + } + require(query.nextIpcBatch() == null, "partition cursor EOF changed"); + } + require(rowsAndSum[0] == 29 && rowsAndSum[1] == 406 && batches == 11, + "multiple partitions did not preserve complete range output"); + } + require(Arrays.equals(retained.data(), expected), "query/session close changed prior IPC"); + checkBatch(retained, root -> require( + ((Number) root.getVector(0).getObject(2)).longValue() == 2, "retained value changed")); + } + + private static void laterPartitionFailure(SparkJvmEngine engine) throws Exception { + try (SparkJvmEngine.Session session = open(engine)) { + config(engine, session, "spark.sql.execution.arrow.maxRecordsPerBatch", "2"); + Relation values = sql("SELECT CASE WHEN id >= 4 " + + "THEN CAST(raise_error('later-partition-ipc-error') AS BIGINT) " + + "ELSE id END AS value FROM range(0, 12, 1, 3)"); + NativeIpcBatch retained; + try (SparkJvmEngine.Query query = prepare(engine, session, values)) { + retained = query.nextIpcBatch(); + checkBatch(retained, root -> require(root.getRowCount() == 2 + && ((Number) root.getVector(0).getObject(0)).longValue() == 0, + "first partition lost output")); + checkBatch(query.nextIpcBatch(), root -> require(root.getRowCount() == 2 + && ((Number) root.getVector(0).getObject(0)).longValue() == 2, + "first partition lost its later batch")); + SparkJvmEngine.EngineException terminal; + try { + query.nextIpcBatch(); + throw new AssertionError("later partition error was swallowed"); + } catch (SparkJvmEngine.EngineException error) { + terminal = error; + require(error.code() == 13 && error.getMessage().contains("later-partition-ipc-error"), + "later partition error lost its INTERNAL status or message"); + } + try { + query.nextIpcBatch(); + throw new AssertionError("failed query became EOF"); + } catch (SparkJvmEngine.EngineException error) { + require(error == terminal, "failed query replaced its terminal error"); + } + } + checkBatch(retained, root -> require( + ((Number) root.getVector(0).getObject(1)).longValue() == 1, + "failed query invalidated prior bytes")); + assertSingle(engine, session, 9); + } + } + + private static void nestedAndEmpty(SparkJvmEngine engine) throws Exception { + try (SparkJvmEngine.Session session = open(engine)) { + Relation values = sql("SELECT 'hello' AS text, CAST(NULL AS INT) AS missing, " + + "array(1, CAST(NULL AS INT), 3) AS items, map('x', 7) AS mapping, " + + "named_struct('a', 1, 'b', 'v') AS nested, " + + "CAST('12.34' AS DECIMAL(10,2)) AS amount, " + + "array(CAST('56.78' AS DECIMAL(10,2))) AS amounts"); + try (SparkJvmEngine.Query query = prepare(engine, session, values)) { + checkBatch(query.nextIpcBatch(), root -> { + require(root.getRowCount() == 1, "nested row missing"); + require(root.getVector(0).getObject(0).toString().equals("hello"), "text changed"); + require(root.getVector(1).getObject(0) == null, "null validity changed"); + List items = (List) root.getVector(2).getObject(0); + require(items.size() == 3 && items.get(1) == null + && ((Number) items.get(2)).intValue() == 3, "list values changed"); + List mapping = (List) root.getVector(3).getObject(0); + Map entry = (Map) mapping.get(0); + require(entry.get("key").toString().equals("x") + && ((Number) entry.get("value")).intValue() == 7, "map values changed"); + require(((Number) ((Map) root.getVector(4).getObject(0)).get("a")) + .intValue() == 1, "struct values changed"); + require(new BigDecimal("12.34").equals(root.getVector(5).getObject(0)), + "decimal changed"); + require(new BigDecimal("56.78").equals( + ((List) root.getVector(6).getObject(0)).get(0)), "nested decimal changed"); + }); + require(query.nextIpcBatch() == null, "nested query EOF changed"); + } + try (SparkJvmEngine.Query empty = prepare(engine, session, + sql("SELECT CAST(NULL AS BIGINT) AS missing WHERE false"))) { + require(empty.nextIpcBatch() == null, "empty query returned a record batch"); + } + } + } + + private static void emptyVariableWidthValues(SparkJvmEngine engine, boolean largeVarTypes) + throws Exception { + try (SparkJvmEngine.Session session = open(engine)) { + config(engine, session, "spark.sql.execution.arrow.maxRecordsPerBatch", "2"); + config(engine, session, "spark.sql.execution.arrow.useLargeVarTypes", + Boolean.toString(largeVarTypes)); + Relation values = sql("SELECT '' AS text, unhex('') AS binary, " + + "CAST(array() AS ARRAY) AS items, " + + "array(CAST(array() AS ARRAY), CAST(array() AS ARRAY)) AS nested, " + + "array(array('', CAST(NULL AS STRING))) AS strings, " + + "CAST(map() AS MAP>) AS mapping, " + + "CAST(NULL AS STRING) AS missing FROM range(5)"); + long rows = 0; + try (SparkJvmEngine.Query query = prepare(engine, session, values)) { + NativeIpcBatch batch; + while ((batch = query.nextIpcBatch()) != null) { + checkBatch(batch, root -> { + require((root.getVector(0) instanceof org.apache.arrow.vector.LargeVarCharVector) + == largeVarTypes, "large variable-width setting changed"); + for (int row = 0; row < root.getRowCount(); row++) { + require(root.getVector(0).getObject(row).toString().isEmpty(), + "empty string changed"); + require(((byte[]) root.getVector(1).getObject(row)).length == 0, + "empty binary changed"); + require(((List) root.getVector(2).getObject(row)).isEmpty(), "empty list changed"); + List nested = (List) root.getVector(3).getObject(row); + require(nested.size() == 2 && ((List) nested.get(0)).isEmpty() + && ((List) nested.get(1)).isEmpty(), "nested empty lists changed"); + List strings = (List) ((List) root.getVector(4).getObject(row)).get(0); + require(strings.size() == 2 && strings.get(0).toString().isEmpty() + && strings.get(1) == null, "nested empty/null strings changed"); + require(((List) root.getVector(5).getObject(row)).isEmpty(), "empty map changed"); + require(root.getVector(6).getObject(row) == null, "all-null string changed"); + } + }); + rows += batch.rowCount(); + } + } + require(rows == 5, "empty variable-width batches lost rows"); + } + } + + private static void cancelSparkWorker(SparkJvmEngine engine, SparkSession spark) + throws Exception { + String description = "cancel worker Arrow conversion " + UUID.randomUUID(); + CountDownLatch started = new CountDownLatch(1); + Set ownStages = ConcurrentHashMap.newKeySet(); + SparkListener listener = new SparkListener() { + @Override + public void onJobStart(SparkListenerJobStart event) { + if (description.equals(event.properties().getProperty("spark.job.description"))) { + scala.collection.Iterator stages = event.stageIds().iterator(); + while (stages.hasNext()) ownStages.add(((Number) stages.next()).intValue()); + } + } + @Override + public void onTaskStart(SparkListenerTaskStart event) { + if (ownStages.contains(event.stageId())) started.countDown(); + } + }; + spark.sparkContext().addSparkListener(listener); + ExecutorService worker = Executors.newSingleThreadExecutor(); + try (SparkJvmEngine.Session session = open(engine); + SparkJvmEngine.Task task = engine.newTask(description); + SparkJvmEngine.Query query = session.prepare(Relation.newBuilder() + .setRange(Range.newBuilder() + .setStart(0).setEnd(1000000000000L).setStep(1).setNumPartitions(1)) + .build().toByteArray(), task)) { + java.util.concurrent.Future fetch = worker.submit(query::nextIpcBatch); + try { + require(started.await(15, TimeUnit.SECONDS), "worker conversion task did not start"); + task.cancel(); + try { + fetch.get(15, TimeUnit.SECONDS); + throw new AssertionError("cancelled worker produced a complete partition"); + } catch (ExecutionException error) { + require(error.getCause() instanceof SparkJvmEngine.EngineException + && ((SparkJvmEngine.EngineException) error.getCause()).code() == 1, + "cancelled Spark worker lost CANCELLED status: " + error.getCause()); + } + } finally { + task.cancel(); + } + assertSingle(engine, session, 11); + } finally { + spark.sparkContext().removeSparkListener(listener); + worker.shutdownNow(); + require(worker.awaitTermination(15, TimeUnit.SECONDS), + "cancelled worker fetch did not drain"); + } + } + + private static void partitionResultSizeFailure(SparkJvmEngine engine) throws Exception { + try (SparkJvmEngine.Session session = open(engine)) { + try (SparkJvmEngine.Query query = prepare(engine, session, + sql("SELECT id FROM range(0, 400000, 1, 1)"))) { + try { + query.nextIpcBatch(); + throw new AssertionError("partition unexpectedly fit the 1 MiB Spark result-size limit"); + } catch (SparkJvmEngine.EngineException error) { + require(error.code() == 13, "Spark result-size abort changed its INTERNAL status"); + require(error.getMessage() != null + && (error.getMessage().contains("maxResultSize") + || error.getMessage().contains("serialized results")), + "result-size fixture failed for a different reason: " + error); + } + } + assertSingle(engine, session, 5); + } + } + + private static void require(boolean value, String message) { + if (!value) throw new AssertionError(message); + } +} diff --git a/crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/NativeEngineLifecycleTest.java b/crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/NativeEngineLifecycleTest.java new file mode 100644 index 00000000..73710e25 --- /dev/null +++ b/crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/NativeEngineLifecycleTest.java @@ -0,0 +1,810 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.spark.sql.connect.nativebridge.engine; + +import java.io.ByteArrayInputStream; +import java.lang.management.ManagementFactory; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.spark.connect.proto.AnalyzePlanRequest; +import org.apache.spark.connect.proto.AnalyzePlanResponse; +import org.apache.spark.connect.proto.ConfigRequest; +import org.apache.spark.connect.proto.ExecutePlanRequest; +import org.apache.spark.connect.proto.ExecutePlanResponse; +import org.apache.spark.connect.proto.Expression; +import org.apache.spark.connect.proto.KeyValue; +import org.apache.spark.connect.proto.Plan; +import org.apache.spark.connect.proto.Project; +import org.apache.spark.connect.proto.Range; +import org.apache.spark.connect.proto.Relation; +import org.apache.spark.connect.proto.ReleaseExecuteRequest; +import org.apache.spark.connect.proto.ReleaseSessionRequest; +import org.apache.spark.connect.proto.SQL; +import org.apache.spark.connect.proto.SparkConnectServiceGrpc; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.Row; +import org.apache.spark.ml.linalg.VectorUDT; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.SparkSessionExtensions; +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan; +import org.apache.spark.sql.catalyst.rules.Rule; +import org.apache.spark.sql.connect.service.SparkConnectService$; +import org.apache.spark.sql.internal.SQLConf; +import org.apache.spark.sql.util.ArrowUtils$; +import org.sparkproject.connect.grpc.ManagedChannel; +import org.sparkproject.connect.grpc.ManagedChannelBuilder; +import org.sparkproject.connect.grpc.Status; +import org.sparkproject.connect.grpc.StatusRuntimeException; + +import scala.runtime.AbstractFunction1; +import scala.runtime.BoxedUnit; + +/** Native reader/writer ordering, worker context reuse and runtime teardown under -Xcheck:jni. */ +public final class NativeEngineLifecycleTest { + private static final String GATE_KEY = "spark-connect.native.analysis.gate"; + private static final String CONTEXT_KEY = "spark-connect.native.context"; + private static final AtomicBoolean RECORD = new AtomicBoolean(); + private static final Set ANALYZER_THREADS = ConcurrentHashMap.newKeySet(); + private static final AtomicReference ANALYSIS_GATE = new AtomicReference<>(); + private static final AtomicReference CONTEXT_PROBE = new AtomicReference<>(); + + private NativeEngineLifecycleTest() {} + + public static void main(String[] arguments) throws Exception { + require(arguments.length == 1, "pass the absolute native library path"); + launcherFailureLifecycle(); + SparkSession spark = SparkSession.builder().appName("Native engine worker lifecycle") + .config("spark.sql.artifact.isolation.enabled", "true") + .config("spark.sql.artifact.isolation.alwaysApplyClassloader", "true") + .withExtensions(new AbstractFunction1() { + @Override + public BoxedUnit apply(SparkSessionExtensions extensions) { + extensions.injectResolutionRule(new AbstractFunction1>() { + @Override + public Rule apply(SparkSession ruleSession) { + return new Rule() { + @Override + public LogicalPlan apply(LogicalPlan plan) { + if (RECORD.get()) ANALYZER_THREADS.add(Thread.currentThread()); + ContextProbe probe = CONTEXT_PROBE.get(); + if (probe != null) probe.observe(ruleSession); + AnalysisGate gate = ANALYSIS_GATE.get(); + if (gate != null + && gate.marker.equals(ruleSession.conf().get(GATE_KEY, ""))) { + gate.enter(ruleSession); + } + return plan; + } + }; + } + }); + return BoxedUnit.UNIT; + } + }).getOrCreate(); + spark.sparkContext().setLogLevel("ERROR"); + ExecutorService callers = Executors.newFixedThreadPool(8); + try { + for (int restart = 0; restart < 3; restart++) { + int port; + try (SparkJvmEngine engine = new SparkJvmEngine(spark); + NativeEngineServer server = new NativeEngineServer( + arguments[0], "127.0.0.1:0", engine)) { + String address = server.address(); + require(server.isRunning() && !server.isDrained(), + "live native listener reported stopped or drained"); + port = Integer.parseInt(address.substring(address.lastIndexOf(':') + 1)); + ManagedChannel channel = ManagedChannelBuilder.forTarget(address).usePlaintext().build(); + try { + String session = UUID.randomUUID().toString(); + AnalyzePlanRequest request = AnalyzePlanRequest.newBuilder().setSessionId(session) + .setSchema(AnalyzePlanRequest.Schema.newBuilder().setPlan(Plan.newBuilder() + .setRoot(Relation.newBuilder().setRange(Range.newBuilder() + .setStart(0).setEnd(10).setStep(1))))).build(); + var stub = SparkConnectServiceGrpc.newBlockingStub(channel); + for (int warmup = 0; warmup < 32; warmup++) check(analyze(stub, request)); + nativeReadersAndWriter(stub, callers, session, request); + cleanupAttachmentReuse(stub, callers, session); + ANALYZER_THREADS.clear(); + RECORD.set(true); + try { + List> work = new ArrayList<>(); + for (int worker = 0; worker < 8; worker++) { + work.add(() -> { + for (int call = 0; call < 32; call++) check(analyze(stub, request)); + return null; + }); + } + for (Future result : callers.invokeAll(work, 60, TimeUnit.SECONDS)) { + result.get(1, TimeUnit.SECONDS); + } + } finally { + RECORD.set(false); + } + // Re-attaching each RPC creates a new Java Thread identity even when + // Tokio reuses its OS thread. A live attached pool reuses identities. + require(!ANALYZER_THREADS.isEmpty(), "analyzer observer was not exercised"); + require(ANALYZER_THREADS.size() < 64, + "native workers did not reuse JVM attachments: " + ANALYZER_THREADS.size()); + String secondSession = UUID.randomUUID().toString(); + twoSessionWorkerReuse(stub, session, secondSession); + nestedArrowIpcValues(stub, session); + emptyArrowIpcValues(stub, session); + lossySchemasAreRejected(stub, session, spark); + for (String released : List.of(session, secondSession)) { + require(stub.withDeadlineAfter(30, TimeUnit.SECONDS).releaseSession( + ReleaseSessionRequest.newBuilder().setSessionId(released).build()) + .getSessionId().equals(released), "session release identity changed"); + } + } finally { + channel.shutdownNow(); + require(channel.awaitTermination(10, TimeUnit.SECONDS), "client channel did not stop"); + } + require(SparkConnectService$.MODULE$.server() == null, "Java Connect listener appeared"); + server.close(); + // Check immediately after nativeStop, before try-with-resources calls engine.close(). + require(engine.isClosed(), + "native stop returned before closing its Java engine factory"); + } + try (ServerSocket rebound = new ServerSocket()) { + rebound.setReuseAddress(true); + rebound.bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), port)); + require(rebound.isBound(), "native listener could not rebind after teardown"); + } + require(!spark.sparkContext().isStopped(), "runtime close stopped shared SparkContext"); + } + concurrentEngineCleanup(spark, arguments[0], callers); + System.out.println("NATIVE_ENGINE_LIFECYCLE_OK: native concurrent metadata/config readers; " + + "synchronous Java engine factory close before native stop returns; " + + "Config writer exclusion/readback; two-session worker context reuse; attachment reuse; " + + "12-level struct/list and all-empty variable-width Arrow IPC values; " + + "lossy schema/shuffle-cleanup rejection; three runtime restarts; session release; " + + "cleanup attachment reuse; concurrent engine shutdown; " + + "listener rebind; shared SparkContext survives"); + } finally { + RECORD.set(false); + CONTEXT_PROBE.set(null); + AnalysisGate gate = ANALYSIS_GATE.get(); + if (gate != null) gate.close(); + callers.shutdownNow(); + require(callers.awaitTermination(10, TimeUnit.SECONDS), "native client workers did not stop"); + spark.stop(); + } + } + + private static void launcherFailureLifecycle() throws InterruptedException { + try { + EmbeddedSparkEngine.awaitListener(new CountDownLatch(1), () -> false, () -> false); + throw new AssertionError("launcher ignored a terminated native listener"); + } catch (IllegalStateException expected) { + require(expected.getMessage().contains("listener stopped unexpectedly"), + "launcher lost listener termination diagnostic"); + } + EmbeddedSparkEngine.awaitListener(new CountDownLatch(0), + () -> { throw new AssertionError("shutdown queried listener health"); }, () -> true); + + AtomicInteger calls = new AtomicInteger(); + RuntimeException terminal = new IllegalStateException("listener failed after drain"); + Throwable primary = new IllegalStateException("launcher observed listener failure"); + EmbeddedSparkEngine.closeAfterListenerExit(() -> { + if (calls.incrementAndGet() == 1) throw terminal; + }, () -> true, primary); + require(calls.get() == 2 && primary.getSuppressed().length == 1 + && primary.getSuppressed()[0] == terminal, + "confirmed drain did not retry close and preserve listener error"); + + calls.set(0); + try { + EmbeddedSparkEngine.closeAfterListenerExit(() -> { + calls.incrementAndGet(); + throw terminal; + }, () -> false, primary); + throw new AssertionError("incomplete native drain was treated as safe"); + } catch (IllegalStateException expected) { + require(expected == terminal && calls.get() == 1, + "launcher retried a close without confirmed drain"); + } + + calls.set(0); + RuntimeException cleanup = new IllegalStateException("factory cleanup remains incomplete"); + try { + EmbeddedSparkEngine.closeAfterListenerExit(() -> { + if (calls.incrementAndGet() == 1) throw terminal; + throw cleanup; + }, () -> true, primary); + throw new AssertionError("failed factory close was treated as safe"); + } catch (IllegalStateException expected) { + require(expected == terminal && calls.get() == 2 + && terminal.getSuppressed().length == 1 + && terminal.getSuppressed()[0] == cleanup, + "launcher lost a retained factory cleanup failure"); + } + } + + private static void cleanupAttachmentReuse( + SparkConnectServiceGrpc.SparkConnectServiceBlockingStub stub, + ExecutorService callers, String session) throws Exception { + String key = "spark.sql.session.timeZone"; + String expected = getConfig(stub, session, key); + configBurst(stub, callers, session, key, expected, 16); + var threads = ManagementFactory.getThreadMXBean(); + long before = threads.getTotalStartedThreadCount(); + configBurst(stub, callers, session, key, expected, 128); + long started = threads.getTotalStartedThreadCount() - before; + // This coarse JVM-wide count includes unrelated background threads. A budget of half + // the 1024 requests allows pool growth, but rejects per-request cleanup attachment. + // The 64-task admission bound requires most task frees before the burst can finish. + require(started < 512, "cleanup did not reuse JVM attachments: " + started + + " threads started for 1024 Config requests"); + System.out.println("NATIVE_CLEANUP_ATTACHMENT_REUSE threads_started=" + started + + " requests=1024"); + } + + private static void configBurst(SparkConnectServiceGrpc.SparkConnectServiceBlockingStub stub, + ExecutorService callers, String session, String key, String expected, int perCaller) + throws Exception { + List> work = new ArrayList<>(); + for (int caller = 0; caller < 8; caller++) { + work.add(() -> { + for (int call = 0; call < perCaller; call++) { + require(expected.equals(getConfig(stub, session, key)), + "Config changed during cleanup attachment burst"); + } + return null; + }); + } + for (Future result : callers.invokeAll(work, 60, TimeUnit.SECONDS)) { + result.get(1, TimeUnit.SECONDS); + } + } + + private static void concurrentEngineCleanup( + SparkSession spark, String library, ExecutorService callers) throws Exception { + try (SparkJvmEngine firstEngine = new SparkJvmEngine(spark); + SparkJvmEngine secondEngine = new SparkJvmEngine(spark); + NativeEngineServer firstServer = new NativeEngineServer( + library, "127.0.0.1:0", firstEngine); + NativeEngineServer secondServer = new NativeEngineServer( + library, "127.0.0.1:0", secondEngine)) { + ManagedChannel firstChannel = ManagedChannelBuilder.forTarget(firstServer.address()) + .usePlaintext().build(); + ManagedChannel secondChannel = ManagedChannelBuilder.forTarget(secondServer.address()) + .usePlaintext().build(); + try { + var first = SparkConnectServiceGrpc.newBlockingStub(firstChannel); + var second = SparkConnectServiceGrpc.newBlockingStub(secondChannel); + String firstSession = UUID.randomUUID().toString(); + String secondSession = UUID.randomUUID().toString(); + String key = "spark-connect.native.multi-engine.value"; + setConfig(first, firstSession, key, "first", CONTEXT_KEY, "first", + "spark.sql.session.timeZone", "UTC", "spark.sql.timestampType", "TIMESTAMP_LTZ"); + setConfig(second, secondSession, key, "before", CONTEXT_KEY, "second", + "spark.sql.session.timeZone", "Asia/Tokyo", "spark.sql.timestampType", "TIMESTAMP_NTZ"); + ContextProbe probe = new ContextProbe(); + require(CONTEXT_PROBE.compareAndSet(null, probe), "another native context probe is active"); + try { + for (int turn = 0; turn < 8; turn++) { + contextCall(first, firstSession, false, probe, turn * 2); + require("first".equals(getConfig(first, firstSession, key)), + "first engine Config changed"); + contextCall(second, secondSession, true, probe, turn * 2 + 1); + require("before".equals(getConfig(second, secondSession, key)), + "second engine Config changed"); + var selected = turn % 2 == 0 ? first : second; + String temporary = UUID.randomUUID().toString(); + setConfig(selected, temporary, key, "temporary"); + releaseSession(selected, temporary); + } + } finally { + CONTEXT_PROBE.compareAndSet(probe, null); + } + Relation range = Relation.newBuilder().setRange(Range.newBuilder() + .setStart(0).setEnd(10).setStep(1)).build(); + AnalyzePlanRequest request = schemaRequest(secondSession, range); + try (AnalysisGate gate = new AnalysisGate(key)) { + setConfig(second, secondSession, GATE_KEY, gate.marker); + require(ANALYSIS_GATE.compareAndSet(null, gate), + "another native analysis gate is active"); + Future firstRead = callers.submit(() -> analyze(second, request)); + Future secondRead = callers.submit(() -> analyze(second, request)); + require(gate.entered.await(10, TimeUnit.SECONDS), + "second engine readers did not enter Spark"); + AtomicBoolean running = new AtomicBoolean(true); + CountDownLatch trafficStarted = new CountDownLatch(1); + Future traffic = callers.submit(() -> { + int completed = 0; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (running.get() && System.nanoTime() < deadline) { + require("before".equals(getConfig(second, secondSession, key)), + "surviving engine Config changed during peer shutdown"); + completed++; + trafficStarted.countDown(); + } + return completed; + }); + try { + require(trafficStarted.await(10, TimeUnit.SECONDS), + "second engine Config traffic did not start"); + releaseSession(first, firstSession); + Future stopped = callers.submit(() -> { firstServer.close(); return null; }); + stopped.get(15, TimeUnit.SECONDS); + require(firstEngine.isClosed(), "peer shutdown did not close its engine"); + require(!secondEngine.isClosed(), "peer shutdown closed the surviving engine"); + require(!spark.sparkContext().isStopped(), "peer shutdown stopped shared SparkContext"); + require(!firstRead.isDone() && !secondRead.isDone(), + "peer shutdown interrupted the surviving engine readers"); + require("before".equals(getConfig(second, secondSession, key)), + "surviving engine stopped serving Config"); + } finally { + running.set(false); + gate.close(); + } + require(traffic.get(10, TimeUnit.SECONDS) > 0, + "second engine completed no Config traffic"); + check(firstRead.get(10, TimeUnit.SECONDS)); + check(secondRead.get(10, TimeUnit.SECONDS)); + } + check(analyze(second, request)); + require("before".equals(getConfig(second, secondSession, key)), + "surviving engine Config changed"); + releaseSession(second, secondSession); + } finally { + firstChannel.shutdownNow(); + secondChannel.shutdownNow(); + require(firstChannel.awaitTermination(10, TimeUnit.SECONDS), + "first engine channel did not stop"); + require(secondChannel.awaitTermination(10, TimeUnit.SECONDS), + "second engine channel did not stop"); + } + secondServer.close(); + require(secondEngine.isClosed(), "second native stop returned before engine close"); + require(!spark.sparkContext().isStopped(), + "concurrent engine close stopped shared SparkContext"); + } + } + + private static void releaseSession( + SparkConnectServiceGrpc.SparkConnectServiceBlockingStub stub, String session) { + require(stub.withDeadlineAfter(30, TimeUnit.SECONDS).releaseSession( + ReleaseSessionRequest.newBuilder().setSessionId(session).build()) + .getSessionId().equals(session), + "session release identity changed"); + } + + private static final class AnalysisGate implements AutoCloseable { + private final String marker = UUID.randomUUID().toString(); + private final String configKey; + private final CountDownLatch entered = new CountDownLatch(2); + private final CountDownLatch proceed = new CountDownLatch(1); + private final Set threads = ConcurrentHashMap.newKeySet(); + + private AnalysisGate(String configKey) { + this.configKey = configKey; + } + + private void enter(SparkSession ruleSession) { + if (!threads.add(Thread.currentThread())) return; + entered.countDown(); + try { + require(proceed.await(30, TimeUnit.SECONDS), "native analysis gate was not released"); + require("before".equals(ruleSession.conf().get(configKey)), + "native Config writer changed state while metadata readers held it"); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("native analysis gate interrupted", error); + } + } + + @Override + public void close() { + ANALYSIS_GATE.compareAndSet(this, null); + proceed.countDown(); + } + } + + private static void nativeReadersAndWriter( + SparkConnectServiceGrpc.SparkConnectServiceBlockingStub stub, ExecutorService callers, + String session, AnalyzePlanRequest request) throws Exception { + String key = "spark-connect.native.concurrent.value"; + try (AnalysisGate gate = new AnalysisGate(key)) { + setConfig(stub, session, GATE_KEY, gate.marker, key, "before"); + require(ANALYSIS_GATE.compareAndSet(null, gate), "another native analysis gate is active"); + Future first = callers.submit(() -> analyze(stub, request)); + Future second = callers.submit(() -> analyze(stub, request)); + require(gate.entered.await(10, TimeUnit.SECONDS), + "Rust/JNI same-session analyses did not enter Spark concurrently"); + + Future read = callers.submit(() -> getConfig(stub, session, key)); + require("before".equals(read.get(10, TimeUnit.SECONDS)), + "native Config read did not overlap metadata readers"); + CountDownLatch writerStarted = new CountDownLatch(1); + Future write = callers.submit(() -> { + writerStarted.countDown(); + setConfig(stub, session, key, "after"); + return null; + }); + require(writerStarted.await(10, TimeUnit.SECONDS), "native Config writer did not start"); + try { + write.get(200, TimeUnit.MILLISECONDS); + throw new AssertionError("native Config writer completed while metadata readers held state"); + } catch (TimeoutException expected) { + // The readers also verify the value after the gate opens, while still inside Spark. + // Together these test writer exclusion, not exact scheduling of Rust waiter futures. + } + + gate.close(); + check(first.get(10, TimeUnit.SECONDS)); + check(second.get(10, TimeUnit.SECONDS)); + write.get(10, TimeUnit.SECONDS); + require("after".equals(getConfig(stub, session, key)), + "native Config write was not visible after metadata readers drained"); + } + } + + private static final class ContextProbe { + private final Map zones = Map.of("first", "UTC", "second", "Asia/Tokyo"); + private final Map types = Map.of( + "first", "TIMESTAMP_LTZ", "second", "TIMESTAMP_NTZ"); + private final Map> threads = new ConcurrentHashMap<>(); + private final Map observations = new ConcurrentHashMap<>(); + + private void observe(SparkSession session) { + String context = session.conf().get(CONTEXT_KEY, ""); + require(zones.containsKey(context), "analyzer used an unexpected native session context"); + require(SparkSession.getActiveSession().isDefined() + && SparkSession.getActiveSession().get() == session, + "native worker retained another session as its active session"); + require(zones.get(context).equals(SQLConf.get().sessionLocalTimeZone()), + "native worker used another session's SQL time zone"); + require(types.get(context).equals(SQLConf.get().getConfString("spark.sql.timestampType")), + "native worker used another session's timestamp type"); + org.apache.spark.sql.classic.SparkSession classic = + (org.apache.spark.sql.classic.SparkSession) session; + require(Thread.currentThread().getContextClassLoader() == classic.artifactManager().classloader(), + "native worker used another session's artifact classloader"); + String jobGroup = session.sparkContext().getLocalProperty("spark.jobGroup.id"); + require(jobGroup != null && jobGroup.startsWith("spark-connect-engine-"), + "native worker lost its current engine job group"); + threads.computeIfAbsent(context, ignored -> ConcurrentHashMap.newKeySet()) + .add(Thread.currentThread()); + observations.computeIfAbsent(context, ignored -> new AtomicInteger()).incrementAndGet(); + } + } + + private static void twoSessionWorkerReuse( + SparkConnectServiceGrpc.SparkConnectServiceBlockingStub stub, + String firstSession, String secondSession) { + setConfig(stub, firstSession, CONTEXT_KEY, "first", "spark.sql.session.timeZone", "UTC", + "spark.sql.timestampType", "TIMESTAMP_LTZ"); + setConfig(stub, secondSession, CONTEXT_KEY, "second", "spark.sql.session.timeZone", "Asia/Tokyo", + "spark.sql.timestampType", "TIMESTAMP_NTZ"); + ContextProbe probe = new ContextProbe(); + require(CONTEXT_PROBE.compareAndSet(null, probe), "another native context probe is active"); + try { + // ABAB can assign each session to a different worker in a stable two-worker rotation. + // Unequal same-session bursts exercise context switches without relying on that schedule. + int[] bursts = {3, 5, 7, 11, 17, 31}; + int call = 0; + for (int phase = 0; phase < bursts.length; phase++) { + for (int turn = 0; turn < 2; turn++) { + boolean first = (phase + turn) % 2 == 0; + for (int index = 0; index < bursts[phase]; index++) { + contextCall(stub, first ? firstSession : secondSession, !first, probe, call++); + } + } + } + Set shared = new HashSet<>(probe.threads.getOrDefault("first", Set.of())); + shared.retainAll(probe.threads.getOrDefault("second", Set.of())); + require(!shared.isEmpty(), "no attached worker was observed serving both native sessions: " + + probe.threads); + } finally { + CONTEXT_PROBE.compareAndSet(probe, null); + } + } + + private static void contextCall(SparkConnectServiceGrpc.SparkConnectServiceBlockingStub stub, + String session, boolean secondSessionContext, ContextProbe probe, int call) { + String context = secondSessionContext ? "second" : "first"; + AtomicInteger observed = + probe.observations.computeIfAbsent(context, ignored -> new AtomicInteger()); + int before = observed.get(); + // A distinct alias keeps every RPC's analyzer observation independently testable. + Relation relation = Relation.newBuilder().setSql(SQL.newBuilder() + .setQuery("SELECT CAST('2026-01-02 03:04:05' AS TIMESTAMP) AS native_context_" + call)).build(); + checkTimestamp(analyze(stub, schemaRequest(session, relation)), secondSessionContext); + require(observed.get() > before, "native analyzer context was not checked for RPC " + call); + } + + private static void nestedArrowIpcValues( + SparkConnectServiceGrpc.SparkConnectServiceBlockingStub stub, String session) throws Exception { + int depth = 12; + String value = "CAST(7 AS BIGINT)"; + for (int level = 0; level < depth; level++) { + value = level % 2 == 0 ? "named_struct('level_" + level + "', " + value + ")" + : "array(" + value + ")"; + } + Relation relation = Relation.newBuilder().setSql(SQL.newBuilder() + .setQuery("SELECT " + value + " AS nested")).build(); + String operation = UUID.randomUUID().toString(); + ExecutePlanRequest request = ExecutePlanRequest.newBuilder().setSessionId(session) + .setOperationId(operation).setPlan(Plan.newBuilder().setRoot(relation)).build(); + long rows = 0; + int batches = 0; + try { + java.util.Iterator stream = stub.withDeadlineAfter(30, TimeUnit.SECONDS) + .executePlan(request); + while (stream.hasNext()) { + ExecutePlanResponse response = stream.next(); + require(response.getSessionId().equals(session) + && response.getOperationId().equals(operation), + "deep Arrow execution changed response identity"); + if (!response.hasArrowBatch()) continue; + require(response.getArrowBatch().getRowCount() == 1, "deep Arrow response row count changed"); + try (BufferAllocator allocator = ArrowUtils$.MODULE$.rootAllocator() + .newChildAllocator("native-deep-arrow", 0, Long.MAX_VALUE); + ArrowStreamReader reader = new ArrowStreamReader(new ByteArrayInputStream( + response.getArrowBatch().getData().toByteArray()), allocator)) { + while (reader.loadNextBatch()) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + require(root.getRowCount() == 1 && root.getFieldVectors().size() == 1, + "deep Arrow batch lost its single row or column"); + Object actual = root.getVector(0).getObject(0); + // Walk every struct/list level to verify that the Arrow IPC round trip + // preserves the nested shape and leaf value. + for (int level = depth - 1; level >= 0; level--) { + if (level % 2 == 0) { + require(actual instanceof Map && ((Map) actual).size() == 1 + && ((Map) actual).containsKey("level_" + level), + "deep Arrow struct changed at level " + level); + actual = ((Map) actual).get("level_" + level); + } else { + require(actual instanceof List && ((List) actual).size() == 1, + "deep Arrow list changed at level " + level); + actual = ((List) actual).get(0); + } + } + require(actual instanceof Number && ((Number) actual).longValue() == 7, + "deep Arrow leaf value changed"); + rows += root.getRowCount(); + batches++; + } + } + } + require(rows == 1 && batches == 1, "deep Arrow execution did not return exactly one batch/row"); + } finally { + var released = stub.withDeadlineAfter(30, TimeUnit.SECONDS).releaseExecute( + ReleaseExecuteRequest.newBuilder().setSessionId(session).setOperationId(operation) + .setReleaseAll(ReleaseExecuteRequest.ReleaseAll.newBuilder()).build()); + // A completed, non-reattachable stream has already removed its operation. Release is + // idempotent: it preserves the session identity and leaves optional operation_id absent. + require(released.getSessionId().equals(session) && !released.hasOperationId(), + "completed deep Arrow operation release was not idempotent"); + } + } + + private static void emptyArrowIpcValues( + SparkConnectServiceGrpc.SparkConnectServiceBlockingStub stub, String session) throws Exception { + Relation relation = Relation.newBuilder().setSql(SQL.newBuilder().setQuery( + "SELECT '' AS text, unhex('') AS binary, CAST(array() AS ARRAY) AS items, " + + "array(CAST(array() AS ARRAY)) AS nested, array('') AS strings, " + + "CAST(map() AS MAP) AS mapping, " + + "CAST(NULL AS STRING) AS missing FROM range(3)")).build(); + String operation = UUID.randomUUID().toString(); + ExecutePlanRequest request = ExecutePlanRequest.newBuilder().setSessionId(session) + .setOperationId(operation).setPlan(Plan.newBuilder().setRoot(relation)).build(); + long rows = 0; + boolean complete = false; + java.util.Iterator stream = stub.withDeadlineAfter(30, TimeUnit.SECONDS) + .executePlan(request); + while (stream.hasNext()) { + ExecutePlanResponse response = stream.next(); + require(response.getSessionId().equals(session) + && response.getOperationId().equals(operation), "empty Arrow response identity changed"); + require(!complete, "empty Arrow response followed completion"); + if (response.hasResultComplete()) complete = true; + if (!response.hasArrowBatch()) continue; + long batchRows = 0; + try (BufferAllocator allocator = ArrowUtils$.MODULE$.rootAllocator() + .newChildAllocator("native-empty-arrow", 0, Long.MAX_VALUE); + ArrowStreamReader reader = new ArrowStreamReader(new ByteArrayInputStream( + response.getArrowBatch().getData().toByteArray()), allocator)) { + while (reader.loadNextBatch()) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + require(root.getFieldVectors().size() == 7, "empty Arrow fields changed"); + for (int row = 0; row < root.getRowCount(); row++) { + require(root.getVector(0).getObject(row).toString().isEmpty(), "empty UTF-8 value changed"); + require(((byte[]) root.getVector(1).getObject(row)).length == 0, "empty binary changed"); + require(((List) root.getVector(2).getObject(row)).isEmpty(), "empty list changed"); + List nested = (List) root.getVector(3).getObject(row); + require(nested.size() == 1 && ((List) nested.get(0)).isEmpty(), + "nested empty list changed during the Arrow IPC round trip"); + List strings = (List) root.getVector(4).getObject(row); + require(strings.size() == 1 && strings.get(0).toString().isEmpty(), + "all-empty string child changed during the Arrow IPC round trip"); + require(((List) root.getVector(5).getObject(row)).isEmpty(), "empty map changed"); + require(root.getVector(6).getObject(row) == null, "all-null string changed"); + } + batchRows += root.getRowCount(); + } + } + require(batchRows == response.getArrowBatch().getRowCount(), + "empty Arrow response row count changed"); + rows += batchRows; + } + require(rows == 3 && complete, "empty Arrow result lost rows or completion"); + } + + private static void lossySchemasAreRejected( + SparkConnectServiceGrpc.SparkConnectServiceBlockingStub stub, String session, + SparkSession spark) throws Exception { + Relation alias = Relation.newBuilder().setProject(Project.newBuilder() + .setInput(Relation.newBuilder().setRange(Range.newBuilder().setStart(0).setEnd(1).setStep(1))) + .addExpressions(Expression.newBuilder().setAlias(Expression.Alias.newBuilder() + .setExpr(Expression.newBuilder().setUnresolvedAttribute( + Expression.UnresolvedAttribute.newBuilder().setUnparsedIdentifier("id"))) + .addName("annotated").setMetadata("{\"comment\":\"must not disappear\"}")))).build(); + for (Relation relation : List.of(alias, Relation.newBuilder().setSql(SQL.newBuilder() + .setQuery("SELECT 'a' COLLATE UTF8_LCASE AS value")).build(), + Relation.newBuilder().setSql(SQL.newBuilder() + .setQuery("SELECT array('a' COLLATE UTF8_LCASE) AS value")).build())) { + unsupportedPlan(stub, session, relation); + } + String view = "native_udt_" + UUID.randomUUID().toString().replace("-", ""); + StructType schema = new StructType(new StructField[] { + DataTypes.createStructField("features", new VectorUDT(), true) + }); + spark.createDataFrame(java.util.Collections.emptyList(), schema) + .createOrReplaceGlobalTempView(view); + try { + unsupportedPlan(stub, session, Relation.newBuilder().setSql(SQL.newBuilder() + .setQuery("SELECT * FROM global_temp." + view)).build()); + } finally { + spark.catalog().dropGlobalTempView(view); + } + String cleanup = "spark.sql.classic.shuffleDependency.fileCleanup.enabled"; + Map before = getAllConfig(stub, session); + setConfig(stub, session, cleanup, "true"); + try { + unsupportedPlan(stub, session, Relation.newBuilder().setRange(Range.newBuilder() + .setStart(0).setEnd(1).setStep(1)).build()); + } finally { + if (before.containsKey(cleanup)) { + setConfig(stub, session, cleanup, before.get(cleanup)); + } else { + stub.withDeadlineAfter(30, TimeUnit.SECONDS).config(ConfigRequest.newBuilder() + .setSessionId(session).setOperation(ConfigRequest.Operation.newBuilder() + .setUnset(ConfigRequest.Unset.newBuilder().addKeys(cleanup))).build()); + } + require(before.equals(getAllConfig(stub, session)), + "cleanup guard test changed explicit configuration state"); + } + } + + private static void unsupportedPlan(SparkConnectServiceGrpc.SparkConnectServiceBlockingStub stub, + String session, Relation relation) { + Plan plan = Plan.newBuilder().setRoot(relation).build(); + for (AnalyzePlanRequest request : List.of(schemaRequest(session, relation), + AnalyzePlanRequest.newBuilder().setSessionId(session) + .setIsLocal(AnalyzePlanRequest.IsLocal.newBuilder().setPlan(plan)).build(), + AnalyzePlanRequest.newBuilder().setSessionId(session) + .setIsStreaming(AnalyzePlanRequest.IsStreaming.newBuilder().setPlan(plan)).build())) { + expectUnsupported(() -> analyze(stub, request)); + } + expectUnsupported(() -> { + java.util.Iterator responses = stub.withDeadlineAfter(30, TimeUnit.SECONDS) + .executePlan(ExecutePlanRequest.newBuilder().setSessionId(session) + .setOperationId(UUID.randomUUID().toString()).setPlan(plan).build()); + require(!responses.hasNext(), "unsupported schema produced an execute response"); + }); + } + + private static void expectUnsupported(Runnable operation) { + try { + operation.run(); + throw new AssertionError("unsupported plan was accepted"); + } catch (StatusRuntimeException error) { + require(error.getStatus().getCode() == Status.Code.UNIMPLEMENTED, + "unsupported plan returned the wrong native status: " + error); + } + } + + private static AnalyzePlanRequest schemaRequest(String session, Relation relation) { + return AnalyzePlanRequest.newBuilder().setSessionId(session) + .setSchema(AnalyzePlanRequest.Schema.newBuilder().setPlan(Plan.newBuilder().setRoot(relation))) + .build(); + } + + private static AnalyzePlanResponse analyze( + SparkConnectServiceGrpc.SparkConnectServiceBlockingStub stub, AnalyzePlanRequest request) { + return stub.withDeadlineAfter(30, TimeUnit.SECONDS).analyzePlan(request); + } + + private static void setConfig(SparkConnectServiceGrpc.SparkConnectServiceBlockingStub stub, + String session, String... alternating) { + require(alternating.length % 2 == 0, "native Config fields must be pairs"); + ConfigRequest.Set.Builder values = ConfigRequest.Set.newBuilder(); + for (int index = 0; index < alternating.length; index += 2) { + values.addPairs(KeyValue.newBuilder().setKey(alternating[index]).setValue(alternating[index + 1])); + } + require(stub.withDeadlineAfter(30, TimeUnit.SECONDS).config(ConfigRequest.newBuilder() + .setSessionId(session).setOperation(ConfigRequest.Operation.newBuilder().setSet(values)).build()) + .getSessionId().equals(session), "native Config Set changed session identity"); + } + + private static String getConfig(SparkConnectServiceGrpc.SparkConnectServiceBlockingStub stub, + String session, String key) { + var response = stub.withDeadlineAfter(30, TimeUnit.SECONDS).config(ConfigRequest.newBuilder() + .setSessionId(session).setOperation(ConfigRequest.Operation.newBuilder() + .setGet(ConfigRequest.Get.newBuilder().addKeys(key))).build()); + require(response.getSessionId().equals(session) && response.getPairsCount() == 1 + && response.getPairs(0).getKey().equals(key) && response.getPairs(0).hasValue(), + "native Config Get changed response identity or fields"); + return response.getPairs(0).getValue(); + } + + private static Map getAllConfig( + SparkConnectServiceGrpc.SparkConnectServiceBlockingStub stub, String session) { + var response = stub.withDeadlineAfter(30, TimeUnit.SECONDS).config(ConfigRequest.newBuilder() + .setSessionId(session).setOperation(ConfigRequest.Operation.newBuilder() + .setGetAll(ConfigRequest.GetAll.newBuilder())).build()); + Map values = new java.util.HashMap<>(); + for (KeyValue pair : response.getPairsList()) values.put(pair.getKey(), pair.getValue()); + return values; + } + + private static void checkTimestamp(AnalyzePlanResponse response, boolean secondSessionContext) { + check(response); + var type = response.getSchema().getSchema().getStruct().getFields(0).getDataType(); + require(secondSessionContext ? type.hasTimestampNtz() : type.hasTimestamp(), + "native analysis used another session's timestamp schema"); + } + + private static void check(AnalyzePlanResponse response) { + require(response.hasSchema() && response.getSchema().getSchema().hasStruct(), + "analysis did not return a struct schema"); + require(response.getSchema().getSchema().getStruct().getFieldsCount() == 1, + "analysis schema field count changed"); + } + + private static void require(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + } +} diff --git a/crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/NativeJniSmokeTest.java b/crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/NativeJniSmokeTest.java new file mode 100644 index 00000000..01671557 --- /dev/null +++ b/crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/NativeJniSmokeTest.java @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.spark.sql.connect.nativebridge.engine; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.spark.connect.proto.ExecutePlanRequest; +import org.apache.spark.connect.proto.ExecutePlanResponse; +import org.apache.spark.connect.proto.Plan; +import org.apache.spark.connect.proto.Range; +import org.apache.spark.connect.proto.Relation; +import org.apache.spark.connect.proto.ReleaseSessionRequest; +import org.apache.spark.connect.proto.ResultChunkingOptions; +import org.apache.spark.connect.proto.SQL; +import org.apache.spark.connect.proto.SparkConnectServiceGrpc; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.connect.service.SparkConnectService$; +import org.apache.spark.sql.util.ArrowUtils$; +import org.sparkproject.connect.grpc.ManagedChannel; +import org.sparkproject.connect.grpc.ManagedChannelBuilder; + +/** A minimal real start/query/stop JNI check against the pinned Spark assembly. */ +public final class NativeJniSmokeTest { + private NativeJniSmokeTest() {} + + public static void main(String[] arguments) throws Exception { + require(arguments.length == 1, "pass the absolute native library path"); + SparkSession spark = SparkSession.builder().appName("Native JNI CI smoke") + .config("spark.sql.artifact.isolation.enabled", "true") + .config("spark.sql.execution.arrow.compression.codec", "none") + .config("spark.sql.artifact.isolation.alwaysApplyClassloader", "true") + .getOrCreate(); + spark.sparkContext().setLogLevel("ERROR"); + try { + int port; + SparkJvmEngine engine = new SparkJvmEngine(spark); + try (engine; + NativeEngineServer server = new NativeEngineServer(arguments[0], "127.0.0.1:0", engine)) { + require(SparkConnectService$.MODULE$.server() == null, "Java Connect listener was started"); + String address = server.address(); + require(server.isRunning() && !server.isDrained(), + "live native listener reported stopped or drained"); + port = Integer.parseInt(address.substring(address.lastIndexOf(':') + 1)); + ManagedChannel channel = ManagedChannelBuilder.forTarget(address).usePlaintext().build(); + try { + var stub = SparkConnectServiceGrpc.newBlockingStub(channel) + .withDeadlineAfter(60, TimeUnit.SECONDS); + String session = UUID.randomUUID().toString(); + String operation = UUID.randomUUID().toString(); + ExecutePlanRequest request = ExecutePlanRequest.newBuilder().setSessionId(session) + .setOperationId(operation).setPlan(Plan.newBuilder().setRoot(Relation.newBuilder() + .setRange(Range.newBuilder().setStart(0).setEnd(25).setStep(1) + .setNumPartitions(2)))).build(); + java.util.Iterator responses = stub.executePlan(request); + long count = 0; + long sum = 0; + boolean schema = false; + boolean complete = false; + while (responses.hasNext()) { + ExecutePlanResponse response = responses.next(); + require(!complete, "response followed ResultComplete"); + require(session.equals(response.getSessionId()) + && operation.equals(response.getOperationId()), "response identity changed"); + if (response.hasSchema()) { + require(!schema && response.getSchema().hasStruct() + && response.getSchema().getStruct().getFieldsCount() == 1 + && response.getSchema().getStruct().getFields(0).getName().equals("id") + && response.getSchema().getStruct().getFields(0).getDataType().hasLong(), + "range schema changed"); + schema = true; + } + if (response.hasArrowBatch()) { + require(schema, "Arrow batch preceded the schema"); + long batchRows = 0; + try (BufferAllocator allocator = ArrowUtils$.MODULE$.rootAllocator() + .newChildAllocator("native-jni-ci-smoke", 0, Long.MAX_VALUE); + ArrowStreamReader reader = new ArrowStreamReader(new ByteArrayInputStream( + response.getArrowBatch().getData().toByteArray()), allocator)) { + while (reader.loadNextBatch()) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + require(root.getFieldVectors().size() == 1, "range Arrow columns changed"); + for (int row = 0; row < root.getRowCount(); row++) { + Object value = root.getVector(0).getObject(row); + require(value instanceof Number && ((Number) value).longValue() == count, + "range Arrow value/order changed at row " + count); + sum += ((Number) value).longValue(); + count++; + batchRows++; + } + } + } + require(batchRows == response.getArrowBatch().getRowCount(), + "range Arrow row count changed"); + } + if (response.hasResultComplete()) complete = true; + } + require(schema && complete && count == 25 && sum == 300, + "range result was incomplete or had incorrect values"); + chunkedRange(stub, session); + wholeLargeRowWithoutChunking(stub, session); + require(session.equals(stub.withDeadlineAfter(60, TimeUnit.SECONDS) + .releaseSession(ReleaseSessionRequest.newBuilder().setSessionId(session).build()) + .getSessionId()), "session release identity changed"); + } finally { + channel.shutdownNow(); + require(channel.awaitTermination(10, TimeUnit.SECONDS), "gRPC channel did not stop"); + } + server.close(); + require(engine.isClosed(), "native stop did not close the owned engine"); + } + require(!spark.sparkContext().isStopped(), "native stop closed the shared SparkContext"); + require(spark.range(1).count() == 1, "shared SparkContext stopped serving work"); + try (ServerSocket rebound = new ServerSocket()) { + rebound.setReuseAddress(true); + rebound.bind(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), port)); + require(rebound.isBound(), "native listener could not rebind after stop"); + } + System.out.println("NATIVE_JNI_SMOKE_OK: real JNI start/query/stop; schema and Arrow values; " + + "ResultComplete; negotiated byte chunks; whole oversized row; " + + "session release; listener health; listener rebind; " + + "shared SparkContext survives"); + } finally { + spark.stop(); + } + } + + private static void chunkedRange( + SparkConnectServiceGrpc.SparkConnectServiceBlockingStub stub, String session) throws Exception { + String operation = UUID.randomUUID().toString(); + ExecutePlanRequest request = ExecutePlanRequest.newBuilder().setSessionId(session) + .setOperationId(operation).setPlan(Plan.newBuilder().setRoot(Relation.newBuilder() + .setRange(Range.newBuilder().setStart(0).setEnd(10000).setStep(1) + .setNumPartitions(2)))) + .addRequestOptions(ExecutePlanRequest.RequestOption.newBuilder() + .setResultChunkingOptions(ResultChunkingOptions.newBuilder() + .setAllowArrowBatchChunking(true).setPreferredArrowChunkSize(1024))) + .build(); + var responses = stub.withDeadlineAfter(60, TimeUnit.SECONDS).executePlan(request); + Set responseIds = new HashSet<>(); + ByteArrayOutputStream pending = null; + long nextIndex = 0; + long expectedChunks = 0; + long expectedRows = 0; + long rows = 0; + int chunks = 0; + int batches = 0; + boolean schema = false; + boolean complete = false; + while (responses.hasNext()) { + ExecutePlanResponse response = responses.next(); + require(!complete, "chunked range response followed ResultComplete"); + require(session.equals(response.getSessionId()) + && operation.equals(response.getOperationId()), "chunked range identity changed"); + require(!response.getResponseId().isEmpty() && responseIds.add(response.getResponseId()), + "chunked range reused a response ID"); + if (response.hasSchema()) { + require(!schema, "chunked range repeated its schema header"); + schema = true; + } + if (response.hasArrowBatch()) { + require(schema, "chunked range data preceded its schema header"); + var chunk = response.getArrowBatch(); + require(chunk.hasChunkIndex() && chunk.hasNumChunksInBatch() + && chunk.hasStartOffset(), "negotiated range omitted chunk metadata"); + require(chunk.getData().size() <= 1024, "range exceeded preferred chunk payload"); + require(chunk.getStartOffset() == rows, "row offset advanced inside a chunked batch"); + require(chunk.getChunkIndex() == nextIndex, "range chunk order changed"); + if (pending == null) { + pending = new ByteArrayOutputStream(); + expectedChunks = chunk.getNumChunksInBatch(); + expectedRows = chunk.getRowCount(); + require(expectedChunks > 0 && expectedRows > 0, "invalid range batch metadata"); + } + require(chunk.getNumChunksInBatch() == expectedChunks + && chunk.getRowCount() == expectedRows, "range metadata changed within a batch"); + chunk.getData().writeTo(pending); + chunks++; + nextIndex++; + require(nextIndex <= expectedChunks, "range sent too many chunks"); + if (nextIndex == expectedChunks) { + long decodedRows = 0; + try (BufferAllocator allocator = ArrowUtils$.MODULE$.rootAllocator() + .newChildAllocator("native-jni-chunked-range", 0, Long.MAX_VALUE); + ArrowStreamReader reader = new ArrowStreamReader( + new ByteArrayInputStream(pending.toByteArray()), allocator)) { + require(reader.loadNextBatch(), "reassembled range omitted its Arrow batch"); + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + require(root.getFieldVectors().size() == 1, "chunked range column count changed"); + for (int row = 0; row < root.getRowCount(); row++) { + Object value = root.getVector(0).getObject(row); + require(value instanceof Number + && ((Number) value).longValue() == rows + decodedRows, + "reassembled range value/order changed"); + decodedRows++; + } + require(!reader.loadNextBatch(), "range IPC contained multiple record batches"); + } + require(decodedRows == expectedRows, "range batch row count changed after reassembly"); + rows += decodedRows; + batches++; + pending = null; + nextIndex = 0; + } + } + if (response.hasResultComplete()) { + require(pending == null, "ResultComplete truncated a chunked Arrow batch"); + complete = true; + } + } + require(schema && complete && pending == null && rows == 10000, + "chunked range was incomplete"); + require(batches > 0 && chunks > batches, "range did not exercise multi-chunk IPC reassembly"); + } + + private static void wholeLargeRowWithoutChunking( + SparkConnectServiceGrpc.SparkConnectServiceBlockingStub stub, String session) throws Exception { + String operation = UUID.randomUUID().toString(); + // Omit chunking opt-in. The IPC exceeds 3 MiB, while the complete protobuf + // message fits this raw client's default 4 MiB receive limit. + ExecutePlanRequest request = ExecutePlanRequest.newBuilder().setSessionId(session) + .setOperationId(operation).setPlan(Plan.newBuilder().setRoot(Relation.newBuilder() + .setSql(SQL.newBuilder().setQuery("SELECT repeat('x', 4000000) AS value")))) + .build(); + var responses = stub.withDeadlineAfter(60, TimeUnit.SECONDS).executePlan(request); + boolean schema = false; + boolean complete = false; + int batches = 0; + while (responses.hasNext()) { + ExecutePlanResponse response = responses.next(); + require(!complete, "large-row response followed ResultComplete"); + require(session.equals(response.getSessionId()) + && operation.equals(response.getOperationId()), "large-row response identity changed"); + if (response.hasSchema()) { + require(!schema, "large-row response repeated its schema header"); + schema = true; + } + if (response.hasArrowBatch()) { + require(schema, "large-row data preceded its schema header"); + var batch = response.getArrowBatch(); + require(!batch.hasChunkIndex() && !batch.hasNumChunksInBatch(), + "server chunked a batch without client opt-in"); + require(batch.hasStartOffset() && batch.getStartOffset() == 0 && batch.getRowCount() == 1, + "large-row offset or count changed"); + require(batch.getData().size() > 3 * 1024 * 1024, + "large-row probe did not exceed 3 MiB"); + try (BufferAllocator allocator = ArrowUtils$.MODULE$.rootAllocator() + .newChildAllocator("native-jni-whole-large-row", 0, Long.MAX_VALUE); + ArrowStreamReader reader = new ArrowStreamReader( + new ByteArrayInputStream(batch.getData().toByteArray()), allocator)) { + require(reader.loadNextBatch(), "large row omitted its Arrow record batch"); + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + require(root.getRowCount() == 1 && root.getFieldVectors().size() == 1, + "large-row Arrow shape changed"); + String value = root.getVector(0).getObject(0).toString(); + require(value.length() == 4000000 && value.chars().allMatch(character -> character == 'x'), + "large-row value changed"); + require(!reader.loadNextBatch(), "large-row IPC contained extra record batches"); + } + batches++; + } + if (response.hasResultComplete()) complete = true; + } + require(schema && complete && batches == 1, "large row was split or incomplete"); + } + + private static void require(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + } +} diff --git a/crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/NativeLibraryLoaderTest.java b/crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/NativeLibraryLoaderTest.java new file mode 100644 index 00000000..e4bc8dde --- /dev/null +++ b/crates/spark-connect-server/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine/NativeLibraryLoaderTest.java @@ -0,0 +1,267 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.spark.sql.connect.nativebridge.engine; + +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; +import java.security.MessageDigest; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HexFormat; +import java.util.Properties; + +/** JDK-only resource, integrity and actual native-loading checks. */ +public final class NativeLibraryLoaderTest { + private static final String ADAPTER_PROPERTIES = + "META-INF/spark-connect-native-adapter.properties"; + private static int checks; + + private NativeLibraryLoaderTest() {} + + public static void main(String[] arguments) throws Exception { + if (arguments.length == 1 && arguments[0].equals("--load-packaged")) { + Path library = NativeLibraryLoader.loadBundled(); + require(Files.isRegularFile(library), "packaged native library was not extracted"); + require(NativeLibraryLoader.loadBundled().equals(library), + "packaged load was not idempotent"); + System.out.println("NATIVE_PACKAGED_LOAD_OK"); + return; + } + require(arguments.length <= 1, "usage: NativeLibraryLoaderTest [absolute-fixture-library]"); + platforms(); + resources(); + if (arguments.length == 1) nativeFixture(Path.of(arguments[0])); + System.out.println("NATIVE_LIBRARY_LOADER_OK checks=" + checks); + } + + private static void platforms() throws Exception { + for (String os : new String[] {"Linux", "LINUX"}) { + for (String arch : new String[] {"amd64", "x86_64"}) { + require(NativeLibraryLoader.platform(os, arch).equals("linux-x86_64"), "Linux alias"); + } + for (String arch : new String[] {"arm64", "aarch64"}) { + require(NativeLibraryLoader.platform(os, arch).equals("linux-aarch64"), "ARM alias"); + } + } + for (String os : new String[] {"Mac OS X", "macOS", "Darwin"}) { + require(NativeLibraryLoader.platform(os, "arm64").equals("macos-aarch64"), "macOS alias"); + require(NativeLibraryLoader.platform(os, "amd64").equals("macos-x86_64"), "macOS x86 alias"); + } + expect(UnsupportedOperationException.class, "operating system", + () -> NativeLibraryLoader.platform("Windows 11", "amd64")); + expect(UnsupportedOperationException.class, "architecture", + () -> NativeLibraryLoader.platform("Linux", "riscv64")); + expect(IllegalArgumentException.class, "absolute", + () -> NativeLibraryLoader.loadAbsolute(Path.of("relative.so"))); + } + + private static void resources() throws Exception { + byte[] bytes = new byte[] {1, 4, 9, 16, 25}; + try (Fixture fixture = new Fixture(bytes)) { + Path extracted = fixture.extract(); + try { + require(Arrays.equals(bytes, Files.readAllBytes(extracted)), "extracted bytes changed"); + require(Files.getPosixFilePermissions(extracted.getParent()).equals( + PosixFilePermissions.fromString("rwx------")), "extraction directory is not private"); + require(Files.getPosixFilePermissions(extracted).equals( + PosixFilePermissions.fromString("rw-------")), "extracted library is not private"); + } finally { + Files.delete(extracted); + Files.delete(extracted.getParent()); + } + } + try (Fixture fixture = new Fixture(bytes)) { + Files.delete(fixture.root.resolve(ADAPTER_PROPERTIES)); + expect(IllegalStateException.class, "missing classpath resource", fixture::extract); + } + try (Fixture fixture = new Fixture(bytes)) { + Files.delete(fixture.root.resolve(fixture.prefix + "native.properties")); + expect(IllegalStateException.class, "matching platform native JAR", fixture::extract); + } + try (Fixture fixture = new Fixture(bytes)) { + Files.delete(fixture.root.resolve(fixture.prefix + fixture.libraryName)); + expect(IllegalStateException.class, "missing classpath resource", fixture::extract); + } + try (Fixture fixture = new Fixture(bytes)) { + fixture.nativeMetadata.setProperty("artifact.version", "other-version"); + fixture.writeNativeMetadata(); + expect(IllegalStateException.class, "artifact version mismatch", fixture::extract); + } + try (Fixture fixture = new Fixture(bytes)) { + fixture.adapterMetadata.setProperty("native.abi.version", "2"); + fixture.writeAdapterMetadata(); + expect(IllegalStateException.class, "adapter native ABI mismatch", fixture::extract); + } + try (Fixture fixture = new Fixture(bytes)) { + fixture.nativeMetadata.setProperty("native.abi.version", "2"); + fixture.writeNativeMetadata(); + expect(IllegalStateException.class, "native ABI mismatch", fixture::extract); + } + try (Fixture fixture = new Fixture(bytes)) { + fixture.nativeMetadata.setProperty("library.name", "../wrong.so"); + fixture.writeNativeMetadata(); + expect(IllegalStateException.class, "library name mismatch", fixture::extract); + } + try (Fixture fixture = new Fixture(bytes)) { + fixture.nativeMetadata.setProperty("library.sha256", "bad-digest"); + fixture.writeNativeMetadata(); + expect(IllegalStateException.class, "64 lowercase hexadecimal", fixture::extract); + } + try (Fixture fixture = new Fixture(bytes)) { + Files.write(fixture.root.resolve(fixture.prefix + fixture.libraryName), new byte[] {0}); + expect(IllegalStateException.class, "SHA-256 mismatch", fixture::extract); + } + try (Fixture fixture = new Fixture(new byte[0])) { + expect(IllegalStateException.class, "resource is empty", fixture::extract); + } + try (Fixture fixture = new Fixture(bytes)) { + fixture.nativeMetadata.remove("native.abi.version"); + fixture.writeNativeMetadata(); + expect(IllegalStateException.class, "missing native artifact property", fixture::extract); + } + try (Fixture fixture = new Fixture(bytes)) { + Files.writeString(fixture.root.resolve(ADAPTER_PROPERTIES), "x".repeat(16 * 1024 + 1)); + expect(IllegalStateException.class, "exceeds 16 KiB", fixture::extract); + } + for (String duplicate : new String[] {ADAPTER_PROPERTIES, "native.properties", "library"}) { + try (Fixture fixture = new Fixture(bytes)) { + Path second = Files.createTempDirectory("native-duplicate-test-"); + try { + String name = duplicate.equals(ADAPTER_PROPERTIES) ? duplicate : fixture.prefix + + (duplicate.equals("library") ? fixture.libraryName : duplicate); + Path destination = second.resolve(name); + Files.createDirectories(destination.getParent()); + Files.copy(fixture.root.resolve(name), destination); + try (URLClassLoader loader = new URLClassLoader(new URL[] { + fixture.root.toUri().toURL(), second.toUri().toURL()}, null)) { + expect(IllegalStateException.class, "multiple classpath resources", + () -> NativeLibraryLoader.extract(loader, fixture.os, fixture.arch)); + } + } finally { + removeTree(second); + } + } + } + } + + private static void nativeFixture(Path library) throws Exception { + require(library.isAbsolute() && Files.isRegularFile(library), + "fixture library must be an absolute file"); + try (Fixture fixture = new Fixture(Files.readAllBytes(library)); + URLClassLoader loader = fixture.loader()) { + Path loaded = NativeLibraryLoader.loadBundled(loader, fixture.os, fixture.arch); + require(fixtureValue() == 37, "fixture JNI symbol did not resolve after packaged loading"); + require(NativeLibraryLoader.loadBundled(loader, fixture.os, fixture.arch).equals(loaded), + "repeat packaged load extracted or loaded another copy"); + require(NativeLibraryLoader.loadAbsolute(loaded).equals(loaded), + "same absolute library was not reused"); + expect(IllegalStateException.class, "different Spark Connect native library", + () -> NativeLibraryLoader.loadAbsolute(library)); + } + } + + private static native int fixtureValue(); + + private static void require(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + checks++; + } + + @FunctionalInterface + private interface CheckedRunnable { + void run() throws Exception; + } + + private static void expect(Class type, String message, + CheckedRunnable action) throws Exception { + try { + action.run(); + } catch (Throwable failure) { + require(type.isInstance(failure), "unexpected exception " + failure); + require(failure.getMessage() != null && failure.getMessage().contains(message), + "missing diagnostic '" + message + "' in " + failure); + return; + } + throw new AssertionError("expected " + type.getSimpleName() + " containing " + message); + } + + private static void removeTree(Path root) throws IOException { + try (var entries = Files.walk(root)) { + for (Path entry : entries.sorted(Comparator.reverseOrder()).toList()) Files.delete(entry); + } + } + + private static final class Fixture implements AutoCloseable { + final Path root = Files.createTempDirectory("native-resource-test-"); + final String os = System.getProperty("os.name"); + final String arch = System.getProperty("os.arch"); + final String platform = NativeLibraryLoader.platform(os, arch); + final String prefix = "META-INF/native/" + platform + "/"; + final String libraryName = "libspark_connect_jni." + + (platform.startsWith("macos-") ? "dylib" : "so"); + final Properties adapterMetadata = new Properties(); + final Properties nativeMetadata = new Properties(); + + Fixture(byte[] library) throws Exception { + Files.createDirectories(root.resolve(prefix)); + adapterMetadata.setProperty("artifact.version", "test-1.0"); + adapterMetadata.setProperty("native.abi.version", "1"); + nativeMetadata.putAll(adapterMetadata); + nativeMetadata.setProperty("library.name", libraryName); + nativeMetadata.setProperty("library.sha256", + HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(library))); + writeAdapterMetadata(); + writeNativeMetadata(); + Files.write(root.resolve(prefix + libraryName), library); + } + + void writeAdapterMetadata() throws IOException { + writeProperties(root.resolve(ADAPTER_PROPERTIES), adapterMetadata); + } + + void writeNativeMetadata() throws IOException { + writeProperties(root.resolve(prefix + "native.properties"), nativeMetadata); + } + + Path extract() throws Exception { + try (URLClassLoader resources = loader()) { + return NativeLibraryLoader.extract(resources, os, arch); + } + } + + URLClassLoader loader() throws IOException { + return new URLClassLoader(new URL[] {root.toUri().toURL()}, null); + } + + @Override + public void close() throws IOException { + removeTree(root); + } + + private static void writeProperties(Path file, Properties values) throws IOException { + try (var output = Files.newBufferedWriter(file, StandardCharsets.ISO_8859_1)) { + values.store(output, "test metadata"); + } + } + } +} diff --git a/crates/spark-connect-server/jvm/test-engine.sh b/crates/spark-connect-server/jvm/test-engine.sh new file mode 100755 index 00000000..87c97456 --- /dev/null +++ b/crates/spark-connect-server/jvm/test-engine.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TEST_PYTHON="${NATIVE_TEST_PYTHON:-python3}" +# Compile and run against the same assembly when an explicit CLI path overrides +# the environment. Let argparse handle both option spellings and repeated flags. +SPARK_HOME="$("$TEST_PYTHON" -B - "$@" <<'PY' +import argparse +import os + +parser = argparse.ArgumentParser(add_help=False) +parser.add_argument("--spark-home", default=os.environ.get("SPARK_HOME")) +args, _ = parser.parse_known_args() +if args.spark_home is None: + parser.error("set SPARK_HOME or pass --spark-home") +print(args.spark_home) +PY +)" +export SPARK_HOME +"$TEST_PYTHON" -B "$PROJECT_DIR/jvm/test_acceptance_guards.py" +"$PROJECT_DIR/jvm/build.sh" --tests +exec "$TEST_PYTHON" -B "$PROJECT_DIR/jvm/run_acceptance.py" "$@" diff --git a/crates/spark-connect-server/jvm/test-jni-smoke.sh b/crates/spark-connect-server/jvm/test-jni-smoke.sh new file mode 100755 index 00000000..8563d58b --- /dev/null +++ b/crates/spark-connect-server/jvm/test-jni-smoke.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +if [[ $# -ne 1 || "$1" != /* || ! -f "$1" ]]; then + echo "usage: $0 /absolute/path/to/libspark_connect_jni.{so,dylib}" >&2 + exit 1 +fi +: "${SPARK_HOME:?Set SPARK_HOME to the pinned Spark assembly checkout}" +PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +"$PROJECT_DIR/jvm/build.sh" --tests +"${NATIVE_TEST_PYTHON:-python3}" -B - "$PROJECT_DIR" "$SPARK_HOME" "$1" <<'PY' +import json +import os +import re +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +root, spark_home, library = (Path(value).resolve(strict=True) for value in sys.argv[1:]) +reports = root / ".build" / "jvm-native" +reports.mkdir(parents=True, exist_ok=True) +build = Path(tempfile.mkdtemp(prefix="jni-smoke-", dir=reports)) +log_path = build / "native-jni-smoke.log" +command = [str(spark_home / "bin" / "spark-submit"), "--master", "local[2]", + "--driver-memory", "1g", "--conf", "spark.ui.enabled=false", + "--conf", "spark.driver.extraJavaOptions=-Xcheck:jni", + "--conf", "spark.driver.host=127.0.0.1", + "--conf", "spark.driver.bindAddress=127.0.0.1", + "--conf", "spark.sql.classic.shuffleDependency.fileCleanup.enabled=false", + "--conf", "spark.sql.warehouse.dir=" + str(build / "warehouse"), + "--class", "org.apache.spark.sql.connect.nativebridge.engine.NativeJniSmokeTest", + str(root / ".build" / "jvm" / "bridge-test.jar"), str(library)] +env = os.environ.copy() +env.update(SPARK_HOME=str(spark_home), SPARK_LOCAL_IP="127.0.0.1", SPARK_CONNECT_MODE="0") +env.pop("SPARK_REMOTE", None) +env.pop("PYSPARK_REMOTE_INIT_CONF_LEN", None) +started = time.monotonic() +report = {"command": command, "log": str(log_path), "passed": False} +try: + with log_path.open("w") as log: + result = subprocess.run(command, cwd=build, env=env, stdout=log, + stderr=subprocess.STDOUT, timeout=180) + report["exit_code"] = result.returncode + content = log_path.read_text(errors="replace") + print(content, end="", flush=True) + if result.returncode != 0: + raise RuntimeError("real JNI smoke failed; see " + str(log_path)) + if re.search(r"WARNING in native method|FATAL ERROR in native method", content): + raise RuntimeError("real JNI smoke triggered a JNI diagnostic; see " + str(log_path)) + if "NATIVE_JNI_SMOKE_OK:" not in content: + raise RuntimeError("real JNI smoke success marker missing; see " + str(log_path)) + report["passed"] = True +except subprocess.TimeoutExpired as error: + report["timed_out"] = True + print(log_path.read_text(errors="replace"), end="", flush=True) + raise RuntimeError("real JNI smoke exceeded 180 seconds; see " + str(log_path)) from error +finally: + report["seconds"] = round(time.monotonic() - started, 3) + (build / "result.json").write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report), flush=True) +PY diff --git a/crates/spark-connect-server/jvm/test-native-loader.sh b/crates/spark-connect-server/jvm/test-native-loader.sh new file mode 100755 index 00000000..f0a0b932 --- /dev/null +++ b/crates/spark-connect-server/jvm/test-native-loader.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +if [[ $# -ne 0 && ( $# -ne 2 || "$1" != "--native-jar" ) ]]; then + echo "usage: $0 [--native-jar ]" >&2 + exit 1 +fi +PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +: "${JAVA_HOME:?Set JAVA_HOME to a JDK 21 installation}" +BUILD_DIR="$(mktemp -d "${TMPDIR:-/tmp}/spark-connect-loader-test.XXXXXX")" +trap 'rm -rf -- "$BUILD_DIR"' EXIT +mkdir -p "$BUILD_DIR/classes/META-INF" "$BUILD_DIR/tmp" +SOURCE_ROOT="$PROJECT_DIR/jvm/src/main/java/org/apache/spark/sql/connect/nativebridge/engine" +TEST_ROOT="$PROJECT_DIR/jvm/src/test/java/org/apache/spark/sql/connect/nativebridge/engine" +"$JAVA_HOME/bin/javac" -proc:none -d "$BUILD_DIR/classes" \ + "$SOURCE_ROOT/NativeLibraryLoader.java" "$TEST_ROOT/NativeLibraryLoaderTest.java" +NATIVE_VERSION="$("${NATIVE_BUILD_PYTHON:-python3}" "$PROJECT_DIR/jvm/package-native.py" --print-version)" +printf 'artifact.version=%s\nnative.abi.version=1\n' "$NATIVE_VERSION" \ + > "$BUILD_DIR/classes/META-INF/spark-connect-native-adapter.properties" +cat > "$BUILD_DIR/fixture.c" <<'C' +#include +JNIEXPORT jint JNICALL +Java_org_apache_spark_sql_connect_nativebridge_engine_NativeLibraryLoaderTest_fixtureValue( + JNIEnv *env, jclass cls) { + (void)env; + (void)cls; + return 37; +} +C +case "$(uname -s)" in + Darwin) JNI_PLATFORM=darwin; LIBRARY="$BUILD_DIR/libfixture.dylib"; LINK_FLAGS=(-dynamiclib) ;; + Linux) JNI_PLATFORM=linux; LIBRARY="$BUILD_DIR/libfixture.so"; LINK_FLAGS=(-shared -fPIC) ;; + *) echo "loader tests require macOS or Linux" >&2; exit 1 ;; +esac +"${CC:-cc}" -std=c11 -Wall -Wextra -Werror "${LINK_FLAGS[@]}" \ + -I "$JAVA_HOME/include" -I "$JAVA_HOME/include/$JNI_PLATFORM" \ + "$BUILD_DIR/fixture.c" -o "$LIBRARY" +TEST_CLASS=org.apache.spark.sql.connect.nativebridge.engine.NativeLibraryLoaderTest +"$JAVA_HOME/bin/java" -Xcheck:jni "-Djava.io.tmpdir=$BUILD_DIR/tmp" \ + -cp "$BUILD_DIR/classes" "$TEST_CLASS" "$LIBRARY" +if [[ $# -eq 2 ]]; then + NATIVE_JAR="$(cd "$(dirname "$2")" && pwd)/$(basename "$2")" + [[ -f "$NATIVE_JAR" ]] || { echo "native JAR not found: $NATIVE_JAR" >&2; exit 1; } + "$JAVA_HOME/bin/java" -Xcheck:jni "-Djava.io.tmpdir=$BUILD_DIR/tmp" \ + -cp "$BUILD_DIR/classes:$NATIVE_JAR" "$TEST_CLASS" --load-packaged +fi +if [[ -n "$(find "$BUILD_DIR/tmp" -mindepth 1 -print -quit)" ]]; then + echo "native loader left extracted files after JVM exit" >&2 + exit 1 +fi diff --git a/crates/spark-connect-server/jvm/test_acceptance_guards.py b/crates/spark-connect-server/jvm/test_acceptance_guards.py new file mode 100644 index 00000000..a2affbf4 --- /dev/null +++ b/crates/spark-connect-server/jvm/test_acceptance_guards.py @@ -0,0 +1,315 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Check acceptance's shutdown and listener ownership evidence; no Spark needed.""" + +from pathlib import Path +import json +import os +import signal +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +from run_acceptance import ( + SHUTDOWN_COMPLETE, listener_owner_pids, shutdown_error, ipc_validation_commands, record_jni_check, +) + + +class IpcValidationCommandsTest(unittest.TestCase): + def test_separate_executor_and_result_size_modes_preserve_common_guards(self): + common = ["spark-submit", "--master", "local[2]", "--driver-memory", "1g", + "--driver-java-options", "-Xcheck:jni", "--conf", "spark.ui.enabled=false", + "--conf", "spark.driver.host=127.0.0.1", + "--conf", "spark.driver.bindAddress=127.0.0.1"] + original = list(common) + cluster, result_size = ipc_validation_commands(common, Path("tests with spaces.jar")) + self.assertEqual(common, original) + name, command, marker = cluster + self.assertEqual(name, "ipc-separate-executor") + self.assertEqual(marker, "IPC_BATCH_SMOKE_OK") + for option, value in (("--master", "local-cluster[1,1,1024]"), + ("--executor-memory", "512m"), ("--executor-cores", "1")): + self.assertEqual(command[command.index(option) + 1], value) + self.assertIn("-Xcheck:jni -XX:ActiveProcessorCount=2", command) + for setting in ("spark.executor.extraJavaOptions=-Xcheck:jni -XX:ActiveProcessorCount=2", + "spark.task.maxFailures=1", "spark.cores.max=1", + "spark.serializer=org.apache.spark.serializer.KryoSerializer", + "spark.kryo.registrationRequired=true", + "spark.rpc.netty.dispatcher.numThreads=2", + "spark.shuffle.io.serverThreads=2", "spark.shuffle.io.clientThreads=2"): + self.assertIn(setting, command) + name, bounded, marker = result_size + self.assertEqual(name, "ipc-result-size") + self.assertEqual(marker, "IPC_RESULT_SIZE_SMOKE_OK") + self.assertEqual(bounded[bounded.index("--master") + 1], "local[2]") + self.assertIn("spark.driver.maxResultSize=1m", bounded) + self.assertEqual(bounded[-1], "--result-size-only") + for command in (command, bounded): + self.assertIn("spark.ui.enabled=false", command) + self.assertIn("spark.driver.host=127.0.0.1", command) + self.assertIn("spark.driver.bindAddress=127.0.0.1", command) + self.assertIn("tests with spaces.jar", command) + self.assertNotIn("--worker-only", command) + + +class ShutdownEvidenceTest(unittest.TestCase): + def test_completed_cleanup_with_sigterm_is_accepted(self): + self.assertIsNone(shutdown_error(SHUTDOWN_COMPLETE + "\n", 128 + signal.SIGTERM)) + + def test_startup_cleanup_is_not_normal_shutdown(self): + content = "Embedded Spark Engine startup failed; cleanup complete\n" + self.assertIsNotNone(shutdown_error(content, 0)) + + def test_missing_cleanup_marker_is_rejected(self): + self.assertIsNotNone(shutdown_error("SparkContext stopped\n", 128 + signal.SIGTERM)) + + def test_native_stop_failure_is_rejected_even_with_marker(self): + content = "JNI stop is incomplete: active calls remain\n" + SHUTDOWN_COMPLETE + "\n" + self.assertIsNotNone(shutdown_error(content, 128 + signal.SIGTERM)) + + def test_launcher_cleanup_failure_is_rejected(self): + content = "Embedded Spark Engine shutdown failed: native stop incomplete\n" + self.assertIsNotNone(shutdown_error(content, 128 + signal.SIGTERM)) + + def test_resource_cleanup_failure_is_rejected_even_with_marker(self): + for diagnostic in ( + "JNI cleanup is incomplete; retained cleanup failed: cannot attach", + "Java engine handle cleanup failed: cannot enter Java engine context", + ): + with self.subTest(diagnostic=diagnostic): + content = diagnostic + "\n" + SHUTDOWN_COMPLETE + "\n" + self.assertIsNotNone(shutdown_error(content, 128 + signal.SIGTERM)) + + def test_successful_per_call_attachment_fallback_is_accepted(self): + content = ("JVM cleanup lane attachment failed; using per-call attachment: temporary failure\n" + + SHUTDOWN_COMPLETE + "\n") + self.assertIsNone(shutdown_error(content, 128 + signal.SIGTERM)) + + def test_abnormal_exit_is_rejected_even_with_marker(self): + self.assertIsNotNone(shutdown_error(SHUTDOWN_COMPLETE + "\n", 1)) + + def test_marker_requires_a_complete_line(self): + self.assertIsNotNone(shutdown_error(SHUTDOWN_COMPLETE + " (not really)\n", 0)) + + def test_other_server_cleanup_marker_is_rejected(self): + self.assertIsNotNone(shutdown_error("Embedded Spark Connect shutdown complete\n", 0)) + + +class JniDiagnosticTest(unittest.TestCase): + def test_warning_preserves_an_earlier_failure(self): + report = {"error": "client failed before shutdown"} + record_jni_check(report, "WARNING in native method: expected probe warning\n") + self.assertFalse(report["jni_check_clean"]) + self.assertEqual(report["error"], "client failed before shutdown") + + def test_warning_is_reported_when_it_is_the_first_failure(self): + report = {} + record_jni_check(report, "FATAL ERROR in native method: expected probe warning\n") + self.assertFalse(report["jni_check_clean"]) + self.assertIn("JNI check warning", report["error"]) + + def test_clean_log_does_not_add_a_failure(self): + report = {} + record_jni_check(report, SHUTDOWN_COMPLETE + "\n") + self.assertTrue(report["jni_check_clean"]) + self.assertNotIn("error", report) + + +class AssertionModeTest(unittest.TestCase): + def test_zero_optimization_reaches_input_validation_and_one_is_rejected(self): + # The deliberately absent library stops the runner before any run directory + # or Spark process can be created. Getting there proves the zero setting passed. + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for setting in ("0", "1"): + with self.subTest(setting=setting): + environment = os.environ.copy() + environment["PYTHONOPTIMIZE"] = setting + result = subprocess.run( + [sys.executable, "-B", str(Path(__file__).with_name("run_acceptance.py")), + "--library", str(root / "missing-native-library"), + "--spark-home", str(root)], + env=environment, capture_output=True, text=True, timeout=10) + self.assertNotEqual(result.returncode, 0) + if setting == "0": + self.assertIn("FileNotFoundError", result.stderr) + self.assertNotIn("acceptance checks require Python assertions", result.stderr) + else: + self.assertIn("acceptance checks require Python assertions", result.stderr) + self.assertNotIn("FileNotFoundError", result.stderr) + + +class ListenerOwnershipTest(unittest.TestCase): + @patch("run_acceptance.subprocess.run") + @patch("run_acceptance.shutil.which", return_value="/usr/sbin/lsof") + def test_listener_ownership_preserves_distinct_processes(self, which, run): + run.return_value = subprocess.CompletedProcess([], 0, "p7\nf1\np7\np11\n", "") + # lsof remains preferred even when /proc is unavailable. + self.assertEqual(listener_owner_pids(12345, proc_root=Path("/missing-proc")), [7, 11]) + + @patch("run_acceptance.subprocess.run") + @patch("run_acceptance.shutil.which", return_value=None) + def test_proc_fallback_finds_only_listening_ipv4_and_ipv6_owners(self, which, run): + with tempfile.TemporaryDirectory() as directory: + proc = Path(directory) + (proc / "net").mkdir() + (proc / "net" / "tcp").write_text( + " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n" + "0: 0100007F:3039 00000000:0000 0A 00000000:00000000 00:00000000 00000000 1000 0 11111\n" + "1: 0100007F:3039 00000000:0000 01 00000000:00000000 00:00000000 00000000 1000 0 33333\n" + "2: 0100007F:303A 00000000:0000 0A 00000000:00000000 00:00000000 00000000 1000 0 44444\n") + (proc / "net" / "tcp6").write_text( + " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n" + "0: 00000000000000000000000001000000:3039 00000000000000000000000000000000:0000 " + "0A 00000000:00000000 00:00000000 00000000 1000 0 22222\n") + for process, targets in ( + ("11", ("socket:[22222]",)), + ("7", ("socket:[11111]", "socket:[11111]", "pipe:[11111]")), + ("22", ("socket:[33333]",)), + ("33", ("socket:[44444]",)), + ("self", ("socket:[11111]",)), + ): + descriptors = proc / process / "fd" + descriptors.mkdir(parents=True) + for number, target in enumerate(targets): + (descriptors / str(number)).symlink_to(target) + # Processes may exit before their descriptors can be inspected. + (proc / "99").mkdir() + + self.assertEqual(listener_owner_pids(12345, proc_root=proc), [7, 11]) + run.assert_not_called() + + @patch("run_acceptance.shutil.which", return_value=None) + def test_proc_fallback_accepts_missing_optional_ipv6_table(self, which): + with tempfile.TemporaryDirectory() as directory: + proc = Path(directory) + (proc / "net").mkdir() + (proc / "net" / "tcp").write_text("sl local_address rem_address st\n") + self.assertEqual(listener_owner_pids(12345, proc_root=proc), []) + + @patch("run_acceptance.shutil.which", return_value=None) + def test_listener_ownership_requires_lsof_or_proc(self, which): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(RuntimeError, "requires lsof or Linux /proc"): + listener_owner_pids(12345, proc_root=Path(directory)) + + +class LauncherArgumentsTest(unittest.TestCase): + def launch(self, bundled, master=None): + # Exercise the actual launcher with paths containing spaces. /bin/bash is + # deliberately used so macOS also checks its system Bash 3.2 behavior. + with tempfile.TemporaryDirectory(prefix="native launch space ") as directory: + root = Path(directory) + project = root / "project" + jvm = project / "jvm" + jvm.mkdir(parents=True) + launcher = jvm / "run-engine.sh" + launcher.write_bytes(Path(__file__).with_name("run-engine.sh").read_bytes()) + build = jvm / "build.sh" + build.write_text("#!/bin/bash\nexit 0\n") + build.chmod(0o755) + spark = root / "spark home" + (spark / "bin").mkdir(parents=True) + submit = spark / "bin" / "spark-submit" + submit.write_text('#!/bin/bash\nprintf "%s\\0" "${SPARK_REMOTE-unset}" ' + '"$SPARK_CONNECT_MODE" "$@" > "$NATIVE_LAUNCH_CAPTURE"\n') + submit.chmod(0o755) + native = root / ("native artifact.jar" if bundled else "native artifact.so") + native.touch() + capture = root / "arguments" + environment = os.environ.copy() + environment.pop("SPARK_MASTER", None) + if master is not None: + environment["SPARK_MASTER"] = master + environment.update(SPARK_HOME=str(spark), SPARK_REMOTE="sc://must-be-cleared", + NATIVE_LAUNCH_CAPTURE=str(capture)) + arguments = ["--native-jar", str(native)] if bundled else [str(native)] + subprocess.run(["/bin/bash", str(launcher), *arguments, "127.0.0.1:0"], + env=environment, check=True, capture_output=True, timeout=10) + actual = capture.read_bytes().decode().rstrip("\0").split("\0") + expected = ["unset", "0", "--class", + "org.apache.spark.sql.connect.nativebridge.engine.EmbeddedSparkEngine", + "--master", master or "local[2]"] + if bundled: + expected += ["--jars", str(native)] + expected += [str(project / ".build/jvm/spark-connect-native-bridge.jar"), + "--bundled" if bundled else str(native), "127.0.0.1:0"] + self.assertEqual(actual, expected) + + def test_explicit_path_on_system_bash(self): + self.launch(False) + + def test_packaged_path_on_system_bash(self): + self.launch(True) + + def test_empty_ambient_master_on_system_bash(self): + self.launch(False, master="") + + def test_custom_master_on_system_bash(self): + self.launch(False, master="local[7]") + + +class AcceptanceAssemblyTest(unittest.TestCase): + def launch(self, option, environment_home): + with tempfile.TemporaryDirectory(prefix="native acceptance space ") as directory: + root = Path(directory) + jvm = root / "project" / "jvm" + jvm.mkdir(parents=True) + wrapper = jvm / "test-engine.sh" + wrapper.write_bytes(Path(__file__).with_name("test-engine.sh").read_bytes()) + (jvm / "test_acceptance_guards.py").write_text("pass\n") + build = jvm / "build.sh" + build.write_text('#!/bin/bash\nprintf "%s" "$SPARK_HOME" > "$NATIVE_BUILD_CAPTURE"\n') + build.chmod(0o755) + (jvm / "run_acceptance.py").write_text( + "import json, os, sys\nfrom pathlib import Path\n" + "Path(os.environ['NATIVE_RUN_CAPTURE']).write_text(" + "json.dumps([os.environ['SPARK_HOME'], sys.argv[1:]]))\n") + selected = str(root / "selected spark home") + arguments = ["--library", "native artifact.so"] + if option == "separate": + arguments += ["--spark-home", selected] + elif option == "equals": + arguments += ["--spark-home=" + selected] + environment = os.environ.copy() + environment.pop("SPARK_HOME", None) + if environment_home: + environment["SPARK_HOME"] = str(root / "environment spark home") + environment.update(NATIVE_TEST_PYTHON=sys.executable, + NATIVE_BUILD_CAPTURE=str(root / "built"), + NATIVE_RUN_CAPTURE=str(root / "ran")) + subprocess.run(["/bin/bash", str(wrapper), *arguments], env=environment, + check=True, capture_output=True, timeout=10) + expected = selected if option else environment["SPARK_HOME"] + self.assertEqual((root / "built").read_text(), expected) + self.assertEqual(json.loads((root / "ran").read_text()), [expected, arguments]) + + def test_cli_assembly_overrides_environment_on_system_bash(self): + self.launch("separate", True) + + def test_equals_assembly_without_environment_on_system_bash(self): + self.launch("equals", False) + + def test_environment_assembly_on_system_bash(self): + self.launch(None, True) + + +if __name__ == "__main__": + unittest.main() diff --git a/crates/spark-connect-server/jvm/test_package_native.py b/crates/spark-connect-server/jvm/test_package_native.py new file mode 100755 index 00000000..77689f57 --- /dev/null +++ b/crates/spark-connect-server/jvm/test_package_native.py @@ -0,0 +1,518 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Artifact checks use synthetic headers; CI separately loads the real release library.""" + +import hashlib +import importlib.util +import json +from pathlib import Path +import stat +import shutil +import struct +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch +import zipfile + + +SPEC = importlib.util.spec_from_file_location("package_native", Path(__file__).with_name("package-native.py")) +packaging = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(packaging) + + +def library_header(classifier): + _, kind, machine = packaging.PLATFORMS[classifier] + data = bytearray(80) + if kind == "elf": + data[:7] = b"\x7fELF\x02\x01\x01" + struct.pack_into("