Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 100 additions & 0 deletions .github/workflows/server-frontend.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#

name: Server frontend CI

on:
pull_request:
branches: [master, "branch-*"]
paths:
- ".github/workflows/server-frontend.yml"
- "Cargo.toml"
- "Cargo.lock"
- ".cargo/**"
- "rust-toolchain*"
- "crates/spark-connect-server/**"
- "crates/spark-connect-proto/**"
push:
branches: [master, "branch-*"]
paths:
- ".github/workflows/server-frontend.yml"
- "Cargo.toml"
- "Cargo.lock"
- ".cargo/**"
- "rust-toolchain*"
- "crates/spark-connect-server/**"
- "crates/spark-connect-proto/**"
workflow_dispatch:

permissions:
contents: read

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
CARGO_BUILD_JOBS: "2"
CARGO_PROFILE_TEST_DEBUG: "0"
# Strict linting uses the pinned toolchain in lint.yml.
RUSTFLAGS: ""

jobs:
frontend:
name: Frontend (Rust ${{ matrix.rust-version }})
runs-on: ubuntu-24.04
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
rust-version: [stable, "1.88.0"]
steps:
- uses: actions/checkout@v4

- name: Install Rust toolchain
# Use runner-provided rustup to keep actions limited to actions/*.
run: |
rustup toolchain install ${{ matrix.rust-version }} --profile minimal
rustup default ${{ matrix.rust-version }}

- name: Cache cargo and target
uses: actions/cache@v4
with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-${{ runner.arch }}-cargo-server-frontend-${{ matrix.rust-version }}-${{ hashFiles('Cargo.lock') }}
restore-keys: ${{ runner.os }}-${{ runner.arch }}-cargo-server-frontend-${{ matrix.rust-version }}-

- name: Install protobuf compiler
# Ubuntu 24.04 provides protoc with proto3 optional support (3.15+).
run: |
sudo apt-get update
sudo apt-get install -y protobuf-compiler
protoc --version

- name: Test frontend
# Protocol and injected-engine tests use local fixtures without an execution backend.
run: cargo test --locked -p spark-connect-server

- name: Check protocol feature combinations
run: |
cargo check --locked -p apache-spark-connect-proto
cargo check --locked -p apache-spark-connect-proto --features server
15 changes: 15 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ members = [
"crates/spark-connect-build",
"crates/pyspark-rs",
"examples",
"crates/spark-connect-server",
]
exclude = [
"examples/wasm-udfs",
Expand All @@ -44,6 +45,8 @@ exclude = [
# (so `cargo test --workspace` stays green without it). Build them explicitly with
# `cargo build --manifest-path examples/wasm-udfs/Cargo.toml` (etc) in CI or when
# the target is installed locally.
# The experimental protocol frontend is opt-in; build or test it with
# `-p spark-connect-server`. It is included in workspace lint checks.
default-members = [
"crates/spark-connect-proto",
"crates/spark-connect-core",
Expand Down Expand Up @@ -87,11 +90,13 @@ should_implement_trait = "allow"
tonic = { version = "0.14", default-features = false, features = ["channel", "codegen"] }
tonic-prost = "0.14"
tonic-prost-build = "0.14"
h2 = "0.4"
prost = "0.14"
prost-types = "0.14"

# async runtime
tokio = { version = "1", features = ["rt-multi-thread", "sync", "macros", "time"] }
tokio-stream = "0.1"
futures = "0.3"

# Arrow (result decoding).
Expand Down
11 changes: 11 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,17 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
See the [WASM UDF guide](https://apache.github.io/spark-connect-rust/udfs/) for
the full setup, SQL registration, supported types, and more.

## Experimental server frontend

The unpublished [`spark-connect-server`](crates/spark-connect-server/README.md)
library provides a Rust Spark Connect protocol frontend. It manages protocol sessions,
operations and Arrow IPC responses, delegating planning and execution to an
application-provided `Engine`. The library includes no execution backend or standalone
server executable; its protocol tests use injected engine fixtures.

The frontend is intended for local, trusted experiments. Read its
[security model](SECURITY.md#experimental-server-frontend) before embedding it.

## Contributing

Issues are tracked in ASF JIRA under
Expand Down
19 changes: 19 additions & 0 deletions SECURITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,22 @@ material, TLS validation) inherit from that model.
Automated security scanning agents should consult that
document for the project's in-scope / out-of-scope
declarations before reporting issues.

## Experimental server frontend

The optional `spark-connect-server` library is an experimental protocol
frontend for local, trusted workloads. It does not provide authentication,
authorization or TLS. Session IDs, operation IDs and client-provided user
context identify protocol state; they do not establish a caller's identity
or access rights.

Bind its listener only to loopback. The embedding application supplies the
listener and the `Engine` implementation; the library does not enforce the
listener's network scope. Any process that can reach it can submit requests
to that engine. Protocol limits and session isolation are resource and
correctness guards, not a security boundary between mutually untrusted users.

This frontend does not include an execution backend or a standalone server.
Its injected-engine tests establish frontend protocol behavior, not safe
exposure of an execution engine to untrusted clients. Apache Spark's operator
security guidance remains relevant to any application that connects an engine.
2 changes: 1 addition & 1 deletion crates/spark-connect-core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ prost-types.workspace = true
crc32fast = "1"
sha2 = "0.10"
# Transport errors that tonic keeps as a status's source, for `RetryPolicy::can_retry`.
h2 = "0.4"
h2.workspace = true
hyper = "1"
# The Basic credential and the percent-decoded userinfo of a proxy URI, already built for tonic.
base64 = "0.22"
Expand Down
3 changes: 3 additions & 0 deletions crates/spark-connect-proto/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ workspace = true
[lib]
name = "spark_connect_proto"

[features]
server = []

[dependencies]
tonic.workspace = true
tonic-prost.workspace = true
Expand Down
4 changes: 3 additions & 1 deletion crates/spark-connect-proto/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").ok_or("OUT_DIR is not set")?);
let full_set = out_dir.join("spark_connect_descriptor_set_full.bin");
tonic_prost_build::configure()
.build_server(false)
.build_server(cfg!(feature = "server"))
.build_client(true)
// async_trait repeats Future's must_use on generated server methods.
.trait_attribute("SparkConnectService", "#[allow(clippy::double_must_use)]")
.bytes(".")
.file_descriptor_set_path(&full_set)
.compile_protos(&protos, &[proto_root])?;
Expand Down
2 changes: 2 additions & 0 deletions crates/spark-connect-proto/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
//! All messages live in the `spark.connect` protobuf package and are re-exported
//! here at the crate root. The generated gRPC client is
//! [`spark_connect_service_client::SparkConnectServiceClient`].
//! Enable the optional `server` feature for `spark_connect_service_server`
//! bindings. Embeddings configure tonic's transport server features separately.
//!
//! Baseline protocol version: see `proto/PROTO_VERSION.txt` / `proto/SPARK_SHA.txt`.

Expand Down
41 changes: 41 additions & 0 deletions crates/spark-connect-server/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

[package]
name = "spark-connect-server"
version = "0.1.0"
edition.workspace = true
rust-version.workspace = true
license.workspace = true
publish = false
description = "Experimental Spark Connect server frontend with an injectable execution engine"

[lints]
workspace = true

[dependencies]
spark-connect-proto = { package = "apache-spark-connect-proto", path = "../spark-connect-proto", features = ["server"] }
arrow.workspace = true
futures.workspace = true
prost.workspace = true
tokio = { workspace = true, features = ["net"] }
tokio-stream.workspace = true
tonic = { workspace = true, features = ["server", "router"] }
uuid.workspace = true

[dev-dependencies]
h2.workspace = true
Loading
Loading