From 26f5662afc545d5b226e2979255995c582638ee9 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Fri, 30 Oct 2026 13:38:05 -0700 Subject: [PATCH 1/3] test(e2e): run the testkit conformance suite in the host lanes The Docker, Podman, Kubernetes and VM host lanes, including external-driver mode and e2e:rust:push, now run the testkit `sandbox` package with nextest instead of the old CLI conformance package. The tests find the lane's gateway the same way the CLI does, through the OPENSHELL_GATEWAY_ENDPOINT and OPENSHELL_GATEWAY values the gateway wrappers export, and `--no-tests fail` makes a lane that selects no tests fail. Signed-off-by: Matthew Grossman --- TESTING.md | 20 ++++++++++---------- e2e/support/conformance.sh | 12 +++++++----- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/TESTING.md b/TESTING.md index a2047177e5..5ca134ae50 100644 --- a/TESTING.md +++ b/TESTING.md @@ -412,23 +412,23 @@ HTTP `Host`, TLS SNI, and mTLS handling. Suites: - Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests. -- CLI conformance (`tests/suites/conformance`) - portable Cargo tests for - lifecycle, mechanistic drafts, file transfer, and the sandbox-local API, - including agent-authored permission requests. Driver E2E runs the complete - Cargo test package. The installed-artifact conformance lanes run the `sandbox` - package through `openshell-test-runner` (see below). +- Conformance (`tests/suites/conformance`) - the testkit `sandbox` package: + lifecycle, the CLI, persistent workspaces, file transfer, and the policy + advisor. Driver E2E runs it with nextest against the lane's gateway. The + installed-artifact conformance lanes run the same package through + `openshell-test-runner` (see above). - Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or - `e2e-vm`) - CLI conformance plus the common and driver-specific coverage for + `e2e-vm`) - conformance plus the common and driver-specific coverage for the selected deployment. - Docker suite (`--features e2e-docker`) - includes Docker-only coverage such as Dockerfile image builds, Docker preflight checks, and managed Docker gateway start. - Docker GPU suite (`--features e2e-docker-gpu`) - Docker suite plus GPU sandbox smoke coverage. -- Configuration push suite (`--features e2e-config-push`) - CLI conformance and the policy tests against a Docker gateway with `config_delivery_mode = "push"`. Run it with `mise run e2e:rust:push`. +- Configuration push suite (`--features e2e-config-push`) - conformance and the policy tests against a Docker gateway with `config_delivery_mode = "push"`. Run it with `mise run e2e:rust:push`. - VM suite (`--features e2e-vm`) - runs e2e tests on a VM. - Kubernetes credential-driver suite (`--features e2e-kubernetes-credential-drivers`) - targeted Kubernetes Secrets and Vault provider credential storage coverage. -The Docker, Podman, Kubernetes, and VM runners execute CLI -conformance before their Rust tests. Set the driver's feature variable -(`OPENSHELL_E2E_DOCKER_FEATURES`, `OPENSHELL_E2E_PODMAN_FEATURES`, +The Docker, Podman, Kubernetes, and VM runners execute conformance before +their Rust tests; a run with no conformance tests fails. Set the driver's +feature variable (`OPENSHELL_E2E_DOCKER_FEATURES`, `OPENSHELL_E2E_PODMAN_FEATURES`, `OPENSHELL_E2E_KUBERNETES_FEATURES`, or `OPENSHELL_E2E_VM_FEATURES`) to `e2e-conformance-only` to skip the Rust tests; the runner logs a notice when it skips them. Unset or empty values select the driver's default Rust diff --git a/e2e/support/conformance.sh b/e2e/support/conformance.sh index 9fc898b057..c0057e50f0 100644 --- a/e2e/support/conformance.sh +++ b/e2e/support/conformance.sh @@ -30,15 +30,17 @@ e2e_run_openshell_conformance() { e2e_require_openshell_bin || return + # The tests find the gateway as the CLI does: OPENSHELL_GATEWAY_ENDPOINT, + # else OPENSHELL_GATEWAY, else the active gateway. The gateway wrappers set + # these for the gateway they started. echo "==> Running conformance suites against the ${gateway_label} gateway" - cargo test \ + cargo nextest run \ --locked \ --manifest-path "${root}/tests/suites/conformance/Cargo.toml" \ - --package openshell-test-conformance-cli \ + --package sandbox \ + --test-threads 1 \ --no-fail-fast \ - -- \ - --test-threads=1 \ - --nocapture + --no-tests fail } if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then From 41426f702cb0d6a0c576ab63f2554ee2426acf07 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Fri, 30 Oct 2026 13:38:10 -0700 Subject: [PATCH 2/3] test(conformance): delete the old CLI conformance suite Nothing runs tests/suites/conformance/cli now that the host lanes run the testkit suite. Remove it from the conformance workspace and its lockfile, drop the e2e:testkit exclusion for it, and point the TESTING.md manual run at the `sandbox` package. e2e:rust:push also loses its dependency on e2e:conformance:build, which no longer exists, and the unused OPENSHELL_CONFORMANCE_BIN it set. crates/openshell-conformance stays for the features and drivers suites. Signed-off-by: Matthew Grossman --- TESTING.md | 44 +++++-------------- tasks/test.toml | 7 ++- tests/suites/conformance/Cargo.lock | 19 -------- tests/suites/conformance/Cargo.toml | 2 +- tests/suites/conformance/cli/Cargo.toml | 14 ------ .../conformance/cli/tests/file_transfer.rs | 40 ----------------- .../suites/conformance/cli/tests/lifecycle.rs | 22 ---------- .../conformance/cli/tests/policy_advisor.rs | 37 ---------------- tests/suites/conformance/cli/tests/smoke.rs | 24 ---------- 9 files changed, 14 insertions(+), 195 deletions(-) delete mode 100644 tests/suites/conformance/cli/Cargo.toml delete mode 100644 tests/suites/conformance/cli/tests/file_transfer.rs delete mode 100644 tests/suites/conformance/cli/tests/lifecycle.rs delete mode 100644 tests/suites/conformance/cli/tests/policy_advisor.rs delete mode 100644 tests/suites/conformance/cli/tests/smoke.rs diff --git a/TESTING.md b/TESTING.md index 5ca134ae50..ba1b903733 100644 --- a/TESTING.md +++ b/TESTING.md @@ -454,52 +454,28 @@ Run the Docker-backed Rust CLI e2e suite: mise run e2e:docker ``` -Run the portable CLI conformance suite against the gateway selected in your -OpenShell CLI configuration: +Run the conformance suite against the gateway selected in your OpenShell CLI +configuration: ```shell cargo build --package openshell-cli OPENSHELL_BIN="$PWD/target/debug/openshell" \ - cargo test \ + cargo nextest run \ --locked \ --manifest-path tests/suites/conformance/Cargo.toml \ - --package openshell-test-conformance-cli \ - --no-fail-fast \ - -- \ - --test-threads=1 \ - --nocapture + --package sandbox \ + --test-threads 1 \ + --no-fail-fast ``` The gateway must already be installed, reachable, and selected before the tests start. The test suite does not provision a gateway or select a compute driver. Set `OPENSHELL_BIN` to another executable to test a different prebuilt CLI. +[tests/testkit/README.md](tests/testkit/README.md) covers gateway selection and +cleanup. -The phase-1 scenario verifies the complete CLI-to-gateway-to-driver path without -depending on how the gateway was installed or which driver is configured. It -requires machine-readable gRPC status, creates a uniquely named detached -sandbox with the configured default image, verifies the sandbox is `Ready` by finding its -unique name in paginated JSON list output, executes `echo` with a run-specific -marker, deletes the sandbox, and verifies that its name no longer appears. -Driver suites enable the same profile -instead of maintaining a separate smoke implementation. Sandbox lifecycle, -label matrices, VM overlay, and TLS-key permission assertions remain regular -E2E coverage. - -Each invocation prints a ten-character run ID before creating resources. -Conformance sandboxes use names such as `ct--01`. The runner tracks the -exact name and uses it for cleanup; phase 1 does not add ownership labels. - -The runner deletes owned resources after both success and failure. If the test -process is interrupted before cleanup, locate leftovers without touching -unrelated gateway state: - -```shell -openshell sandbox list --output json -openshell sandbox delete -``` - -Gateway-backed Rust E2E tasks run the CLI conformance Cargo tests against the -configured gateway, then run any lane-specific Rust tests that still apply. +Gateway-backed Rust E2E tasks run the conformance suite against the configured +gateway, then run any lane-specific Rust tests that still apply. Run the Podman-backed Rust CLI e2e suite: ```shell diff --git a/tasks/test.toml b/tasks/test.toml index 6bafa95e9e..e646f3cd2f 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -169,13 +169,12 @@ run = "OPENSHELL_TEST_CONTAINER_ENGINE=podman e2e/with-podman-gateway.sh cargo t ["e2e:testkit"] description = "Run the testkit suites in tests/suites/conformance against a Docker-backed gateway" -run = "e2e/with-docker-gateway.sh cargo nextest run --locked --manifest-path tests/suites/conformance/Cargo.toml --workspace --exclude openshell-test-conformance-cli --test-threads 1 --no-fail-fast" +run = "e2e/with-docker-gateway.sh cargo nextest run --locked --manifest-path tests/suites/conformance/Cargo.toml --workspace --test-threads 1 --no-fail-fast" ["e2e:rust:push"] -description = "Run CLI conformance and policy e2e tests against a Docker gateway that pushes configuration" -depends = ["e2e:conformance:build"] +description = "Run conformance and policy e2e tests against a Docker gateway that pushes configuration" env = { OPENSHELL_CONFIG_DELIVERY_MODE = "push", OPENSHELL_E2E_DOCKER_FEATURES = "e2e-config-push", OPENSHELL_E2E_DOCKER_TEST = "config_push live_policy_update policy_activation" } -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-docker.sh" +run = "e2e/rust/e2e-docker.sh" ["e2e:workload:build"] description = "Build the Docker E2E workload fixture" diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index d8b1b77bea..c4155fe13d 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -1879,17 +1879,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "openshell-conformance" -version = "0.0.0" -dependencies = [ - "rand 0.9.4", - "serde", - "serde_json", - "tempfile", - "tokio", -] - [[package]] name = "openshell-core" version = "0.0.0" @@ -1992,14 +1981,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "openshell-test-conformance-cli" -version = "0.0.0" -dependencies = [ - "openshell-conformance", - "tokio", -] - [[package]] name = "openshell-testkit" version = "0.0.0" diff --git a/tests/suites/conformance/Cargo.toml b/tests/suites/conformance/Cargo.toml index 8dad4ea1ff..51acf3bb41 100644 --- a/tests/suites/conformance/Cargo.toml +++ b/tests/suites/conformance/Cargo.toml @@ -3,4 +3,4 @@ [workspace] resolver = "2" -members = ["cli", "sandbox"] +members = ["sandbox"] diff --git a/tests/suites/conformance/cli/Cargo.toml b/tests/suites/conformance/cli/Cargo.toml deleted file mode 100644 index 5edcefe687..0000000000 --- a/tests/suites/conformance/cli/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -[package] -name = "openshell-test-conformance-cli" -version = "0.0.0" -edition = "2024" -rust-version = "1.94" -license = "Apache-2.0" -repository = "https://github.com/NVIDIA/OpenShell" - -[dependencies] -openshell-conformance = { path = "../../../../crates/openshell-conformance" } -tokio = { version = "1.43", features = ["macros", "rt"] } diff --git a/tests/suites/conformance/cli/tests/file_transfer.rs b/tests/suites/conformance/cli/tests/file_transfer.rs deleted file mode 100644 index 167506b87b..0000000000 --- a/tests/suites/conformance/cli/tests/file_transfer.rs +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Driver-agnostic sandbox file-transfer conformance tests. - -use openshell_conformance::{ - FILE_TRANSFER_GIT_FILTERING_SCENARIO, FILE_TRANSFER_PATH_SAFETY_SCENARIO, - FILE_TRANSFER_ROUND_TRIP_SCENARIO, OpenShellRunner, Scenario, -}; - -/// Exercise file and directory round trips through the candidate CLI. -#[tokio::test] -async fn round_trip() { - run(FILE_TRANSFER_ROUND_TRIP_SCENARIO).await; -} - -/// Exercise Git-aware upload filtering through the candidate CLI. -#[tokio::test] -async fn git_filtering() { - run(FILE_TRANSFER_GIT_FILTERING_SCENARIO).await; -} - -/// Exercise workspace boundary and filename safety through the candidate CLI. -#[tokio::test] -async fn path_safety() { - run(FILE_TRANSFER_PATH_SAFETY_SCENARIO).await; -} - -async fn run(scenario: Scenario) { - let mut runner = OpenShellRunner::from_env(scenario.name) - .expect("candidate openshell CLI is available"); - let result = async { - runner.check_gateway_status().await?; - scenario.run(&mut runner).await - } - .await; - if let Err(error) = runner.finish(result).await { - panic!("{} conformance scenario failed:\n{error}", scenario.name); - } -} diff --git a/tests/suites/conformance/cli/tests/lifecycle.rs b/tests/suites/conformance/cli/tests/lifecycle.rs deleted file mode 100644 index 1a18adef86..0000000000 --- a/tests/suites/conformance/cli/tests/lifecycle.rs +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Driver-agnostic sandbox lifecycle conformance tests. - -use openshell_conformance::{OpenShellRunner, SANDBOX_LIFECYCLE_SCENARIO}; - -/// Exercise stop, start, and stopped-deletion behavior through the candidate CLI. -#[tokio::test] -async fn sandbox_lifecycle() { - let mut runner = OpenShellRunner::from_env(SANDBOX_LIFECYCLE_SCENARIO.name) - .expect("candidate openshell CLI is available"); - - let result = async { - runner.check_gateway_status().await?; - SANDBOX_LIFECYCLE_SCENARIO.run(&mut runner).await - } - .await; - if let Err(error) = runner.finish(result).await { - panic!("sandbox lifecycle conformance scenario failed:\n{error}"); - } -} diff --git a/tests/suites/conformance/cli/tests/policy_advisor.rs b/tests/suites/conformance/cli/tests/policy_advisor.rs deleted file mode 100644 index 7da2186dd7..0000000000 --- a/tests/suites/conformance/cli/tests/policy_advisor.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Installed-artifact policy advisor conformance. - -use openshell_conformance::{ - MECHANISTIC_PROPOSAL_SCENARIO, NEW_HOSTNAME_PROPOSAL_SCENARIO, OpenShellRunner, - POLICY_LOCAL_SCENARIO, Scenario, -}; - -async fn run(scenario: &'static Scenario) { - let mut runner = - OpenShellRunner::from_env(scenario.name).expect("candidate openshell CLI is available"); - let result = async { - runner.check_gateway_status().await?; - scenario.run(&mut runner).await - } - .await; - if let Err(error) = runner.finish(result).await { - panic!("{} conformance scenario failed:\n{error}", scenario.name); - } -} - -#[tokio::test] -async fn mechanistic_proposal() { - run(&MECHANISTIC_PROPOSAL_SCENARIO).await; -} - -#[tokio::test] -async fn new_hostname_proposal() { - run(&NEW_HOSTNAME_PROPOSAL_SCENARIO).await; -} - -#[tokio::test] -async fn policy_local() { - run(&POLICY_LOCAL_SCENARIO).await; -} diff --git a/tests/suites/conformance/cli/tests/smoke.rs b/tests/suites/conformance/cli/tests/smoke.rs deleted file mode 100644 index 55306e852f..0000000000 --- a/tests/suites/conformance/cli/tests/smoke.rs +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Driver-agnostic `OpenShell` CLI conformance tests. - -use openshell_conformance::{OpenShellRunner, SMOKE_SCENARIO}; - -/// Exercise the public CLI against a provisioned `OpenShell` gateway. -/// -/// The test runner supplies the candidate CLI explicitly so the same archive -/// can validate artifacts installed into any supported test guest. -#[tokio::test] -async fn smoke() { - let mut runner = OpenShellRunner::from_env(SMOKE_SCENARIO.name) - .expect("candidate openshell CLI is available"); - let result = async { - runner.check_gateway_status().await?; - SMOKE_SCENARIO.run(&mut runner).await - } - .await; - if let Err(error) = runner.finish(result).await { - panic!("conformance smoke scenario failed:\n{error}"); - } -} From 81cdd4ef4c100e701e5172f60538774fe9533f8c Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 2 Nov 2026 08:42:20 -0800 Subject: [PATCH 3/3] test(e2e): drop the package filter from the host-lane conformance run The conformance workspace has one package now that the old CLI suite is gone, so run the whole workspace. Signed-off-by: Matthew Grossman --- TESTING.md | 1 - e2e/support/conformance.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/TESTING.md b/TESTING.md index ba1b903733..cedc77fbf6 100644 --- a/TESTING.md +++ b/TESTING.md @@ -463,7 +463,6 @@ OPENSHELL_BIN="$PWD/target/debug/openshell" \ cargo nextest run \ --locked \ --manifest-path tests/suites/conformance/Cargo.toml \ - --package sandbox \ --test-threads 1 \ --no-fail-fast ``` diff --git a/e2e/support/conformance.sh b/e2e/support/conformance.sh index c0057e50f0..26cb3ed9f6 100644 --- a/e2e/support/conformance.sh +++ b/e2e/support/conformance.sh @@ -37,7 +37,6 @@ e2e_run_openshell_conformance() { cargo nextest run \ --locked \ --manifest-path "${root}/tests/suites/conformance/Cargo.toml" \ - --package sandbox \ --test-threads 1 \ --no-fail-fast \ --no-tests fail