diff --git a/.agents/skills/watch-github-actions/SKILL.md b/.agents/skills/watch-github-actions/SKILL.md index 47ffa02413..f2f59f741e 100644 --- a/.agents/skills/watch-github-actions/SKILL.md +++ b/.agents/skills/watch-github-actions/SKILL.md @@ -131,7 +131,7 @@ descriptions, events, and current/previous container logs. A CNI rollout timeout occurs before the listener tests run; use these diagnostics to distinguish image pulls, init failures, and readiness failures. -For tmachine Kubernetes conformance failures, inspect the `e2e-results-conformance-sandbox-ubuntu-k3s-*` artifact. Its `diagnostics/` holds the K3s report with Agent Sandbox controller state, leader leases, events, and current and previous controller logs. The k3s installers check reconciliation after guest boot with a temporary scheduling-gated Sandbox probe, so a failed probe fails provisioning before any test runs. Distinguish a failed controller probe from a failed conformance scenario; gateway connectivity and controller Deployment availability do not prove reconciliation works. +For tmachine Kubernetes conformance failures, inspect the `e2e-results-conformance-conformance-ubuntu-k3s-*` artifact. Its `diagnostics/` holds the K3s report with Agent Sandbox controller state, leader leases, events, and current and previous controller logs. The k3s installers check reconciliation after guest boot with a temporary scheduling-gated Sandbox probe, so a failed probe fails provisioning before any test runs. Distinguish a failed controller probe from a failed conformance scenario; gateway connectivity and controller Deployment availability do not prove reconciliation works. For the `e2e-suites` lane (`.github/workflows/e2e-suites.yml`), the last lines of the job log give the runner's result: PASS, FAIL (exit 1) after the end of the nextest log (which names the failed tests), or ERROR (exit 2) with the step that broke. FAIL and ERROR also appear as annotations on the run's summary page. Then inspect the `e2e-results----` artifact, which holds `junit.xml`, `nextest.log` and, after a failure, the guest's `diagnostics/`. diff --git a/.github/workflows/branch-checks.yml b/.github/workflows/branch-checks.yml index f4341db4b6..a62d442bc9 100644 --- a/.github/workflows/branch-checks.yml +++ b/.github/workflows/branch-checks.yml @@ -233,7 +233,7 @@ jobs: cargo fmt --manifest-path tests/testkit/Cargo.toml --all -- --check cargo fmt --manifest-path tests/tmachine/Cargo.toml --all -- --check cargo fmt --manifest-path tests/runner/Cargo.toml --all -- --check - cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox -- --check + cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p conformance -- --check cargo fmt --manifest-path tests/suites/features/Cargo.toml -p keycloak-provider-refresh -- --check cargo fmt --manifest-path tests/support/Cargo.toml --all -- --check @@ -297,7 +297,7 @@ jobs: cargo clippy --locked --manifest-path tests/testkit/Cargo.toml --all-targets -- -D warnings cargo clippy --locked --manifest-path tests/tmachine/Cargo.toml --all-targets -- -D warnings cargo clippy --locked --manifest-path tests/runner/Cargo.toml --all-targets -- -D warnings - cargo clippy --locked --manifest-path tests/suites/conformance/Cargo.toml -p sandbox --all-targets -- -D warnings + cargo clippy --locked --manifest-path tests/suites/conformance/Cargo.toml -p conformance --all-targets -- -D warnings cargo clippy --locked --manifest-path tests/suites/features/Cargo.toml -p keycloak-provider-refresh --all-targets -- -D warnings cargo clippy --locked --manifest-path tests/support/Cargo.toml --all-targets -- -D warnings diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index ea75dfebaa..5e47052d91 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -64,11 +64,11 @@ jobs: # installer registers with the CLI: `openshell` for the packages, # `tmachine` for the k3s installers. rows='[ - {"environment":"ubuntu-docker-rootful","installer":"deb","family":"conformance","package":"sandbox","gateway":"openshell","runner":"ubuntu-24.04"}, - {"environment":"fedora-podman-rootful","installer":"rpm","family":"conformance","package":"sandbox","gateway":"openshell","runner":"ubuntu-24.04"}, - {"environment":"fedora-podman-rootless","installer":"rpm","family":"conformance","package":"sandbox","gateway":"openshell","runner":"ubuntu-24.04"}, - {"environment":"ubuntu-k3s","installer":"k3s","family":"conformance","package":"sandbox","gateway":"tmachine","runner":"ubuntu-24.04"}, - {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"conformance","package":"sandbox","gateway":"tmachine","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-docker-rootful","installer":"deb","family":"conformance","package":"conformance","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"fedora-podman-rootful","installer":"rpm","family":"conformance","package":"conformance","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"fedora-podman-rootless","installer":"rpm","family":"conformance","package":"conformance","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-k3s","installer":"k3s","family":"conformance","package":"conformance","gateway":"tmachine","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"conformance","package":"conformance","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"ubuntu-docker-rootful-keycloak","installer":"deb","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"} ]' matrix=$(jq -c --arg family "${FAMILY}" \ diff --git a/CI.md b/CI.md index 04df595f3b..6fe38b9a8b 100644 --- a/CI.md +++ b/CI.md @@ -204,7 +204,7 @@ JUnit report, nextest log and, after a failure, the guest's diagnostics as an ### Run a subset of the conformance tests -The conformance lanes run the `sandbox` package of `tests/suites/conformance` +The conformance lanes run the `conformance` crate in `tests/suites/conformance` through `.github/workflows/e2e-suites.yml`, which keeps the list of environments. To run one suite in one environment from a checkout, build the candidate inputs (see [TESTING.md](TESTING.md)) and narrow the tests with a @@ -213,8 +213,8 @@ nextest filterset: ```shell nix run .#openshell-test-runner -- \ --environment ubuntu-docker-rootful --installer deb \ - --family conformance --package sandbox \ - --filter 'binary_id(=sandbox::policy_advisor)' + --family conformance --package conformance \ + --filter 'binary_id(=conformance::policy_advisor)' ``` A local run does not replace the required PR E2E gate. diff --git a/TESTING.md b/TESTING.md index a2047177e5..f914f254a3 100644 --- a/TESTING.md +++ b/TESTING.md @@ -223,7 +223,7 @@ OpenShell that the VM's installer set up. ```shell nix run .#openshell-test-runner -- \ --environment ubuntu-docker-rootful --installer deb \ - --family conformance --package sandbox + --family conformance --package conformance ``` It needs KVM, and tmachine only boots guests of the host's architecture. The @@ -412,23 +412,23 @@ HTTP `Host`, TLS SNI, and mTLS handling. Suites: - Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests. -- CLI conformance (`tests/suites/conformance`) - portable Cargo tests for - lifecycle, mechanistic drafts, file transfer, and the sandbox-local API, - including agent-authored permission requests. Driver E2E runs the complete - Cargo test package. The installed-artifact conformance lanes run the `sandbox` - package through `openshell-test-runner` (see below). +- Conformance (`tests/suites/conformance`) - the testkit `conformance` crate: + lifecycle, the CLI, persistent workspaces, file transfer, and the policy + advisor. Driver E2E runs it with nextest against the lane's gateway. The + installed-artifact conformance lanes run the same crate through + `openshell-test-runner` (see above). - Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or - `e2e-vm`) - CLI conformance plus the common and driver-specific coverage for + `e2e-vm`) - conformance plus the common and driver-specific coverage for the selected deployment. - Docker suite (`--features e2e-docker`) - includes Docker-only coverage such as Dockerfile image builds, Docker preflight checks, and managed Docker gateway start. - Docker GPU suite (`--features e2e-docker-gpu`) - Docker suite plus GPU sandbox smoke coverage. -- Configuration push suite (`--features e2e-config-push`) - CLI conformance and the policy tests against a Docker gateway with `config_delivery_mode = "push"`. Run it with `mise run e2e:rust:push`. +- Configuration push suite (`--features e2e-config-push`) - conformance and the policy tests against a Docker gateway with `config_delivery_mode = "push"`. Run it with `mise run e2e:rust:push`. - VM suite (`--features e2e-vm`) - runs e2e tests on a VM. - Kubernetes credential-driver suite (`--features e2e-kubernetes-credential-drivers`) - targeted Kubernetes Secrets and Vault provider credential storage coverage. -The Docker, Podman, Kubernetes, and VM runners execute CLI -conformance before their Rust tests. Set the driver's feature variable -(`OPENSHELL_E2E_DOCKER_FEATURES`, `OPENSHELL_E2E_PODMAN_FEATURES`, +The Docker, Podman, Kubernetes, and VM runners execute conformance before +their Rust tests; a run with no conformance tests fails. Set the driver's +feature variable (`OPENSHELL_E2E_DOCKER_FEATURES`, `OPENSHELL_E2E_PODMAN_FEATURES`, `OPENSHELL_E2E_KUBERNETES_FEATURES`, or `OPENSHELL_E2E_VM_FEATURES`) to `e2e-conformance-only` to skip the Rust tests; the runner logs a notice when it skips them. Unset or empty values select the driver's default Rust @@ -454,52 +454,27 @@ Run the Docker-backed Rust CLI e2e suite: mise run e2e:docker ``` -Run the portable CLI conformance suite against the gateway selected in your -OpenShell CLI configuration: +Run the conformance suite against the gateway selected in your OpenShell CLI +configuration: ```shell cargo build --package openshell-cli OPENSHELL_BIN="$PWD/target/debug/openshell" \ - cargo test \ + cargo nextest run \ --locked \ --manifest-path tests/suites/conformance/Cargo.toml \ - --package openshell-test-conformance-cli \ - --no-fail-fast \ - -- \ - --test-threads=1 \ - --nocapture + --test-threads 1 \ + --no-fail-fast ``` The gateway must already be installed, reachable, and selected before the tests start. The test suite does not provision a gateway or select a compute driver. Set `OPENSHELL_BIN` to another executable to test a different prebuilt CLI. +[tests/testkit/README.md](tests/testkit/README.md) covers gateway selection and +cleanup. -The phase-1 scenario verifies the complete CLI-to-gateway-to-driver path without -depending on how the gateway was installed or which driver is configured. It -requires machine-readable gRPC status, creates a uniquely named detached -sandbox with the configured default image, verifies the sandbox is `Ready` by finding its -unique name in paginated JSON list output, executes `echo` with a run-specific -marker, deletes the sandbox, and verifies that its name no longer appears. -Driver suites enable the same profile -instead of maintaining a separate smoke implementation. Sandbox lifecycle, -label matrices, VM overlay, and TLS-key permission assertions remain regular -E2E coverage. - -Each invocation prints a ten-character run ID before creating resources. -Conformance sandboxes use names such as `ct--01`. The runner tracks the -exact name and uses it for cleanup; phase 1 does not add ownership labels. - -The runner deletes owned resources after both success and failure. If the test -process is interrupted before cleanup, locate leftovers without touching -unrelated gateway state: - -```shell -openshell sandbox list --output json -openshell sandbox delete -``` - -Gateway-backed Rust E2E tasks run the CLI conformance Cargo tests against the -configured gateway, then run any lane-specific Rust tests that still apply. +Gateway-backed Rust E2E tasks run the conformance suite against the configured +gateway, then run any lane-specific Rust tests that still apply. Run the Podman-backed Rust CLI e2e suite: ```shell @@ -528,7 +503,7 @@ rebuilds only the RPMs. Run conformance against the installed RPMs: nix run .#build-artifacts nix run .#openshell-test-runner -- \ --environment fedora-podman-rootful --installer rpm \ - --family conformance --package sandbox + --family conformance --package conformance ``` Run the portable subset in a disposable rootless Podman guest: diff --git a/e2e/policy-advisor/README.md b/e2e/policy-advisor/README.md index 73bc5f67b5..c5c4e79ca0 100644 --- a/e2e/policy-advisor/README.md +++ b/e2e/policy-advisor/README.md @@ -62,14 +62,14 @@ requests through `policy.local`. See [TESTING.md](../../TESTING.md#rust-cli-e2e) for the supported source and installed-artifact test paths. To run only the policy-related conformance tests against an installed -candidate, run the `policy-advisor` suite of the `sandbox` package through the +candidate, run the `policy-advisor` suite of the `conformance` crate through the test runner: ```bash nix run .#openshell-test-runner -- \ --environment ubuntu-docker-rootful --installer deb \ - --family conformance --package sandbox \ - --filter 'binary_id(=sandbox::policy_advisor)' + --family conformance --package conformance \ + --filter 'binary_id(=conformance::policy_advisor)' ``` The GitHub write test above and the regressions below still exercise distinct diff --git a/e2e/support/conformance.sh b/e2e/support/conformance.sh index 9fc898b057..26cb3ed9f6 100644 --- a/e2e/support/conformance.sh +++ b/e2e/support/conformance.sh @@ -30,15 +30,16 @@ e2e_run_openshell_conformance() { e2e_require_openshell_bin || return + # The tests find the gateway as the CLI does: OPENSHELL_GATEWAY_ENDPOINT, + # else OPENSHELL_GATEWAY, else the active gateway. The gateway wrappers set + # these for the gateway they started. echo "==> Running conformance suites against the ${gateway_label} gateway" - cargo test \ + cargo nextest run \ --locked \ --manifest-path "${root}/tests/suites/conformance/Cargo.toml" \ - --package openshell-test-conformance-cli \ + --test-threads 1 \ --no-fail-fast \ - -- \ - --test-threads=1 \ - --nocapture + --no-tests fail } if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then diff --git a/tasks/rust.toml b/tasks/rust.toml index 1aa0cf0935..e8d1c69f4f 100644 --- a/tasks/rust.toml +++ b/tasks/rust.toml @@ -39,7 +39,7 @@ run = [ "cargo clippy --manifest-path tests/testkit/Cargo.toml --all-targets -- -D warnings", "cargo clippy --manifest-path tests/tmachine/Cargo.toml --all-targets -- -D warnings", "cargo clippy --manifest-path tests/runner/Cargo.toml --all-targets -- -D warnings", - "cargo clippy --manifest-path tests/suites/conformance/Cargo.toml -p sandbox --all-targets -- -D warnings", + "cargo clippy --manifest-path tests/suites/conformance/Cargo.toml -p conformance --all-targets -- -D warnings", "cargo clippy --manifest-path tests/suites/features/Cargo.toml -p keycloak-provider-refresh --all-targets -- -D warnings", "cargo clippy --manifest-path tests/support/Cargo.toml --all-targets -- -D warnings", ] @@ -57,7 +57,7 @@ run = [ "cargo fmt --manifest-path tests/testkit/Cargo.toml --all", "cargo fmt --manifest-path tests/tmachine/Cargo.toml --all", "cargo fmt --manifest-path tests/runner/Cargo.toml --all", - "cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox", + "cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p conformance", "cargo fmt --manifest-path tests/suites/features/Cargo.toml -p keycloak-provider-refresh", "cargo fmt --manifest-path tests/support/Cargo.toml --all", ] @@ -74,7 +74,7 @@ run = [ "cargo fmt --manifest-path tests/testkit/Cargo.toml --all -- --check", "cargo fmt --manifest-path tests/tmachine/Cargo.toml --all -- --check", "cargo fmt --manifest-path tests/runner/Cargo.toml --all -- --check", - "cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox -- --check", + "cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p conformance -- --check", "cargo fmt --manifest-path tests/suites/features/Cargo.toml -p keycloak-provider-refresh -- --check", "cargo fmt --manifest-path tests/support/Cargo.toml --all -- --check", ] diff --git a/tasks/test.toml b/tasks/test.toml index 6541b1c11f..558d7a2401 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -169,13 +169,12 @@ run = "OPENSHELL_TEST_CONTAINER_ENGINE=podman e2e/with-podman-gateway.sh cargo t ["e2e:testkit"] description = "Run the testkit suites in tests/suites/conformance against a Docker-backed gateway" -run = "e2e/with-docker-gateway.sh cargo nextest run --locked --manifest-path tests/suites/conformance/Cargo.toml --workspace --exclude openshell-test-conformance-cli --test-threads 1 --no-fail-fast" +run = "e2e/with-docker-gateway.sh cargo nextest run --locked --manifest-path tests/suites/conformance/Cargo.toml --workspace --test-threads 1 --no-fail-fast" ["e2e:rust:push"] -description = "Run CLI conformance and policy e2e tests against a Docker gateway that pushes configuration" -depends = ["e2e:conformance:build"] +description = "Run conformance and policy e2e tests against a Docker gateway that pushes configuration" env = { OPENSHELL_CONFIG_DELIVERY_MODE = "push", OPENSHELL_E2E_DOCKER_FEATURES = "e2e-config-push", OPENSHELL_E2E_DOCKER_TEST = "config_push live_policy_update policy_activation" } -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-docker.sh" +run = "e2e/rust/e2e-docker.sh" ["e2e:workload:build"] description = "Build the Docker E2E workload fixture" diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index d8b1b77bea..f45f1866ee 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -406,6 +406,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "conformance" +version = "0.0.0" +dependencies = [ + "openshell-testkit", + "serde_json", + "tempfile", + "tokio", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -1879,17 +1889,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "openshell-conformance" -version = "0.0.0" -dependencies = [ - "rand 0.9.4", - "serde", - "serde_json", - "tempfile", - "tokio", -] - [[package]] name = "openshell-core" version = "0.0.0" @@ -1992,14 +1991,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "openshell-test-conformance-cli" -version = "0.0.0" -dependencies = [ - "openshell-conformance", - "tokio", -] - [[package]] name = "openshell-testkit" version = "0.0.0" @@ -2662,16 +2653,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "sandbox" -version = "0.0.0" -dependencies = [ - "openshell-testkit", - "serde_json", - "tempfile", - "tokio", -] - [[package]] name = "schannel" version = "0.1.29" diff --git a/tests/suites/conformance/Cargo.toml b/tests/suites/conformance/Cargo.toml index 8dad4ea1ff..40541d88c4 100644 --- a/tests/suites/conformance/Cargo.toml +++ b/tests/suites/conformance/Cargo.toml @@ -1,6 +1,21 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# The conformance family is one crate; each file in `tests/` is a suite. +[package] +name = "conformance" +description = "Conformance suite: sandbox behavior on every compute driver" +version = "0.0.0" +edition = "2024" +rust-version = "1.94" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA/OpenShell" +publish = false + +[dependencies] +openshell-testkit = { path = "../../testkit/testkit" } +serde_json = "1" +tempfile = "3" +tokio = { version = "1", features = ["time"] } + [workspace] -resolver = "2" -members = ["cli", "sandbox"] diff --git a/tests/suites/conformance/sandbox/README.md b/tests/suites/conformance/README.md similarity index 86% rename from tests/suites/conformance/sandbox/README.md rename to tests/suites/conformance/README.md index ee0a557e73..ee0502d072 100644 --- a/tests/suites/conformance/sandbox/README.md +++ b/tests/suites/conformance/README.md @@ -1,8 +1,8 @@ -# Sandbox conformance suite +# Conformance suites -The `sandbox` crate holds testkit suites for sandbox behavior that every compute -driver must provide. It is a member of the `conformance` workspace. Each file in -`tests/` is one suite. +The `conformance` crate holds the testkit suites for sandbox behavior that every +compute driver must provide. The family is this one crate, its own workspace. +Each file in `tests/` is one suite. | Suite | Covers | |---|---| @@ -23,14 +23,14 @@ environments). ## Running it -[TESTING.md](../../../../TESTING.md) covers running it against a local Docker +[TESTING.md](../../../TESTING.md) covers running it against a local Docker gateway and in a tmachine VM, as CI does. ## Adding a test Add a `#[testkit::test]` function to a suite, or a new file in `tests/` for a new area. No playbook, test archive or workflow entry is needed: the runner builds -and runs every test in the package. +and runs every test in the crate. ## What a test may assert diff --git a/tests/suites/conformance/cli/Cargo.toml b/tests/suites/conformance/cli/Cargo.toml deleted file mode 100644 index 5edcefe687..0000000000 --- a/tests/suites/conformance/cli/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -[package] -name = "openshell-test-conformance-cli" -version = "0.0.0" -edition = "2024" -rust-version = "1.94" -license = "Apache-2.0" -repository = "https://github.com/NVIDIA/OpenShell" - -[dependencies] -openshell-conformance = { path = "../../../../crates/openshell-conformance" } -tokio = { version = "1.43", features = ["macros", "rt"] } diff --git a/tests/suites/conformance/cli/tests/file_transfer.rs b/tests/suites/conformance/cli/tests/file_transfer.rs deleted file mode 100644 index 167506b87b..0000000000 --- a/tests/suites/conformance/cli/tests/file_transfer.rs +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Driver-agnostic sandbox file-transfer conformance tests. - -use openshell_conformance::{ - FILE_TRANSFER_GIT_FILTERING_SCENARIO, FILE_TRANSFER_PATH_SAFETY_SCENARIO, - FILE_TRANSFER_ROUND_TRIP_SCENARIO, OpenShellRunner, Scenario, -}; - -/// Exercise file and directory round trips through the candidate CLI. -#[tokio::test] -async fn round_trip() { - run(FILE_TRANSFER_ROUND_TRIP_SCENARIO).await; -} - -/// Exercise Git-aware upload filtering through the candidate CLI. -#[tokio::test] -async fn git_filtering() { - run(FILE_TRANSFER_GIT_FILTERING_SCENARIO).await; -} - -/// Exercise workspace boundary and filename safety through the candidate CLI. -#[tokio::test] -async fn path_safety() { - run(FILE_TRANSFER_PATH_SAFETY_SCENARIO).await; -} - -async fn run(scenario: Scenario) { - let mut runner = OpenShellRunner::from_env(scenario.name) - .expect("candidate openshell CLI is available"); - let result = async { - runner.check_gateway_status().await?; - scenario.run(&mut runner).await - } - .await; - if let Err(error) = runner.finish(result).await { - panic!("{} conformance scenario failed:\n{error}", scenario.name); - } -} diff --git a/tests/suites/conformance/cli/tests/lifecycle.rs b/tests/suites/conformance/cli/tests/lifecycle.rs deleted file mode 100644 index 1a18adef86..0000000000 --- a/tests/suites/conformance/cli/tests/lifecycle.rs +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Driver-agnostic sandbox lifecycle conformance tests. - -use openshell_conformance::{OpenShellRunner, SANDBOX_LIFECYCLE_SCENARIO}; - -/// Exercise stop, start, and stopped-deletion behavior through the candidate CLI. -#[tokio::test] -async fn sandbox_lifecycle() { - let mut runner = OpenShellRunner::from_env(SANDBOX_LIFECYCLE_SCENARIO.name) - .expect("candidate openshell CLI is available"); - - let result = async { - runner.check_gateway_status().await?; - SANDBOX_LIFECYCLE_SCENARIO.run(&mut runner).await - } - .await; - if let Err(error) = runner.finish(result).await { - panic!("sandbox lifecycle conformance scenario failed:\n{error}"); - } -} diff --git a/tests/suites/conformance/cli/tests/policy_advisor.rs b/tests/suites/conformance/cli/tests/policy_advisor.rs deleted file mode 100644 index 7da2186dd7..0000000000 --- a/tests/suites/conformance/cli/tests/policy_advisor.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Installed-artifact policy advisor conformance. - -use openshell_conformance::{ - MECHANISTIC_PROPOSAL_SCENARIO, NEW_HOSTNAME_PROPOSAL_SCENARIO, OpenShellRunner, - POLICY_LOCAL_SCENARIO, Scenario, -}; - -async fn run(scenario: &'static Scenario) { - let mut runner = - OpenShellRunner::from_env(scenario.name).expect("candidate openshell CLI is available"); - let result = async { - runner.check_gateway_status().await?; - scenario.run(&mut runner).await - } - .await; - if let Err(error) = runner.finish(result).await { - panic!("{} conformance scenario failed:\n{error}", scenario.name); - } -} - -#[tokio::test] -async fn mechanistic_proposal() { - run(&MECHANISTIC_PROPOSAL_SCENARIO).await; -} - -#[tokio::test] -async fn new_hostname_proposal() { - run(&NEW_HOSTNAME_PROPOSAL_SCENARIO).await; -} - -#[tokio::test] -async fn policy_local() { - run(&POLICY_LOCAL_SCENARIO).await; -} diff --git a/tests/suites/conformance/cli/tests/smoke.rs b/tests/suites/conformance/cli/tests/smoke.rs deleted file mode 100644 index 55306e852f..0000000000 --- a/tests/suites/conformance/cli/tests/smoke.rs +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Driver-agnostic `OpenShell` CLI conformance tests. - -use openshell_conformance::{OpenShellRunner, SMOKE_SCENARIO}; - -/// Exercise the public CLI against a provisioned `OpenShell` gateway. -/// -/// The test runner supplies the candidate CLI explicitly so the same archive -/// can validate artifacts installed into any supported test guest. -#[tokio::test] -async fn smoke() { - let mut runner = OpenShellRunner::from_env(SMOKE_SCENARIO.name) - .expect("candidate openshell CLI is available"); - let result = async { - runner.check_gateway_status().await?; - SMOKE_SCENARIO.run(&mut runner).await - } - .await; - if let Err(error) = runner.finish(result).await { - panic!("conformance smoke scenario failed:\n{error}"); - } -} diff --git a/tests/suites/conformance/sandbox/Cargo.toml b/tests/suites/conformance/sandbox/Cargo.toml deleted file mode 100644 index d213c6b23a..0000000000 --- a/tests/suites/conformance/sandbox/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -[package] -name = "sandbox" -description = "Conformance suite: sandbox behavior on every compute driver" -version = "0.0.0" -edition = "2024" -rust-version = "1.94" -license = "Apache-2.0" -repository = "https://github.com/NVIDIA/OpenShell" -publish = false - -[dependencies] -openshell-testkit = { path = "../../../testkit/testkit" } -serde_json = "1" -tempfile = "3" -tokio = { version = "1", features = ["time"] } diff --git a/tests/suites/conformance/sandbox/tests/cli.rs b/tests/suites/conformance/tests/cli.rs similarity index 100% rename from tests/suites/conformance/sandbox/tests/cli.rs rename to tests/suites/conformance/tests/cli.rs diff --git a/tests/suites/conformance/sandbox/tests/file_transfer.rs b/tests/suites/conformance/tests/file_transfer.rs similarity index 100% rename from tests/suites/conformance/sandbox/tests/file_transfer.rs rename to tests/suites/conformance/tests/file_transfer.rs diff --git a/tests/suites/conformance/sandbox/tests/lifecycle.rs b/tests/suites/conformance/tests/lifecycle.rs similarity index 100% rename from tests/suites/conformance/sandbox/tests/lifecycle.rs rename to tests/suites/conformance/tests/lifecycle.rs diff --git a/tests/suites/conformance/sandbox/tests/persistent_workspace.rs b/tests/suites/conformance/tests/persistent_workspace.rs similarity index 100% rename from tests/suites/conformance/sandbox/tests/persistent_workspace.rs rename to tests/suites/conformance/tests/persistent_workspace.rs diff --git a/tests/suites/conformance/sandbox/tests/policy_advisor.rs b/tests/suites/conformance/tests/policy_advisor.rs similarity index 100% rename from tests/suites/conformance/sandbox/tests/policy_advisor.rs rename to tests/suites/conformance/tests/policy_advisor.rs diff --git a/tests/suites/conformance/sandbox/tests/smoke.rs b/tests/suites/conformance/tests/smoke.rs similarity index 100% rename from tests/suites/conformance/sandbox/tests/smoke.rs rename to tests/suites/conformance/tests/smoke.rs