diff --git a/.github/actions/setup-e2e-cli/action.yml b/.github/actions/setup-e2e-cli/action.yml index 8b30970d8d..6bc2a78ec2 100644 --- a/.github/actions/setup-e2e-cli/action.yml +++ b/.github/actions/setup-e2e-cli/action.yml @@ -1,6 +1,12 @@ name: Setup E2E CLI description: Download architecture-matched prebuilt OpenShell host CLIs for E2E tests +inputs: + conformance: + description: Also download the prebuilt openshell-conformance program and set OPENSHELL_CONFORMANCE_BIN + required: false + default: "false" + runs: using: composite steps: @@ -17,3 +23,19 @@ runs: chmod +x "$cli" "$cli" --version echo "OPENSHELL_BIN=$cli" >> "$GITHUB_ENV" + + - name: Download prebuilt openshell-conformance + if: inputs.conformance == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ runner.arch == 'X64' && 'openshell-conformance-x86_64-unknown-linux-musl' || 'openshell-conformance-aarch64-unknown-linux-musl' }} + path: .e2e/prebuilt-conformance + + - name: Configure prebuilt openshell-conformance + if: inputs.conformance == 'true' + shell: bash + run: | # zizmor: ignore[github-env] persists only a fixed path under the trusted workspace + program="$GITHUB_WORKSPACE/.e2e/prebuilt-conformance/openshell-conformance" + chmod +x "$program" + "$program" list + echo "OPENSHELL_CONFORMANCE_BIN=$program" >> "$GITHUB_ENV" diff --git a/.github/workflows/branch-checks.yml b/.github/workflows/branch-checks.yml index a62d442bc9..7af659acd2 100644 --- a/.github/workflows/branch-checks.yml +++ b/.github/workflows/branch-checks.yml @@ -389,6 +389,7 @@ jobs: cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/testkit/Cargo.toml cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/tmachine/Cargo.toml cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/runner/Cargo.toml + tasks/scripts/check-conformance-program.sh rust-postgres-test: name: Rust PostgreSQL tests diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 3647f3ee02..18843fcaa5 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -144,6 +144,60 @@ jobs: cargo-version: ${{ needs.version.outputs.cargo }} extra-cargo-flags: --no-default-features + # The conformance program, built once per architecture for the host lanes. + # It lives in its own workspace and is not released, so it is not part of + # build-binaries.yml. + build-conformance: + name: openshell-conformance (${{ matrix.triple }}) + needs: [pr_metadata] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_any_e2e == 'true' + permissions: + contents: read + strategy: + matrix: + include: + - triple: x86_64-unknown-linux-musl + runner: linux-amd64-cpu8 + - triple: aarch64-unknown-linux-musl + runner: linux-arm64-cpu8 + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.sha }} + - uses: ./.github/actions/setup-nix + with: + prepare-shell: "true" + cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} + - uses: ./.github/actions/setup-rust + with: + cache-key: binaries-v2-openshell-conformance-${{ matrix.triple }} + save-cache: ${{ github.ref == 'refs/heads/main' }} + - name: Build openshell-conformance + shell: nix develop -c bash -euo pipefail {0} + env: + # Build into the root target directory, which setup-rust caches. + CARGO_TARGET_DIR: ${{ github.workspace }}/target + TRIPLE: ${{ matrix.triple }} + run: | + cargo build --release --locked --target "$TRIPLE" \ + --manifest-path tests/suites/conformance/Cargo.toml \ + --bin openshell-conformance + binary="target/$TRIPLE/release/openshell-conformance" + # The program must run on any lane host. + tasks/scripts/verify-static-binary.sh "$binary" + # Fails if the program has no tests linked. + "$binary" list + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: openshell-conformance-${{ matrix.triple }} + path: target/${{ matrix.triple }}/release/openshell-conformance + compression-level: 0 + retention-days: 5 + if-no-files-found: error + build-external-drivers: name: ${{ matrix.binary }} (${{ matrix.triple }}) needs: [pr_metadata, version] @@ -316,7 +370,7 @@ jobs: CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} docker-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -324,11 +378,12 @@ jobs: packages: read uses: ./.github/workflows/e2e-docker-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} runner: linux-arm64-cpu8 podman-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -336,12 +391,13 @@ jobs: packages: read uses: ./.github/workflows/e2e-podman-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} suite-matrix: >- [{"suite":"ci","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:ci"}] podman-additional-ca-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -349,21 +405,24 @@ jobs: packages: read uses: ./.github/workflows/e2e-podman-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} suite-matrix: >- [{"suite":"additional-ca","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"OPENSHELL_E2E_PODMAN_TEST=additional_ca mise run --no-deps --skip-deps e2e:podman"}] vm-e2e: - needs: [pr_metadata, build-binaries, build-vm-driver] + needs: [pr_metadata, build-binaries, build-conformance, build-vm-driver] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read contents: read packages: read uses: ./.github/workflows/e2e-vm-test.yml + with: + prebuilt-conformance: true vm-additional-ca-e2e: - needs: [pr_metadata, build-binaries, build-vm-driver] + needs: [pr_metadata, build-binaries, build-conformance, build-vm-driver] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -371,11 +430,12 @@ jobs: packages: read uses: ./.github/workflows/e2e-vm-test.yml with: + prebuilt-conformance: true suite-name: additional-ca test-name: additional_ca docker-external-driver-e2e: - needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-gateway-plain, build-external-drivers, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -383,6 +443,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-docker-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} runner: linux-arm64-cpu8 gateway-artifact: openshell-gateway-plain-aarch64-unknown-linux-gnu @@ -391,7 +452,7 @@ jobs: [{"suite":"external-driver","cmd":"mise run --no-deps --skip-deps e2e:docker:external-driver","apt_packages":"openssh-client","python_proto":false,"mcp":false}] podman-external-driver-e2e: - needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-gateway-plain, build-external-drivers, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -399,6 +460,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-podman-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu external-driver-binary: openshell-driver-podman @@ -406,7 +468,7 @@ jobs: [{"suite":"external-driver","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:external-driver"}] vm-external-driver-e2e: - needs: [pr_metadata, build-binaries, build-gateway-plain, build-vm-driver] + needs: [pr_metadata, build-binaries, build-conformance, build-gateway-plain, build-vm-driver] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -414,6 +476,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-vm-test.yml with: + prebuilt-conformance: true gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu suite-name: external-driver e2e-task: e2e:vm:external-driver @@ -430,7 +493,7 @@ jobs: image-tag: ${{ github.sha }} kubernetes-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' strategy: fail-fast: false @@ -454,13 +517,14 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: ${{ matrix.job_name }} agent-sandbox-version: ${{ matrix.agent_sandbox_version }} test-name: ${{ matrix.test_name }} kubernetes-app-listener-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' strategy: fail-fast: false @@ -488,6 +552,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: Kubernetes app listener (${{ matrix.name }}) e2e-task: e2e:kubernetes:app-listener @@ -497,7 +562,7 @@ jobs: listener-takeover: ${{ matrix.takeover }} kubernetes-workspace-managed-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -505,12 +570,13 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: Kubernetes E2E (workspace managed mode) e2e-task: e2e:kubernetes:workspace-managed kubernetes-external-driver-e2e: - needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-gateway-plain, build-external-drivers, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -518,6 +584,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: Kubernetes E2E (external compute driver) e2e-task: e2e:kubernetes:external-driver @@ -526,7 +593,7 @@ jobs: cluster-images: sandbox supervisor kubernetes-workspace-operator-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -534,12 +601,13 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: Kubernetes E2E (workspace operator mode) e2e-task: e2e:kubernetes:workspace-operator kubernetes-ha-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true' permissions: actions: read @@ -547,6 +615,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: Kubernetes HA E2E (Rust smoke) extra-helm-values: deploy/helm/openshell/ci/values-high-availability.yaml @@ -556,7 +625,7 @@ jobs: use-envoy-gateway: true kubernetes-credential-drivers-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true' permissions: actions: read @@ -564,6 +633,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: Kubernetes Credential Drivers E2E e2e-task: e2e:kubernetes:credential-drivers diff --git a/.github/workflows/e2e-docker-test.yml b/.github/workflows/e2e-docker-test.yml index 501ded8d34..faa0121cd2 100644 --- a/.github/workflows/e2e-docker-test.yml +++ b/.github/workflows/e2e-docker-test.yml @@ -21,6 +21,10 @@ on: required: false type: string default: "" + prebuilt-conformance: + required: false + type: boolean + default: false external-driver-binary: required: false type: string @@ -67,6 +71,8 @@ jobs: persist-credentials: false - uses: ./.github/actions/setup-e2e-cli + with: + conformance: ${{ inputs.prebuilt-conformance }} - uses: ./.github/actions/setup-e2e-gateway with: diff --git a/.github/workflows/e2e-kubernetes-test.yml b/.github/workflows/e2e-kubernetes-test.yml index 867b8c1dd7..1ae616e36e 100644 --- a/.github/workflows/e2e-kubernetes-test.yml +++ b/.github/workflows/e2e-kubernetes-test.yml @@ -67,6 +67,11 @@ on: required: false type: string default: "" + prebuilt-conformance: + description: "Run conformance with the openshell-conformance artifact instead of building it" + required: false + type: boolean + default: false external-driver-binary: description: "Optional standalone compute driver binary used to compose a local test image" required: false @@ -124,6 +129,8 @@ jobs: - name: Use prebuilt OpenShell CLI uses: ./.github/actions/setup-e2e-cli + with: + conformance: ${{ inputs.prebuilt-conformance }} - name: Use prebuilt OpenShell gateway if: inputs.gateway-artifact != '' diff --git a/.github/workflows/e2e-podman-test.yml b/.github/workflows/e2e-podman-test.yml index d70ad6c60c..ac6d7d66ef 100644 --- a/.github/workflows/e2e-podman-test.yml +++ b/.github/workflows/e2e-podman-test.yml @@ -17,6 +17,10 @@ on: required: false type: string default: "" + prebuilt-conformance: + required: false + type: boolean + default: false external-driver-binary: required: false type: string @@ -52,6 +56,8 @@ jobs: persist-credentials: false - uses: ./.github/actions/setup-e2e-cli + with: + conformance: ${{ inputs.prebuilt-conformance }} - uses: ./.github/actions/setup-e2e-gateway with: diff --git a/.github/workflows/e2e-vm-test.yml b/.github/workflows/e2e-vm-test.yml index 28b4c81db5..82d86058b8 100644 --- a/.github/workflows/e2e-vm-test.yml +++ b/.github/workflows/e2e-vm-test.yml @@ -14,6 +14,10 @@ on: required: false type: string default: "" + prebuilt-conformance: + required: false + type: boolean + default: false suite-name: required: false type: string @@ -47,6 +51,8 @@ jobs: persist-credentials: false - uses: ./.github/actions/setup-e2e-cli + with: + conformance: ${{ inputs.prebuilt-conformance }} - uses: ./.github/actions/setup-e2e-gateway with: diff --git a/CI.md b/CI.md index 6fe38b9a8b..db4a8e7995 100644 --- a/CI.md +++ b/CI.md @@ -214,7 +214,7 @@ nextest filterset: nix run .#openshell-test-runner -- \ --environment ubuntu-docker-rootful --installer deb \ --family conformance --package conformance \ - --filter 'binary_id(=conformance::policy_advisor)' + --filter 'test(/^policy_advisor::/)' ``` A local run does not replace the required PR E2E gate. diff --git a/TESTING.md b/TESTING.md index f914f254a3..287078a900 100644 --- a/TESTING.md +++ b/TESTING.md @@ -181,6 +181,12 @@ by `OPENSHELL_BIN`. See [tests/testkit/README.md](tests/testkit/README.md) for how to write and run tests. +The conformance crate also builds `openshell-conformance`, a program that +runs the conformance suite against a gateway without cargo or nextest +(`openshell-conformance list`, `ready`, `run`). It covers only the +conformance suite. See +[tests/suites/conformance/README.md](tests/suites/conformance/README.md). + Suites live under `tests/suites//`. Each family is a Cargo workspace with its own lockfile. Run the `conformance` suites against a local Docker gateway with: @@ -414,7 +420,10 @@ Suites: - Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests. - Conformance (`tests/suites/conformance`) - the testkit `conformance` crate: lifecycle, the CLI, persistent workspaces, file transfer, and the policy - advisor. Driver E2E runs it with nextest against the lane's gateway. The + advisor. Driver E2E runs it with `openshell-conformance` against the lane's + gateway and writes a JUnit report under `results/`; set + `OPENSHELL_CONFORMANCE_BIN` to use a prebuilt program, as Branch E2E does + with the static build it makes once per architecture. The installed-artifact conformance lanes run the same crate through `openshell-test-runner` (see above). - Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or diff --git a/e2e/policy-advisor/README.md b/e2e/policy-advisor/README.md index c5c4e79ca0..adb700306f 100644 --- a/e2e/policy-advisor/README.md +++ b/e2e/policy-advisor/README.md @@ -69,7 +69,7 @@ test runner: nix run .#openshell-test-runner -- \ --environment ubuntu-docker-rootful --installer deb \ --family conformance --package conformance \ - --filter 'binary_id(=conformance::policy_advisor)' + --filter 'test(/^policy_advisor::/)' ``` The GitHub write test above and the regressions below still exercise distinct diff --git a/e2e/rust/e2e-kubernetes.sh b/e2e/rust/e2e-kubernetes.sh index 9bdcf4e6a2..fc90a827ee 100755 --- a/e2e/rust/e2e-kubernetes.sh +++ b/e2e/rust/e2e-kubernetes.sh @@ -79,7 +79,8 @@ run_conformance() { return 0 fi - e2e_run_openshell_conformance "Kubernetes" + OPENSHELL_CONFORMANCE_JUNIT="${OPENSHELL_CONFORMANCE_JUNIT:-${ROOT}/results/${OPENSHELL_E2E_REPORT_NAME:-e2e-kubernetes}-conformance.xml}" \ + e2e_run_openshell_conformance "Kubernetes" } # `OPENSHELL_E2E_REPORT_NAME` (default `e2e-kubernetes`) names the report diff --git a/e2e/support/conformance.sh b/e2e/support/conformance.sh index 26cb3ed9f6..82594439c0 100644 --- a/e2e/support/conformance.sh +++ b/e2e/support/conformance.sh @@ -16,30 +16,55 @@ e2e_require_openshell_bin() { return 2 fi - # Cargo runs each test from its package directory, so resolve the candidate - # relative to the caller's working directory before passing it to the tests. + # Tests may run from another directory, so resolve the candidate relative + # to the caller's working directory before passing it to them. local binary_dir binary_dir="$(cd "$(dirname "${OPENSHELL_BIN}")" && pwd -P)" || return export OPENSHELL_BIN="${binary_dir}/$(basename "${OPENSHELL_BIN}")" } +# Print the path of the openshell-conformance program: OPENSHELL_CONFORMANCE_BIN +# if set, else a debug build from this checkout. +e2e_openshell_conformance_bin() { + local root=$1 + + if [ -n "${OPENSHELL_CONFORMANCE_BIN:-}" ]; then + if [ ! -x "${OPENSHELL_CONFORMANCE_BIN}" ]; then + echo "ERROR: openshell-conformance is not executable: ${OPENSHELL_CONFORMANCE_BIN}" >&2 + return 2 + fi + echo "${OPENSHELL_CONFORMANCE_BIN}" + return + fi + + cargo build \ + --locked \ + --manifest-path "${root}/tests/suites/conformance/Cargo.toml" \ + --bin openshell-conformance >&2 || return + # Cargo resolves a relative CARGO_TARGET_DIR against the working directory. + local target_dir + target_dir="$(cd "${CARGO_TARGET_DIR:-${root}/tests/suites/conformance/target}" && pwd)" || return + echo "${target_dir}/debug/openshell-conformance" +} + +# Run the conformance suite with openshell-conformance. The JUnit report goes +# to OPENSHELL_CONFORMANCE_JUNIT, else results/conformance-.xml. e2e_run_openshell_conformance() { local gateway_label=${1:-OpenShell} - local root + local root program junit root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" e2e_require_openshell_bin || return + program="$(e2e_openshell_conformance_bin "${root}")" || return + junit="${OPENSHELL_CONFORMANCE_JUNIT:-${root}/results/conformance-$(echo "${gateway_label}" | tr '[:upper:]' '[:lower:]').xml}" + mkdir -p "$(dirname "${junit}")" || return - # The tests find the gateway as the CLI does: OPENSHELL_GATEWAY_ENDPOINT, + # The program finds the gateway as the CLI does: OPENSHELL_GATEWAY_ENDPOINT, # else OPENSHELL_GATEWAY, else the active gateway. The gateway wrappers set # these for the gateway they started. echo "==> Running conformance suites against the ${gateway_label} gateway" - cargo nextest run \ - --locked \ - --manifest-path "${root}/tests/suites/conformance/Cargo.toml" \ - --test-threads 1 \ - --no-fail-fast \ - --no-tests fail + "${program}" ready || return + "${program}" run --junit "${junit}" } if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then diff --git a/tasks/scripts/check-conformance-program.sh b/tasks/scripts/check-conformance-program.sh new file mode 100755 index 0000000000..3284f20418 --- /dev/null +++ b/tasks/scripts/check-conformance-program.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Check that `openshell-conformance list` names the same tests that nextest +# lists for the conformance crate, so the program runs the whole suite. +# Neither command needs a gateway. + +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +manifest="${root}/tests/suites/conformance/Cargo.toml" +work="$(mktemp -d)" +trap 'rm -rf "${work}"' EXIT + +cargo run --quiet --locked --manifest-path "${manifest}" --bin openshell-conformance -- list \ + | awk '{print $1}' | sort > "${work}/program" +cargo nextest list --locked --color never --manifest-path "${manifest}" --message-format oneline \ + | awk '{print $2}' | sort > "${work}/nextest" + +if ! diff -u "${work}/nextest" "${work}/program"; then + echo "error: openshell-conformance list differs from nextest's list (-nextest +program)" >&2 + exit 1 +fi +if [[ ! -s "${work}/program" ]]; then + echo "error: openshell-conformance lists no tests" >&2 + exit 1 +fi +echo "openshell-conformance lists the same $(wc -l < "${work}/program" | tr -d ' ') tests as nextest" diff --git a/tasks/test.toml b/tasks/test.toml index 558d7a2401..81b0585ace 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -127,6 +127,7 @@ run = [ "cargo nextest run --config-file .config/nextest.toml --manifest-path tests/testkit/Cargo.toml", "cargo nextest run --config-file .config/nextest.toml --manifest-path tests/tmachine/Cargo.toml", "cargo nextest run --config-file .config/nextest.toml --manifest-path tests/runner/Cargo.toml", + "tasks/scripts/check-conformance-program.sh", ] run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 test-precommit native" hide = true diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index dfe8958ada..59a8dd7a83 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -37,7 +37,7 @@ struct Cli { #[arg(long = "package", short = 'p', value_name = "PACKAGE")] packages: Vec, /// A nextest filterset narrowing the tests that run, e.g. - /// `binary(smoke)`. + /// `test(/^smoke::/)`. #[arg(long, value_name = "FILTERSET")] filter: Option, /// The gateway name the installer registers with the CLI. The packaged diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index f45f1866ee..ea04cd150d 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -54,6 +54,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -382,6 +432,46 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmake" version = "0.1.58" @@ -397,6 +487,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -679,6 +775,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + [[package]] name = "etcetera" version = "0.11.0" @@ -1395,6 +1497,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.14.0" @@ -1641,6 +1749,38 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1758,6 +1898,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "newtype-uuid" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa56da58097c7ae893c4097bb6d91bb5f7fa330053e07e033349db4d5f83ba8" +dependencies = [ + "uuid", +] + [[package]] name = "nix" version = "0.29.0" @@ -1873,6 +2022,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openshell-bootstrap" version = "0.0.0" @@ -1995,11 +2150,16 @@ dependencies = [ name = "openshell-testkit" version = "0.0.0" dependencies = [ + "clap", "futures", + "libc", + "libtest-mimic", + "linkme", "openshell-bootstrap", "openshell-core", "openshell-sdk", "openshell-testkit-macros", + "quick-junit", "rand 0.9.4", "serde", "serde_json", @@ -2357,6 +2517,30 @@ dependencies = [ "pulldown-cmark", ] +[[package]] +name = "quick-junit" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae23b8e1a59a90f7bf98caa2f4c8c2eedf6df011c6c88ed633e85a87c84526e" +dependencies = [ + "chrono", + "indexmap", + "newtype-uuid", + "quick-xml", + "strip-ansi-escapes", + "thiserror 2.0.20", + "uuid", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.45" @@ -3134,6 +3318,15 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3720,6 +3913,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.23.1" @@ -3743,6 +3942,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "want" version = "0.3.1" diff --git a/tests/suites/conformance/Cargo.toml b/tests/suites/conformance/Cargo.toml index 40541d88c4..99274d8dfa 100644 --- a/tests/suites/conformance/Cargo.toml +++ b/tests/suites/conformance/Cargo.toml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# The conformance family is one crate; each file in `tests/` is a suite. +# The conformance family is one crate; each module in `src/` is a suite. [package] name = "conformance" description = "Conformance suite: sandbox behavior on every compute driver" @@ -12,6 +12,14 @@ license = "Apache-2.0" repository = "https://github.com/NVIDIA/OpenShell" publish = false +# `openshell-conformance`: the suite as one program to run against a gateway. +# No test target of its own: cargo test and nextest run the tests through the +# library. +[[bin]] +name = "openshell-conformance" +path = "src/main.rs" +test = false + [dependencies] openshell-testkit = { path = "../../testkit/testkit" } serde_json = "1" diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index ee0502d072..7592e776e5 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -2,7 +2,9 @@ The `conformance` crate holds the testkit suites for sandbox behavior that every compute driver must provide. The family is this one crate, its own workspace. -Each file in `tests/` is one suite. +It is a library: each module in `src/` is one suite, and `cargo test` and +nextest run the tests as the library's unit tests. The crate also builds the +`openshell-conformance` program, which runs the same tests. | Suite | Covers | |---|---| @@ -26,11 +28,31 @@ environments). [TESTING.md](../../../TESTING.md) covers running it against a local Docker gateway and in a tmachine VM, as CI does. +### The `openshell-conformance` program + +`openshell-conformance` runs the suite against a gateway without cargo or +nextest, so a driver author or operator can check their own gateway. It finds +the gateway and the CLI as the tests do. Build it and run it with: + +```shell +cargo build --locked --manifest-path tests/suites/conformance/Cargo.toml --bin openshell-conformance +tests/suites/conformance/target/debug/openshell-conformance list +tests/suites/conformance/target/debug/openshell-conformance ready +tests/suites/conformance/target/debug/openshell-conformance run --junit conformance.xml +``` + +The program runs only this suite: passing it means a gateway meets the +conformance contract below. Feature and driver suites are not part of it. The +[testkit README](../../testkit/README.md#test-programs) describes the commands +and exit codes. `tasks/scripts/check-conformance-program.sh` checks that it +lists the same tests as nextest. + ## Adding a test -Add a `#[testkit::test]` function to a suite, or a new file in `tests/` for a new -area. No playbook, test archive or workflow entry is needed: the runner builds -and runs every test in the crate. +Add a `#[testkit::test]` function to a suite, or for a new area, a new file in +`src/` and a `mod` line for it in `src/lib.rs`. No playbook, test archive or +workflow entry is needed: the runner builds and runs every test in the +crate, and `openshell-conformance` includes it. ## What a test may assert diff --git a/tests/suites/conformance/tests/cli.rs b/tests/suites/conformance/src/cli.rs similarity index 100% rename from tests/suites/conformance/tests/cli.rs rename to tests/suites/conformance/src/cli.rs diff --git a/tests/suites/conformance/tests/file_transfer.rs b/tests/suites/conformance/src/file_transfer.rs similarity index 100% rename from tests/suites/conformance/tests/file_transfer.rs rename to tests/suites/conformance/src/file_transfer.rs diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs new file mode 100644 index 0000000000..00a4d45027 --- /dev/null +++ b/tests/suites/conformance/src/lib.rs @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Conformance suite: sandbox behavior that every compute driver must +//! provide. Each module is one suite. +//! +//! `cargo test` and nextest run the tests as this library's unit tests. +//! `#[testkit::test]` also registers them, so the `openshell-conformance` +//! program in `src/main.rs` runs them by linking this library. + +mod cli; +mod file_transfer; +mod lifecycle; +mod persistent_workspace; +mod policy_advisor; +mod smoke; diff --git a/tests/suites/conformance/tests/lifecycle.rs b/tests/suites/conformance/src/lifecycle.rs similarity index 100% rename from tests/suites/conformance/tests/lifecycle.rs rename to tests/suites/conformance/src/lifecycle.rs diff --git a/tests/suites/conformance/src/main.rs b/tests/suites/conformance/src/main.rs new file mode 100644 index 0000000000..7b4e899c60 --- /dev/null +++ b/tests/suites/conformance/src/main.rs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `openshell-conformance`: the conformance suite as one program, to run +//! against a gateway without cargo or nextest. +//! +//! The tests are registered by `#[testkit::test]` in this crate's library; +//! see `testkit::program` for the commands. + +use std::process::ExitCode; + +// A dependency that is never named is not linked, and its tests would be +// missing. +use conformance as _; + +fn main() -> ExitCode { + testkit::main() +} diff --git a/tests/suites/conformance/tests/persistent_workspace.rs b/tests/suites/conformance/src/persistent_workspace.rs similarity index 100% rename from tests/suites/conformance/tests/persistent_workspace.rs rename to tests/suites/conformance/src/persistent_workspace.rs diff --git a/tests/suites/conformance/tests/policy_advisor.rs b/tests/suites/conformance/src/policy_advisor.rs similarity index 100% rename from tests/suites/conformance/tests/policy_advisor.rs rename to tests/suites/conformance/src/policy_advisor.rs diff --git a/tests/suites/conformance/tests/smoke.rs b/tests/suites/conformance/src/smoke.rs similarity index 100% rename from tests/suites/conformance/tests/smoke.rs rename to tests/suites/conformance/src/smoke.rs diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 1206d952e4..12945d3f1f 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -54,6 +54,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -425,6 +475,46 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmake" version = "0.1.58" @@ -440,6 +530,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -721,6 +817,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + [[package]] name = "etcetera" version = "0.11.0" @@ -1477,6 +1579,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.14.0" @@ -1741,6 +1849,38 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1878,6 +2018,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "newtype-uuid" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa56da58097c7ae893c4097bb6d91bb5f7fa330053e07e033349db4d5f83ba8" +dependencies = [ + "uuid", +] + [[package]] name = "nix" version = "0.29.0" @@ -2002,6 +2151,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openshell-bootstrap" version = "0.0.0" @@ -2270,11 +2425,16 @@ dependencies = [ name = "openshell-testkit" version = "0.0.0" dependencies = [ + "clap", "futures", + "libc", + "libtest-mimic", + "linkme", "openshell-bootstrap", "openshell-core", "openshell-sdk", "openshell-testkit-macros", + "quick-junit", "rand 0.9.4", "serde", "serde_json", @@ -2715,6 +2875,30 @@ dependencies = [ "pulldown-cmark", ] +[[package]] +name = "quick-junit" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae23b8e1a59a90f7bf98caa2f4c8c2eedf6df011c6c88ed633e85a87c84526e" +dependencies = [ + "chrono", + "indexmap", + "newtype-uuid", + "quick-xml", + "strip-ansi-escapes", + "thiserror 2.0.20", + "uuid", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.45" @@ -3518,6 +3702,15 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -4173,6 +4366,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.23.1" @@ -4202,6 +4401,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "want" version = "0.3.1" diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index a4ba9ea16c..f85db132ef 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -54,6 +54,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -382,6 +432,46 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmake" version = "0.1.58" @@ -397,6 +487,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -669,6 +765,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + [[package]] name = "etcetera" version = "0.11.0" @@ -1385,6 +1487,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.14.0" @@ -1640,6 +1748,38 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1757,6 +1897,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "newtype-uuid" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa56da58097c7ae893c4097bb6d91bb5f7fa330053e07e033349db4d5f83ba8" +dependencies = [ + "uuid", +] + [[package]] name = "nix" version = "0.29.0" @@ -1872,6 +2021,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openshell-bootstrap" version = "0.0.0" @@ -1994,11 +2149,16 @@ dependencies = [ name = "openshell-testkit" version = "0.0.0" dependencies = [ + "clap", "futures", + "libc", + "libtest-mimic", + "linkme", "openshell-bootstrap", "openshell-core", "openshell-sdk", "openshell-testkit-macros", + "quick-junit", "rand 0.9.4", "serde", "serde_json", @@ -2356,6 +2516,30 @@ dependencies = [ "pulldown-cmark", ] +[[package]] +name = "quick-junit" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae23b8e1a59a90f7bf98caa2f4c8c2eedf6df011c6c88ed633e85a87c84526e" +dependencies = [ + "chrono", + "indexmap", + "newtype-uuid", + "quick-xml", + "strip-ansi-escapes", + "thiserror 2.0.20", + "uuid", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.45" @@ -3133,6 +3317,15 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3719,6 +3912,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.23.1" @@ -3742,6 +3941,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "want" version = "0.3.1" diff --git a/tests/testkit/Cargo.lock b/tests/testkit/Cargo.lock index 6889775edc..689ecea41c 100644 --- a/tests/testkit/Cargo.lock +++ b/tests/testkit/Cargo.lock @@ -54,6 +54,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -382,6 +432,46 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmake" version = "0.1.58" @@ -397,6 +487,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -669,6 +765,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + [[package]] name = "etcetera" version = "0.11.0" @@ -1385,6 +1487,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.14.0" @@ -1631,6 +1739,38 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1748,6 +1888,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "newtype-uuid" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa56da58097c7ae893c4097bb6d91bb5f7fa330053e07e033349db4d5f83ba8" +dependencies = [ + "uuid", +] + [[package]] name = "nix" version = "0.29.0" @@ -1863,6 +2012,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openshell-bootstrap" version = "0.0.0" @@ -1985,14 +2140,20 @@ dependencies = [ name = "openshell-testkit" version = "0.0.0" dependencies = [ + "clap", "futures", + "libc", + "libtest-mimic", + "linkme", "openshell-bootstrap", "openshell-core", "openshell-sdk", "openshell-testkit-macros", + "quick-junit", "rand 0.9.4", "serde", "serde_json", + "tempfile", "tokio", ] @@ -2347,6 +2508,30 @@ dependencies = [ "pulldown-cmark", ] +[[package]] +name = "quick-junit" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae23b8e1a59a90f7bf98caa2f4c8c2eedf6df011c6c88ed633e85a87c84526e" +dependencies = [ + "chrono", + "indexmap", + "newtype-uuid", + "quick-xml", + "strip-ansi-escapes", + "thiserror 2.0.20", + "uuid", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.45" @@ -3124,6 +3309,15 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3710,6 +3904,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.23.1" @@ -3733,6 +3933,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "want" version = "0.3.1" diff --git a/tests/testkit/README.md b/tests/testkit/README.md index ae71d0f81b..4bba57931a 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -7,7 +7,7 @@ This is a separate Cargo workspace with its own lockfile: | Crate | Purpose | |---|---| -| `testkit` | Finds and connects to the gateway, creates and cleans up sandboxes, runs the CLI. Also builds `testkit-ready`. | +| `testkit` | Finds and connects to the gateway, creates and cleans up sandboxes, runs the CLI, and runs test programs. Also builds `testkit-ready`. | | `testkit-macros` | The `#[testkit::test]` attribute. | ## Writing a test @@ -99,6 +99,67 @@ In a tmachine VM, the test runner (`tests/runner/`) does the same: it sets `OPENSHELL_BIN` to the installed CLI, then runs the suites with nextest. See [TESTING.md](../../TESTING.md). +## Test programs + +`#[testkit::test]` also registers the test with testkit. A binary whose +`main` calls `testkit::main()` runs every registered test it links, without +cargo or nextest: + +```rust +use conformance as _; + +fn main() -> std::process::ExitCode { + testkit::main() +} +``` + +The program must name each suite crate it runs, as `use conformance as _;` does +above: Rust doesn't link a dependency that is never named, and its tests would +be missing. A program that links no tests exits 2 with an error under every +command. `openshell-conformance`, a binary in the conformance crate, is the +program for the conformance suite; see +[its README](../suites/conformance/README.md). + +| Command | What it does | +|---|---| +| `list [FILTER]...` | Prints each test's name and description. Ignored tests are included and marked. | +| `ready [--timeout SECS]` | Waits for the gateway, like `testkit-ready` (default 60 seconds). Exits 2 if it is not ready in time. | +| `run [FILTER]...` | Runs the tests one at a time, each in its own process. Unix only. | + +A filter selects tests whose name contains it. `run` also takes: + +- `--include-ignored`: runs ignored tests too, instead of reporting them as + skipped. +- `--timeout SECS` (default 900): a test that runs longer gets SIGTERM, then + SIGKILL after `--grace SECS` (default 10). +- `--junit PATH`: writes a JUnit report with one test suite per family. + +Each test runs in its own process group. When the test's process exits, `run` +kills anything left in the group, such as CLI processes the test started. +Interrupting `run` stops the current test the same way and runs no more +tests; a second interrupt skips the grace period. If `run` itself is killed +with SIGKILL, the current test's process group keeps running. In a container +where `run` is PID 1, start it under an init, for example with +`docker run --init`. + +`run` exits 0 when every test passed, 1 when a test failed, timed out, or +crashed, and 2 when tests could not run: no tests matched, a test process +could not start or run its test, or the run was interrupted. + +A test that is stopped does not clean up: sandboxes it created stay on the +gateway. Run against a gateway you can throw away, or delete them by hand. + +Without a subcommand, the program takes libtest's arguments instead. A +`[[test]]` target with `harness = false` therefore also runs under `cargo test` +and nextest. In that mode tests run in one process, one at a time unless +`--test-threads` says otherwise. + +A test's name is its module path without the crate name, then its function +name, as in nextest. Its family, which groups tests in the JUnit report, is +the first directory under the last `suites` directory in the crate's path: +`conformance` for `tests/suites/conformance`, `features` for +`tests/suites/features/`. + ## Checks ```shell diff --git a/tests/testkit/testkit-macros/src/lib.rs b/tests/testkit/testkit-macros/src/lib.rs index fc3dbbdb68..b8a614f3c6 100644 --- a/tests/testkit/testkit-macros/src/lib.rs +++ b/tests/testkit/testkit-macros/src/lib.rs @@ -13,12 +13,28 @@ //! async fn expose_service(cx: &mut Context) -> testkit::Result { /* ... */ } //! ``` //! -//! The attribute expands to a `#[tokio::test]` wrapper with the same name, so -//! the nextest test name is the function name. The original function is nested -//! in the wrapper and passed to `testkit::run`. Other attributes, such as -//! `#[ignore]` or `#[cfg(...)]`, move to the wrapper. `#[should_panic]` is -//! rejected: a testkit test panics on any failure, so it would pass for the -//! wrong reason. Check the expected failure in the body instead. +//! The attribute generates three items: +//! +//! - a hidden `async fn` that runs the original function through +//! `testkit::run`, with the original function nested in it; +//! - under `#[cfg(test)]`, a `#[tokio::test]` wrapper with the same name that +//! calls it, so `cargo test` and nextest run the test like any other and the +//! test name is the function name; +//! - an entry in `testkit`'s test registry, with the test's family, name, +//! description, and `#[ignore]`, which `testkit::main()` runs. +//! +//! Other attributes, such as `#[ignore]` or `#[cfg(...)]`, move to the +//! wrapper. `#[cfg(...)]` also applies to the other two items, and lint +//! attributes (`allow`, `warn`, `deny`, `forbid`) also to the hidden +//! function. `#[should_panic]` is rejected: a testkit test panics on any +//! failure, so it would pass for the wrong reason. Check the expected failure +//! in the body instead. `#[cfg_attr]` and `#[expect]` are rejected too. +//! +//! A test's family is the first directory under `tests/suites/` in its +//! crate's path: `conformance` for `tests/suites/conformance`, `features` for +//! `tests/suites/features/`. + +use std::path::Path; use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; @@ -27,12 +43,17 @@ use syn::{Attribute, Expr, ExprLit, ItemFn, Lit, LitStr, Meta}; #[proc_macro_attribute] pub fn test(attr: TokenStream, item: TokenStream) -> TokenStream { - expand(attr.into(), item.into()) + let family = std::env::var_os("CARGO_MANIFEST_DIR").and_then(|dir| family_of(Path::new(&dir))); + expand(attr.into(), item.into(), family.as_deref()) .unwrap_or_else(syn::Error::into_compile_error) .into() } -fn expand(attr: TokenStream2, item: TokenStream2) -> syn::Result { +fn expand( + attr: TokenStream2, + item: TokenStream2, + family: Option<&str>, +) -> syn::Result { let mut body: ItemFn = syn::parse2(item)?; if !attr.is_empty() { return Err(syn::Error::new_spanned( @@ -58,17 +79,24 @@ fn expand(attr: TokenStream2, item: TokenStream2) -> syn::Result { "`#[testkit::test]` already makes this a test; remove this attribute", )); } - if let Some(attr) = body - .attrs - .iter() - .find(|attr| attr.path().is_ident("should_panic")) - { - return Err(syn::Error::new_spanned( - attr, + for attr in &body.attrs { + let message = if attr.path().is_ident("should_panic") { "testkit tests panic on any failure, so `#[should_panic]` would pass for the \ wrong reason; check the expected failure in the body, for example with \ - `CliOutput::failure` and `testkit::ensure!`", - )); + `CliOutput::failure` and `testkit::ensure!`" + } else if attr.path().is_ident("cfg_attr") { + // The registration reads `#[ignore]` and `#[cfg]` directly, so it + // would not see them inside `cfg_attr`. + "`#[cfg_attr]` is not supported on testkit tests; use `#[cfg]` or `#[ignore]` \ + directly" + } else if attr.path().is_ident("expect") { + // The lint attributes are copied to two functions, and only one + // of them would raise the lint. + "`#[expect]` is not supported on testkit tests; use `#[allow]`" + } else { + continue; + }; + return Err(syn::Error::new_spanned(attr, message)); } let description = doc_comment(&body.attrs); if description.is_empty() { @@ -77,20 +105,46 @@ fn expand(attr: TokenStream2, item: TokenStream2) -> syn::Result { "testkit tests need a doc comment describing the behavior they verify", )); } + let ignore = ignore(&body.attrs)?; let name = body.sig.ident.clone(); let name_str = LitStr::new(&name.to_string(), name.span()); let description = LitStr::new(&description, name.span()); + let family = match family { + Some(family) => quote!(::core::option::Option::Some(#family)), + None => quote!(::core::option::Option::None), + }; + let (ignored, ignore_reason) = match ignore { + None => (false, quote!(::core::option::Option::None)), + Some(None) => (true, quote!(::core::option::Option::None)), + Some(Some(reason)) => (true, quote!(::core::option::Option::Some(#reason))), + }; let vis = body.vis.clone(); let attrs = std::mem::take(&mut body.attrs); + let cfgs: Vec<&Attribute> = attrs + .iter() + .filter(|attr| attr.path().is_ident("cfg")) + .collect(); + // `#[cfg]` and lint attributes such as `#[allow(...)]` must reach the + // body, which is nested in the hidden function. Anything else is for the + // test itself, so goes on the wrapper only. + let body_attrs: Vec<&Attribute> = attrs + .iter() + .filter(|attr| { + ["cfg", "allow", "warn", "deny", "forbid"] + .iter() + .any(|name| attr.path().is_ident(name)) + }) + .collect(); body.vis = syn::Visibility::Inherited; body.sig.ident = format_ident!("body"); + let entry = format_ident!("__testkit_{}", name); Ok(quote! { - #(#attrs)* - #[::testkit::__private::tokio::test(crate = "::testkit::__private::tokio")] - #vis async fn #name() { + #(#body_attrs)* + #[doc(hidden)] + async fn #entry() { #body ::testkit::run( ::testkit::TestInfo { @@ -101,6 +155,30 @@ fn expand(attr: TokenStream2, item: TokenStream2) -> syn::Result { ) .await; } + + #(#attrs)* + #[cfg(test)] + #[::testkit::__private::tokio::test(crate = "::testkit::__private::tokio")] + #vis async fn #name() { + #entry().await; + } + + #(#cfgs)* + const _: () = { + #[::testkit::__private::linkme::distributed_slice(::testkit::__private::TESTS)] + #[linkme(crate = ::testkit::__private::linkme)] + static TEST: ::testkit::Test = ::testkit::Test { + info: ::testkit::TestInfo { + name: #name_str, + description: #description, + }, + family: #family, + module_path: ::core::module_path!(), + ignored: #ignored, + ignore_reason: #ignore_reason, + run: || ::std::boxed::Box::pin(#entry()), + }; + }; }) } @@ -112,6 +190,31 @@ fn is_test_attr(attr: &Attribute) -> bool { .is_some_and(|segment| segment.ident == "test") } +/// `None` if the test is not ignored, else the `#[ignore = "..."]` reason if +/// it has one. +fn ignore(attrs: &[Attribute]) -> syn::Result>> { + let Some(attr) = attrs.iter().find(|attr| attr.path().is_ident("ignore")) else { + return Ok(None); + }; + match &attr.meta { + Meta::Path(_) => Ok(Some(None)), + Meta::NameValue(meta) => match &meta.value { + Expr::Lit(ExprLit { + lit: Lit::Str(reason), + .. + }) => Ok(Some(Some(reason.clone()))), + value => Err(syn::Error::new_spanned( + value, + "the `#[ignore]` reason must be a string literal", + )), + }, + Meta::List(_) => Err(syn::Error::new_spanned( + attr, + "use `#[ignore]` or `#[ignore = \"reason\"]`", + )), + } +} + /// The doc comment, joined into one line. Empty if there is none. fn doc_comment(attrs: &[Attribute]) -> String { attrs @@ -131,21 +234,40 @@ fn doc_comment(attrs: &[Attribute]) -> String { .join(" ") } +/// The family of a suite crate in `manifest_dir`: the first directory under +/// the nearest enclosing `suites/`. +fn family_of(manifest_dir: &Path) -> Option { + let components: Vec<_> = manifest_dir + .components() + .filter_map(|component| component.as_os_str().to_str()) + .collect(); + let suites = components + .iter() + .rposition(|component| *component == "suites")?; + components + .get(suites + 1) + .map(|family| (*family).to_string()) +} + #[cfg(test)] mod tests { - use super::expand; + use std::path::Path; + + use super::{expand, family_of}; + + const OK_FN: &str = "async fn t(cx: &mut Context) -> testkit::Result { Ok(()) }"; // A suite only compiles correct tests, so it never reaches these errors. + // `testkit/tests/macro.rs` checks what correct tests expand to. #[test] fn rejects_misuse() { - let ok_fn = "async fn t(cx: &mut Context) -> testkit::Result { Ok(()) }"; for (attr, item, error) in [ ( "id = \"t-1\"", - format!("/// Doc.\n{ok_fn}"), + format!("/// Doc.\n{OK_FN}"), "takes no arguments", ), - ("", ok_fn.to_string(), "doc comment"), + ("", OK_FN.to_string(), "doc comment"), ( "", "/// Doc.\nfn t(cx: &mut Context) -> testkit::Result { Ok(()) }".to_string(), @@ -158,17 +280,53 @@ mod tests { ), ( "", - format!("/// Doc.\n#[tokio::test]\n{ok_fn}"), + format!("/// Doc.\n#[tokio::test]\n{OK_FN}"), "already makes this a test", ), ( "", - format!("/// Doc.\n#[should_panic]\n{ok_fn}"), + format!("/// Doc.\n#[should_panic]\n{OK_FN}"), "`#[should_panic]`", ), + ( + "", + format!("/// Doc.\n#[ignore(later)]\n{OK_FN}"), + "`#[ignore]` or", + ), + ( + "", + format!("/// Doc.\n#[ignore = 1]\n{OK_FN}"), + "string literal", + ), + ( + "", + format!("/// Doc.\n#[cfg_attr(unix, ignore)]\n{OK_FN}"), + "`#[cfg_attr]` is not supported", + ), + ( + "", + format!("/// Doc.\n#[expect(unused)]\n{OK_FN}"), + "`#[expect]` is not supported", + ), ] { - let err = expand(attr.parse().unwrap(), item.parse().unwrap()).unwrap_err(); + let err = expand(attr.parse().unwrap(), item.parse().unwrap(), None).unwrap_err(); assert!(err.to_string().contains(error), "{item}: {err}"); } } + + #[test] + fn family_is_the_directory_under_suites() { + for (dir, family) in [ + ("/repo/tests/suites/conformance", Some("conformance")), + ( + "/repo/tests/suites/features/provider-refresh/keycloak", + Some("features"), + ), + ("/a/suites/x/tests/suites/drivers/podman", Some("drivers")), + ("/repo/tests/testkit/testkit", None), + ("/repo/tests/suites", None), + ] { + assert_eq!(family_of(Path::new(dir)).as_deref(), family, "{dir}"); + } + } } diff --git a/tests/testkit/testkit/Cargo.toml b/tests/testkit/testkit/Cargo.toml index cfcf0391ee..f8f489c828 100644 --- a/tests/testkit/testkit/Cargo.toml +++ b/tests/testkit/testkit/Cargo.toml @@ -19,8 +19,24 @@ openshell-bootstrap = { path = "../../../crates/openshell-bootstrap" } openshell-core = { path = "../../../crates/openshell-core", default-features = false } openshell-sdk = { path = "../../../crates/openshell-sdk" } openshell-testkit-macros = { path = "../testkit-macros" } +clap = { version = "4", features = ["derive"] } futures = "0.3" +libtest-mimic = "0.8" +linkme = "0.3" +quick-junit = "0.8" rand = "0.9" serde = "1" serde_json = "1" tokio = { version = "1.43", features = ["macros", "process", "rt", "time"] } + +[target.'cfg(unix)'.dependencies] +libc = "0.2" + +[dev-dependencies] +tempfile = "3" + +# A test program, run by `cargo test` and nextest through `testkit::main`'s +# libtest interface, that checks the program's own subcommands. +[[test]] +name = "program" +harness = false diff --git a/tests/testkit/testkit/src/bin/testkit-ready.rs b/tests/testkit/testkit/src/bin/testkit-ready.rs index 4f41b3bed4..03db9d3547 100644 --- a/tests/testkit/testkit/src/bin/testkit-ready.rs +++ b/tests/testkit/testkit/src/bin/testkit-ready.rs @@ -6,13 +6,12 @@ //! Usage: `testkit-ready [TIMEOUT_SECS]` (default 60), with the gateway //! selected as for tests. Run it once per environment before the test //! binaries; exits non-zero with the last error if the gateway is not healthy -//! in time, or if the CLI does not reach it. +//! in time, or if the CLI does not reach it. A test program's `ready` +//! subcommand does the same. use std::process::ExitCode; use std::time::Duration; -use testkit::{Cli, Gateway}; - const DEFAULT_TIMEOUT_SECS: u64 = 60; const USAGE: &str = "usage: testkit-ready [TIMEOUT_SECS] (default 60)"; @@ -32,25 +31,9 @@ async fn main() -> ExitCode { } }, }; - let result = async { - let gateway = Gateway::from_env()?; - eprintln!("testkit-ready: {}", gateway.describe()); - let health = gateway.wait_ready(Duration::from_secs(timeout)).await?; - Cli::from_env(&gateway).check_connected().await?; - testkit::Result::Ok(health) - } - .await; - match result { - Ok(health) => { - eprintln!( - "testkit-ready: gateway {} is healthy and the CLI reaches it", - health.version - ); - ExitCode::SUCCESS - } - Err(error) => { - eprintln!("testkit-ready: {error}"); - ExitCode::FAILURE - } + if testkit::check_ready("testkit-ready", Duration::from_secs(timeout)).await { + ExitCode::SUCCESS + } else { + ExitCode::FAILURE } } diff --git a/tests/testkit/testkit/src/gateway.rs b/tests/testkit/testkit/src/gateway.rs index ef40cb1bf9..cba28b88a4 100644 --- a/tests/testkit/testkit/src/gateway.rs +++ b/tests/testkit/testkit/src/gateway.rs @@ -243,6 +243,23 @@ mod tests { use super::Gateway; + #[test] + fn unsupported_auth_modes_are_named() { + let error = Gateway::registered( + "edge".to_string(), + "https://gateway.example".to_string(), + Some("oidc"), + "/nonexistent".into(), + "default".to_string(), + false, + ) + .expect_err("OIDC is not supported"); + assert!( + error.to_string().contains("gateway edge uses oidc auth"), + "{error}" + ); + } + /// A peer that accepts connections and never answers must not hang the /// readiness wait: the timeout covers the connect and health calls too. #[tokio::test] diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 168fdbe894..e2c49e3caf 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -21,12 +21,19 @@ //! //! Tests find the gateway the same way the `openshell` CLI does (see //! [`gateway`]). +//! +//! `#[testkit::test]` also registers each test, so a binary that calls +//! [`main`] is a test program that lists, runs, and reports its tests without +//! Cargo (see [`program`]). mod cli; mod context; mod error; pub mod gateway; mod poll; +pub mod program; +mod ready; +mod registry; mod run; pub use cli::{BIN_ENV, CLI_TIMEOUT, Cli, CliOutput}; @@ -38,10 +45,15 @@ pub use error::{Error, Result, ResultExt}; pub use gateway::Gateway; pub use openshell_sdk as sdk; pub use poll::{Poll, poll_until}; +pub use program::main; +pub use ready::check as check_ready; +pub use registry::Test; pub use run::{TestInfo, run}; pub use testkit_macros::test; #[doc(hidden)] pub mod __private { + pub use crate::registry::TESTS; + pub use linkme; pub use tokio; } diff --git a/tests/testkit/testkit/src/program/capture.rs b/tests/testkit/testkit/src/program/capture.rs new file mode 100644 index 0000000000..329c22ac68 --- /dev/null +++ b/tests/testkit/testkit/src/program/capture.rs @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Copying a test's output to the console while keeping it for the report. + +use std::collections::VecDeque; +use std::io::{self, Read, Write}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +/// How much of each stream is kept for the report. Output beyond this is +/// still shown; the report keeps the end, where failures are. +const KEPT_BYTES: usize = 4 << 20; + +/// How long to wait for a stream to close after the test's process group is +/// gone. A process that left the group can hold it open. +const CLOSE_TIMEOUT: Duration = Duration::from_secs(5); + +/// One stream being copied. +pub struct Capture { + kept: Arc>, + thread: JoinHandle<()>, +} + +#[derive(Default)] +struct Kept { + bytes: VecDeque, + dropped: usize, +} + +impl Capture { + /// Copy `stream` to `console` on a thread, keeping a copy. + pub fn start( + mut stream: impl Read + Send + 'static, + mut console: impl Write + Send + 'static, + ) -> Self { + let kept = Arc::new(Mutex::new(Kept::default())); + let thread = std::thread::spawn({ + let kept = Arc::clone(&kept); + move || { + let mut buffer = [0; 8192]; + loop { + let read = match stream.read(&mut buffer) { + Ok(0) => return, + Ok(read) => read, + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(_) => return, + }; + let chunk = &buffer[..read]; + let _ = console.write_all(chunk).and_then(|()| console.flush()); + kept.lock() + .unwrap_or_else(PoisonError::into_inner) + .push(chunk); + } + } + }); + Self { kept, thread } + } + + /// The output kept, once the stream closes or after a short wait. + pub fn finish(self) -> Vec { + let deadline = Instant::now() + CLOSE_TIMEOUT; + while !self.thread.is_finished() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + if self.thread.is_finished() { + let _ = self.thread.join(); + } + let kept = self.kept.lock().unwrap_or_else(PoisonError::into_inner); + let mut bytes = Vec::with_capacity(kept.bytes.len() + 64); + if kept.dropped > 0 { + bytes.extend(format!("[{} earlier bytes not kept]\n", kept.dropped).as_bytes()); + } + bytes.extend(&kept.bytes); + bytes + } +} + +impl Kept { + fn push(&mut self, chunk: &[u8]) { + self.bytes.extend(chunk); + let excess = self.bytes.len().saturating_sub(KEPT_BYTES); + self.bytes.drain(..excess); + self.dropped += excess; + } +} + +#[cfg(test)] +mod tests { + use super::{Capture, KEPT_BYTES, Kept}; + + #[test] + fn copies_and_keeps_the_stream() { + let capture = Capture::start(&b"one\ntwo\n"[..], std::io::sink()); + assert_eq!(capture.finish(), b"one\ntwo\n"); + } + + #[test] + fn keeps_the_end_of_long_output() { + let mut kept = Kept::default(); + kept.push(&vec![b'a'; KEPT_BYTES]); + kept.push(b"end"); + assert_eq!(kept.bytes.len(), KEPT_BYTES); + let tail: Vec = kept.bytes.iter().rev().take(4).rev().copied().collect(); + assert_eq!(tail, b"aend"); + assert_eq!(kept.dropped, 3); + } +} diff --git a/tests/testkit/testkit/src/program/junit.rs b/tests/testkit/testkit/src/program/junit.rs new file mode 100644 index 0000000000..0a1e66d648 --- /dev/null +++ b/tests/testkit/testkit/src/program/junit.rs @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The `JUnit` report of a run. +//! +//! One `` per family. Each `` is named by the test name, +//! with the module path as its class name and the description and family as +//! properties. + +use std::collections::BTreeMap; +use std::path::Path; + +use quick_junit::{NonSuccessKind, Property, Report, TestCase, TestCaseStatus, TestSuite}; + +use super::supervisor::{Outcome, TestRun}; + +pub fn report(program: &str, runs: &[TestRun<'_>]) -> Report { + let mut suites: BTreeMap<&str, TestSuite> = BTreeMap::new(); + for run in runs { + let family = run.test.family(); + suites + .entry(family) + .or_insert_with(|| TestSuite::new(family)) + .add_test_case(test_case(run)); + } + let mut report = Report::new(program); + report.add_test_suites(suites.into_values()); + report +} + +fn test_case(run: &TestRun<'_>) -> TestCase { + let status = match &run.outcome { + Outcome::Passed => TestCaseStatus::success(), + Outcome::Skipped => { + let mut status = TestCaseStatus::skipped(); + status.set_message(match run.test.ignore_reason { + Some(reason) => format!("ignored: {reason}"), + None => "ignored".to_string(), + }); + status + } + Outcome::Failed(message) => non_success(NonSuccessKind::Failure, "failure", message), + Outcome::TimedOut(message) => non_success(NonSuccessKind::Failure, "timeout", message), + Outcome::Error(message) => non_success(NonSuccessKind::Error, "error", message), + Outcome::Interrupted => non_success(NonSuccessKind::Error, "interrupted", "interrupted"), + }; + let mut case = TestCase::new(run.test.name(), status); + case.set_classname(run.test.module_path) + .add_property(Property::new("family", run.test.family())) + .add_property(Property::new("description", run.test.info.description)); + if let Some(duration) = run.duration { + case.set_time(duration); + } + if !run.stdout.is_empty() { + case.set_system_out_lossy(&run.stdout); + } + if !run.stderr.is_empty() { + case.set_system_err_lossy(&run.stderr); + } + case +} + +fn non_success(kind: NonSuccessKind, ty: &str, message: &str) -> TestCaseStatus { + let mut status = TestCaseStatus::non_success(kind); + status.set_type(ty).set_message(message); + status +} + +pub fn write(path: &Path, report: &Report) -> std::io::Result<()> { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + std::fs::create_dir_all(parent)?; + } + let xml = report.to_string().map_err(std::io::Error::other)?; + std::fs::write(path, xml) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::report; + use crate::program::supervisor::{Outcome, TestRun}; + use crate::registry::Test; + use crate::registry::tests::test; + + const PASSED: Test = test("sandbox::smoke", "create_exec_delete", false); + const FAILED: Test = test("sandbox::lifecycle", "stop_start", false); + const SKIPPED: Test = Test { + ignore_reason: Some("https://github.com/NVIDIA/OpenShell/issues/1"), + ..test("sandbox::lifecycle", "later", true) + }; + + #[test] + fn reports_every_outcome() { + let run = |test, outcome, duration| TestRun { + test, + outcome, + duration, + stdout: b"out\n".to_vec(), + stderr: Vec::new(), + }; + let runs = [ + run(&PASSED, Outcome::Passed, Some(Duration::from_millis(1500))), + run( + &FAILED, + Outcome::TimedOut("timed out after 1s".into()), + Some(Duration::from_secs(1)), + ), + run(&SKIPPED, Outcome::Skipped, None), + ]; + let xml = report("openshell-conformance", &runs).to_string().unwrap(); + for expected in [ + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + "out\n", + ] { + assert!(xml.contains(expected), "missing {expected} in\n{xml}"); + } + } +} diff --git a/tests/testkit/testkit/src/program/mod.rs b/tests/testkit/testkit/src/program/mod.rs new file mode 100644 index 0000000000..c7ca98f3ba --- /dev/null +++ b/tests/testkit/testkit/src/program/mod.rs @@ -0,0 +1,383 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `testkit::main()`: a test program anyone can run against their gateway. +//! +//! A program is a binary whose `main` calls [`main`], linked with the crates +//! holding its `#[testkit::test]`s: +//! +//! ```ignore +//! fn main() -> std::process::ExitCode { +//! testkit::main() +//! } +//! ``` +//! +//! It has three subcommands: +//! +//! - `list [FILTER]...` prints each test's name and description. +//! - `ready [--timeout SECS]` waits for the gateway, as `testkit-ready` does. +//! - `run [FILTER]...` runs tests one at a time, each in its own process with +//! a timeout, and can write a `JUnit` report. Unix only. +//! +//! Any other arguments are libtest's, so the program also works as a +//! `harness = false` test target under `cargo test` and nextest: it answers +//! `--list --format terse`, `--ignored`, and ` --exact --nocapture`. +//! In that mode tests run in the program's own process, one at a time unless +//! `--test-threads` says otherwise. + +#[cfg(unix)] +mod capture; +#[cfg(unix)] +mod junit; +#[cfg(unix)] +mod process; +#[cfg(unix)] +mod supervisor; + +use std::ffi::OsString; +use std::path::PathBuf; +use std::process::ExitCode; +use std::time::Duration; + +use clap::{Parser, Subcommand}; + +use crate::registry::{RESERVED_NAMES, Selection, TESTS, Test, validate}; + +/// Exit code when a test failed. +const FAILED_EXIT: u8 = 1; +/// Exit code when tests could not run, including usage errors. +const ERROR_EXIT: u8 = 2; +/// Set by `run` for the test processes it starts. +const SUPERVISED_ENV: &str = "TESTKIT_SUPERVISED"; + +/// Run the program: the subcommands, or libtest's interface. +pub fn main() -> ExitCode { + main_with(&TESTS) +} + +/// [`main`] over `tests` instead of every registered test. +#[doc(hidden)] +pub fn main_with(tests: &'static [Test]) -> ExitCode { + let args: Vec = std::env::args_os().collect(); + let program = program_name(&args); + // A suite crate that the program never names is not linked, and its + // tests are silently missing. Fail instead of passing with nothing. + if tests.is_empty() { + eprintln!( + "{program}: no tests are linked into this program; it must name each suite crate \ + it runs, for example with `use as _;`" + ); + return ExitCode::from(ERROR_EXIT); + } + if let Err(error) = validate(tests) { + eprintln!("{program}: {error}"); + return ExitCode::from(ERROR_EXIT); + } + let first = args.get(1).and_then(|arg| arg.to_str()); + if first.is_some_and(|arg| RESERVED_NAMES.contains(&arg) || arg == "-h" || arg == "--help") { + let cli = match Cli::try_parse_from(&args) { + Ok(cli) => cli, + Err(error) => { + let _ = error.print(); + return ExitCode::from(if error.use_stderr() { ERROR_EXIT } else { 0 }); + } + }; + command(&program, tests, cli.command) + } else { + libtest(tests, args) + } +} + +#[derive(Parser)] +#[command( + about = "Run OpenShell tests against a gateway", + after_help = "The gateway is the one the openshell CLI selects: OPENSHELL_GATEWAY_ENDPOINT, \ + OPENSHELL_GATEWAY, or the active gateway. OPENSHELL_WORKSPACE selects the \ + workspace and OPENSHELL_BIN the CLI. Only plaintext and mTLS gateways are \ + supported.\n\nWithout a subcommand the program takes libtest's arguments, \ + as under cargo test and nextest." +)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Print each test's name and description, including ignored tests. + List { + /// Show tests whose name contains any of these (all if none). + filters: Vec, + }, + /// Wait until the gateway is healthy and the CLI reaches it. Exits 2 if + /// it is not ready in time. + Ready { + /// How long to wait, in seconds. + #[arg(long, default_value = "60", value_name = "SECS", value_parser = seconds)] + timeout: Duration, + }, + /// Run tests one at a time, each in its own process. Exits 1 if a test + /// failed and 2 if tests could not run. + Run { + /// Run tests whose name contains any of these (all if none). + filters: Vec, + /// Run ignored tests too. + #[arg(long)] + include_ignored: bool, + /// Stop a test that runs longer than this, in seconds. Longer than + /// testkit's own step timeouts, so those report first. + #[arg(long, default_value = "900", value_name = "SECS", value_parser = seconds)] + timeout: Duration, + /// How long a stopped test has between SIGTERM and SIGKILL, in seconds. + #[arg(long, default_value = "10", value_name = "SECS", value_parser = seconds)] + grace: Duration, + /// Write a JUnit report here. + #[arg(long, value_name = "PATH")] + junit: Option, + }, +} + +/// A positive, finite number of seconds. +fn seconds(value: &str) -> Result { + let seconds: f64 = value + .parse() + .map_err(|_| format!("{value:?} is not a number"))?; + Duration::try_from_secs_f64(seconds) + .ok() + .filter(|duration| !duration.is_zero()) + .ok_or_else(|| format!("{value:?} is not a positive number of seconds")) +} + +fn command(program: &str, tests: &'static [Test], command: Command) -> ExitCode { + match command { + Command::List { filters } => { + list(tests, filters); + ExitCode::SUCCESS + } + Command::Ready { timeout } => ready(program, timeout), + Command::Run { + filters, + include_ignored, + timeout, + grace, + junit, + } => run( + program, + tests, + Selection { + filters, + include_ignored, + }, + timeout, + grace, + junit, + ), + } +} + +#[cfg(unix)] +fn run( + program: &str, + tests: &'static [Test], + selection: Selection, + timeout: Duration, + grace: Duration, + junit: Option, +) -> ExitCode { + let exe = match std::env::current_exe() { + Ok(exe) => exe, + Err(error) => { + eprintln!("{program}: find this program's path: {error}"); + return ExitCode::from(ERROR_EXIT); + } + }; + let options = supervisor::Options { + selection, + timeout, + grace, + junit, + }; + supervisor::run(program, &exe, tests, &options) +} + +#[cfg(not(unix))] +fn run( + program: &str, + _tests: &'static [Test], + _selection: Selection, + _timeout: Duration, + _grace: Duration, + _junit: Option, +) -> ExitCode { + eprintln!( + "{program}: `run` is only supported on Unix; run the tests with cargo test or nextest" + ); + ExitCode::from(ERROR_EXIT) +} + +/// One line per test: name and description, in aligned columns. +/// An ignored test's description ends with `[ignored]` or its reason. +fn list(tests: &[Test], filters: Vec) { + let selection = Selection { + filters, + include_ignored: true, + }; + let rows: Vec<_> = selection + .apply(tests) + .into_iter() + .map(|(test, _)| (test.name(), describe(test))) + .collect(); + let name_width = rows.iter().map(|row| row.0.len()).max().unwrap_or(0); + for (name, description) in rows { + println!("{name:name_width$} {description}"); + } +} + +fn describe(test: &Test) -> String { + let description = test.info.description; + match (test.ignored, test.ignore_reason) { + (false, _) => description.to_string(), + (true, None) => format!("{description} [ignored]"), + (true, Some(reason)) => format!("{description} [ignored: {reason}]"), + } +} + +fn ready(program: &str, timeout: Duration) -> ExitCode { + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + eprintln!("{program}: start the async runtime: {error}"); + return ExitCode::from(ERROR_EXIT); + } + }; + // A gateway that is not ready means the tests cannot run. + if runtime.block_on(crate::ready::check(program, timeout)) { + ExitCode::SUCCESS + } else { + ExitCode::from(ERROR_EXIT) + } +} + +/// libtest's interface, for `cargo test` and nextest, and for `run`'s test +/// processes. +fn libtest(tests: &'static [Test], args: Vec) -> ExitCode { + let mut args = libtest_mimic::Arguments::from_iter(args); + // Tests share one gateway; run them one at a time unless asked otherwise. + if args.test_threads.is_none() { + args.test_threads = Some(1); + } + let mut tests: Vec<&'static Test> = tests.iter().collect(); + tests.sort_by_cached_key(|test| test.name()); + let trials = tests + .into_iter() + .map(|test| { + libtest_mimic::Trial::test(test.name(), move || { + run_in_process(test); + Ok(()) + }) + .with_ignored_flag(test.ignored) + }) + .collect(); + let conclusion = libtest_mimic::run(&args, trials); + // `run` starts this process for exactly one test. Running none would + // look like a pass. + if std::env::var_os(SUPERVISED_ENV).is_some() + && !args.list + && conclusion.num_passed + conclusion.num_failed != 1 + { + eprintln!( + "testkit: expected to run exactly one test, ran {}", + conclusion.num_passed + conclusion.num_failed + ); + return ExitCode::from(ERROR_EXIT); + } + conclusion.exit_code() +} + +/// Run one test on a fresh current-thread runtime, as `#[tokio::test]` does. +/// Panics to fail it. +fn run_in_process(test: &Test) { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("start the async runtime") + .block_on((test.run)()); +} + +fn program_name(args: &[OsString]) -> String { + args.first() + .map(PathBuf::from) + .and_then(|path| { + path.file_name() + .map(|name| name.to_string_lossy().into_owned()) + }) + .unwrap_or_else(|| "testkit".to_string()) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use clap::Parser; + + use super::{Cli, Command}; + + #[test] + fn cli_is_consistent() { + ::command().debug_assert(); + } + + #[test] + fn run_parses_selection_and_limits() { + let cli = Cli::try_parse_from([ + "p", + "run", + "smoke", + "life", + "--include-ignored", + "--timeout", + "1.5", + "--junit", + "out/junit.xml", + ]) + .unwrap(); + let Command::Run { + filters, + include_ignored, + timeout, + grace, + junit, + } = cli.command + else { + panic!("not run"); + }; + assert_eq!(filters, ["smoke", "life"]); + assert!(include_ignored); + assert_eq!(timeout, Duration::from_millis(1500)); + assert_eq!(grace, Duration::from_secs(10)); + assert_eq!(junit.unwrap().to_str(), Some("out/junit.xml")); + + let Command::Ready { timeout } = Cli::try_parse_from(["p", "ready"]).unwrap().command + else { + panic!("not ready"); + }; + assert_eq!(timeout, Duration::from_secs(60)); + } + + #[test] + fn rejects_bad_limits() { + for bad in ["0", "-1", "soon", "inf", "NaN"] { + assert!( + Cli::try_parse_from(["p", "run", "--timeout", bad]).is_err(), + "{bad}" + ); + assert!( + Cli::try_parse_from(["p", "ready", "--timeout", bad]).is_err(), + "{bad}" + ); + } + } +} diff --git a/tests/testkit/testkit/src/program/process.rs b/tests/testkit/testkit/src/program/process.rs new file mode 100644 index 0000000000..1959452953 --- /dev/null +++ b/tests/testkit/testkit/src/program/process.rs @@ -0,0 +1,216 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! One test process: started in its own process group, waited for with a +//! timeout, and stopped with `SIGTERM`, a grace period, then `SIGKILL`. +//! +//! The process group is killed whenever the test process ends, so a test's +//! helpers, such as CLI processes, cannot outlive it. + +use std::io; +use std::os::unix::process::CommandExt as _; +use std::process::{Child, Command, ExitStatus}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +const POLL_INTERVAL: Duration = Duration::from_millis(20); + +/// How the test process ended. +#[derive(Debug)] +pub enum Ended { + Exited(ExitStatus), + /// Stopped after the timeout. + TimedOut, + /// Stopped because the supervisor was asked to stop. + Interrupted, +} + +/// Start `command` in its own process group. +pub fn spawn(command: &mut Command) -> io::Result { + command.process_group(0).spawn() +} + +/// Wait for `child`, stopping it after `timeout` or once the supervisor has +/// been asked to stop (see [`install_stop_handlers`]). The child is reaped +/// and its group killed on every path, including errors. +pub fn wait(child: &mut Child, timeout: Duration, grace: Duration) -> io::Result { + let result = wait_or_stop(child, timeout, grace); + if result.is_err() { + let _ = reap(child); + } + result +} + +fn wait_or_stop(child: &mut Child, timeout: Duration, grace: Duration) -> io::Result { + let deadline = Instant::now() + timeout; + loop { + if has_exited(child)? { + return Ok(Ended::Exited(reap(child)?)); + } + if stop_requested() { + stop(child, grace)?; + return Ok(Ended::Interrupted); + } + if Instant::now() >= deadline { + stop(child, grace)?; + return Ok(Ended::TimedOut); + } + std::thread::sleep(POLL_INTERVAL); + } +} + +/// `SIGTERM` the group, wait up to `grace`, then `SIGKILL` it. A stop signal +/// during the grace period, or a second one in all, cuts it short. +fn stop(child: &mut Child, grace: Duration) -> io::Result { + signal_group(child, libc::SIGTERM); + let deadline = Instant::now() + grace; + let before = stop_requests(); + let cut_short = || { + let now = stop_requests(); + now > before || now >= 2 + }; + while Instant::now() < deadline && !cut_short() && !has_exited(child)? { + std::thread::sleep(POLL_INTERVAL); + } + reap(child) +} + +/// Kill whatever is left of the group, then collect the exit status. +fn reap(child: &mut Child) -> io::Result { + signal_group(child, libc::SIGKILL); + child.wait() +} + +static STOP_REQUESTS: AtomicUsize = AtomicUsize::new(0); + +fn stop_requests() -> usize { + STOP_REQUESTS.load(Ordering::SeqCst) +} + +/// Whether the supervisor has been asked to stop since it started. +pub fn stop_requested() -> bool { + stop_requests() > 0 +} + +fn pid(child: &Child) -> libc::pid_t { + libc::pid_t::try_from(child.id()).expect("pid fits in pid_t") +} + +/// Whether the child has exited, without reaping it. While it is unreaped +/// its PID, and so its process group ID, cannot be reused, so signalling the +/// group cannot reach an unrelated process. +#[cfg(target_os = "linux")] +fn has_exited(child: &mut Child) -> io::Result { + // SAFETY: `info` is a valid out-pointer, and `WNOWAIT` leaves the child + // for `Child::wait` to reap. + unsafe { + let mut info: libc::siginfo_t = std::mem::zeroed(); + let result = libc::waitid( + libc::P_PID, + child.id(), + &raw mut info, + libc::WEXITED | libc::WNOHANG | libc::WNOWAIT, + ); + if result == -1 { + return Err(io::Error::last_os_error()); + } + Ok(info.si_pid() != 0) + } +} + +/// Whether the child has exited, reaping it. `waitid` with `WNOWAIT` is not +/// reliable outside Linux (see golang/go#19314), so the group is killed just +/// after the leader is reaped. Its ID could in principle be reused in that +/// window, though a new process would have to get the same PID and lead a +/// group. +#[cfg(not(target_os = "linux"))] +fn has_exited(child: &mut Child) -> io::Result { + Ok(child.try_wait()?.is_some()) +} + +fn signal_group(child: &Child, signal: libc::c_int) { + // SAFETY: plain syscall to the child's own group (see `spawn`). ESRCH, an + // empty group, is fine. + unsafe { + libc::kill(-pid(child), signal); + } +} + +extern "C" fn on_stop(_signal: libc::c_int) { + STOP_REQUESTS.fetch_add(1, Ordering::SeqCst); +} + +/// Count `SIGINT`, `SIGTERM`, and `SIGHUP` as stop requests instead of dying. +/// Tests run in their own process group, so a Ctrl-C at the terminal reaches +/// only the supervisor, which must stop the test. +pub fn install_stop_handlers() -> io::Result<()> { + for signal in [libc::SIGINT, libc::SIGTERM, libc::SIGHUP] { + // SAFETY: the handler only touches an atomic, which is + // async-signal-safe. + unsafe { + let mut action: libc::sigaction = std::mem::zeroed(); + action.sa_sigaction = on_stop as *const () as libc::sighandler_t; + action.sa_flags = libc::SA_RESTART; + libc::sigemptyset(&raw mut action.sa_mask); + if libc::sigaction(signal, &raw const action, std::ptr::null_mut()) == -1 { + return Err(io::Error::last_os_error()); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::process::{Command, Stdio}; + use std::time::{Duration, Instant}; + + use super::{Ended, spawn, wait}; + + fn sh(script: &str) -> Command { + let mut command = Command::new("sh"); + command.args(["-c", script]).stdin(Stdio::null()); + command + } + + #[test] + fn exit_status_is_kept() { + let mut child = spawn(&mut sh("exit 3")).unwrap(); + let ended = wait(&mut child, Duration::from_secs(10), Duration::from_secs(1)).unwrap(); + assert!( + matches!(ended, Ended::Exited(status) if status.code() == Some(3)), + "{ended:?}" + ); + } + + #[test] + fn timeout_terminates_the_test() { + let started = Instant::now(); + let mut child = spawn(&mut sh("sleep 30")).unwrap(); + let ended = wait( + &mut child, + Duration::from_millis(200), + Duration::from_secs(10), + ) + .unwrap(); + assert!(matches!(ended, Ended::TimedOut), "{ended:?}"); + // `sleep` dies on SIGTERM, well inside the grace period. + assert!(started.elapsed() < Duration::from_secs(5)); + } + + #[test] + fn a_test_ignoring_sigterm_is_killed_after_the_grace_period() { + let started = Instant::now(); + let mut child = spawn(&mut sh("trap '' TERM; sleep 30")).unwrap(); + let ended = wait( + &mut child, + Duration::from_millis(200), + Duration::from_millis(500), + ) + .unwrap(); + assert!(matches!(ended, Ended::TimedOut), "{ended:?}"); + let elapsed = started.elapsed(); + assert!(elapsed >= Duration::from_millis(700), "{elapsed:?}"); + assert!(elapsed < Duration::from_secs(5), "{elapsed:?}"); + } +} diff --git a/tests/testkit/testkit/src/program/supervisor.rs b/tests/testkit/testkit/src/program/supervisor.rs new file mode 100644 index 0000000000..6dced12ae1 --- /dev/null +++ b/tests/testkit/testkit/src/program/supervisor.rs @@ -0,0 +1,334 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `run`: run the selected tests one at a time, each in a new process. +//! +//! The program starts itself once per test as ` --exact --nocapture`, +//! the same interface nextest uses, in a new process group. A test that runs +//! past its timeout gets `SIGTERM`, a grace period, then `SIGKILL`. Output is +//! shown as it comes and kept for the `JUnit` report. +//! +//! A test stopped this way does not run its cleanup: testkit tests do not +//! handle `SIGTERM` yet, so sandboxes named `test--*` can be left +//! behind. + +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; +use std::process::{Command, ExitCode, ExitStatus, Stdio}; +use std::time::{Duration, Instant}; + +use super::capture::Capture; +use super::process::{self, Ended}; +use super::{ERROR_EXIT, FAILED_EXIT, SUPERVISED_ENV, junit}; +use crate::registry::{Disposition, Selection, Test}; + +/// libtest's exit code when a test failed. +const LIBTEST_FAILED: i32 = 101; + +pub struct Options { + pub selection: Selection, + pub timeout: Duration, + pub grace: Duration, + pub junit: Option, +} + +/// One test's result. +pub struct TestRun<'a> { + pub test: &'a Test, + pub outcome: Outcome, + /// `None` for a test that did not run. + pub duration: Option, + pub stdout: Vec, + pub stderr: Vec, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Outcome { + Passed, + /// Ignored and not asked for. + Skipped, + Failed(String), + TimedOut(String), + /// The test could not be run. + Error(String), + /// Stopped because the run was interrupted. + Interrupted, +} + +impl Outcome { + fn label(&self) -> &'static str { + match self { + Self::Passed => "PASS", + Self::Skipped => "SKIP", + Self::Failed(_) => "FAIL", + Self::TimedOut(_) => "TIMEOUT", + Self::Error(_) | Self::Interrupted => "ERROR", + } + } + + fn message(&self) -> Option<&str> { + match self { + Self::Passed | Self::Skipped => None, + Self::Failed(message) | Self::TimedOut(message) | Self::Error(message) => Some(message), + Self::Interrupted => Some("interrupted"), + } + } +} + +pub fn run(program: &str, exe: &Path, tests: &[Test], options: &Options) -> ExitCode { + let selected = options.selection.apply(tests); + let to_run = selected + .iter() + .filter(|(_, disposition)| *disposition == Disposition::Run) + .count(); + if to_run == 0 { + eprintln!("{program}: no tests to run; `{program} list` shows the tests"); + return ExitCode::from(ERROR_EXIT); + } + if let Err(error) = process::install_stop_handlers() { + eprintln!("{program}: install signal handlers: {error}"); + return ExitCode::from(ERROR_EXIT); + } + let skipped = selected.len() - to_run; + eprintln!( + "{program}: running {to_run} test{} one at a time{}", + plural(to_run), + if skipped > 0 { + format!(", skipping {skipped} ignored") + } else { + String::new() + } + ); + + let started = Instant::now(); + let selected_count = selected.len(); + let mut runs = Vec::with_capacity(selected_count); + let mut position = 0; + for (test, disposition) in selected { + // A stop signal that arrives between tests still ends the run. + if process::stop_requested() { + break; + } + if disposition == Disposition::Skip { + runs.push(TestRun { + test, + outcome: Outcome::Skipped, + duration: None, + stdout: Vec::new(), + stderr: Vec::new(), + }); + continue; + } + position += 1; + let name = test.name(); + eprintln!("==> [{position}/{to_run}] {name}"); + let run = run_one(exe, test, options); + let elapsed = run.duration.unwrap_or_default(); + match run.outcome.message() { + None => eprintln!("<== {} {name} ({elapsed:.1?})", run.outcome.label()), + Some(message) => eprintln!( + "<== {} {name} ({elapsed:.1?}): {message}", + run.outcome.label() + ), + } + runs.push(run); + } + + let not_run = selected_count - runs.len(); + let mut status = summarize(program, &runs, not_run, started.elapsed()); + if let Some(path) = &options.junit { + match junit::write(path, &junit::report(program, &runs)) { + Ok(()) => eprintln!("{program}: wrote {}", path.display()), + Err(error) => { + eprintln!("{program}: write {}: {error}", path.display()); + status = ERROR_EXIT; + } + } + } + ExitCode::from(status) +} + +fn run_one<'a>(exe: &Path, test: &'a Test, options: &Options) -> TestRun<'a> { + let name = test.name(); + let mut command = Command::new(exe); + command + .args([name.as_str(), "--exact", "--nocapture"]) + .env(SUPERVISED_ENV, "1") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if test.ignored { + command.arg("--ignored"); + } + + let started = Instant::now(); + let mut child = match process::spawn(&mut command) { + Ok(child) => child, + Err(error) => { + return TestRun { + test, + outcome: Outcome::Error(format!("start {}: {error}", exe.display())), + duration: Some(started.elapsed()), + stdout: Vec::new(), + stderr: Vec::new(), + }; + } + }; + let stdout = child + .stdout + .take() + .map(|stream| Capture::start(stream, std::io::stdout())); + let stderr = child + .stderr + .take() + .map(|stream| Capture::start(stream, std::io::stderr())); + let ended = process::wait(&mut child, options.timeout, options.grace); + let duration = started.elapsed(); + let outcome = match ended { + Ok(ended) => outcome(ended, options.timeout), + Err(error) => Outcome::Error(format!("wait for the test process: {error}")), + }; + TestRun { + test, + outcome, + duration: Some(duration), + stdout: stdout.map(Capture::finish).unwrap_or_default(), + stderr: stderr.map(Capture::finish).unwrap_or_default(), + } +} + +fn outcome(ended: Ended, timeout: Duration) -> Outcome { + match ended { + Ended::Exited(status) => exited(status), + Ended::TimedOut => Outcome::TimedOut(format!( + "timed out after {}; stopped without cleanup", + seconds(timeout) + )), + Ended::Interrupted => Outcome::Interrupted, + } +} + +fn exited(status: ExitStatus) -> Outcome { + match status.code() { + Some(0) => Outcome::Passed, + Some(LIBTEST_FAILED) => Outcome::Failed("test failed".to_string()), + Some(code) if code == i32::from(ERROR_EXIT) => { + Outcome::Error("the test process could not run the test; see its output".to_string()) + } + Some(code) => Outcome::Failed(format!("test process exited with code {code}")), + None => Outcome::Failed(format!("test process ended by {status}")), + } +} + +/// Print the totals and the tests that did not pass; return the exit code: +/// 1 if a test failed, 2 if one could not run or the run was interrupted +/// before `not_run` tests. +fn summarize(program: &str, runs: &[TestRun<'_>], not_run: usize, elapsed: Duration) -> u8 { + let count = + |predicate: fn(&Outcome) -> bool| runs.iter().filter(|run| predicate(&run.outcome)).count(); + let passed = count(|outcome| matches!(outcome, Outcome::Passed)); + let failed = count(|outcome| matches!(outcome, Outcome::Failed(_) | Outcome::TimedOut(_))); + let errors = count(|outcome| matches!(outcome, Outcome::Error(_) | Outcome::Interrupted)); + let skipped = count(|outcome| matches!(outcome, Outcome::Skipped)); + let mut summary = format!("{program}: {passed} passed, {failed} failed"); + if errors > 0 { + let _ = write!(summary, ", {errors} could not run"); + } + if skipped > 0 { + let _ = write!(summary, ", {skipped} skipped"); + } + let _ = write!(summary, " in {elapsed:.1?}"); + eprintln!("{summary}"); + for run in runs { + if let Some(message) = run.outcome.message() { + eprintln!(" {} {}: {message}", run.outcome.label(), run.test.name()); + } + } + if not_run > 0 { + eprintln!( + "{program}: interrupted; {not_run} later test{} did not run", + plural(not_run) + ); + } + if errors > 0 || not_run > 0 { + ERROR_EXIT + } else if failed > 0 { + FAILED_EXIT + } else { + 0 + } +} + +fn seconds(duration: Duration) -> String { + format!("{}s", duration.as_secs_f64()) +} + +fn plural(count: usize) -> &'static str { + if count == 1 { "" } else { "s" } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::{Outcome, TestRun, summarize}; + use crate::registry::Test; + use crate::registry::tests::test; + + const TEST: Test = test("suite::m", "t", false); + + fn runs(outcomes: Vec) -> Vec> { + outcomes + .into_iter() + .map(|outcome| TestRun { + test: &TEST, + outcome, + duration: None, + stdout: Vec::new(), + stderr: Vec::new(), + }) + .collect() + } + + #[test] + fn exit_code_separates_failures_from_runs_that_could_not_complete() { + let code = |outcomes| summarize("p", &runs(outcomes), 0, Duration::ZERO); + assert_eq!(code(vec![Outcome::Passed, Outcome::Skipped]), 0); + assert_eq!(code(vec![Outcome::Passed, Outcome::Failed("f".into())]), 1); + assert_eq!(code(vec![Outcome::TimedOut("t".into())]), 1); + assert_eq!( + code(vec![ + Outcome::Failed("f".into()), + Outcome::Error("e".into()) + ]), + 2 + ); + assert_eq!(code(vec![Outcome::Interrupted]), 2); + // Interrupted between tests: every test that ran passed. + assert_eq!( + summarize("p", &runs(vec![Outcome::Passed]), 1, Duration::ZERO), + 2 + ); + } + + #[test] + fn exit_statuses_map_to_outcomes() { + use std::os::unix::process::ExitStatusExt; + + use super::exited; + + let status = |code: i32| std::process::ExitStatus::from_raw(code << 8); + assert_eq!(exited(status(0)), Outcome::Passed); + assert_eq!(exited(status(101)), Outcome::Failed("test failed".into())); + assert!(matches!(exited(status(2)), Outcome::Error(_))); + assert_eq!( + exited(status(7)), + Outcome::Failed("test process exited with code 7".into()) + ); + let killed = exited(std::process::ExitStatus::from_raw(libc::SIGSEGV)); + assert!( + matches!(&killed, Outcome::Failed(message) if message.contains("signal")), + "{killed:?}" + ); + } +} diff --git a/tests/testkit/testkit/src/ready.rs b/tests/testkit/testkit/src/ready.rs new file mode 100644 index 0000000000..db36f67659 --- /dev/null +++ b/tests/testkit/testkit/src/ready.rs @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Waiting until the gateway is ready for tests. + +use std::time::Duration; + +use crate::cli::Cli; +use crate::gateway::Gateway; + +/// Wait until the gateway selected as for tests is healthy and the CLI +/// reports it connected, logging to stderr as `program`. Returns whether it +/// is ready; if not, the last error has been logged. +pub async fn check(program: &str, timeout: Duration) -> bool { + let result = async { + let gateway = Gateway::from_env()?; + eprintln!("{program}: {}", gateway.describe()); + let health = gateway.wait_ready(timeout).await?; + Cli::from_env(&gateway).check_connected().await?; + crate::Result::Ok(health) + } + .await; + match result { + Ok(health) => { + eprintln!( + "{program}: gateway {} is healthy and the CLI reaches it", + health.version + ); + true + } + Err(error) => { + eprintln!("{program}: {error}"); + false + } + } +} diff --git a/tests/testkit/testkit/src/registry.rs b/tests/testkit/testkit/src/registry.rs new file mode 100644 index 0000000000..aceaed673f --- /dev/null +++ b/tests/testkit/testkit/src/registry.rs @@ -0,0 +1,208 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The tests linked into a program, registered by `#[testkit::test]`. + +use std::collections::BTreeSet; +use std::future::Future; +use std::pin::Pin; + +use crate::run::TestInfo; + +/// Every `#[testkit::test]` linked into the program. +#[linkme::distributed_slice] +pub static TESTS: [Test]; + +/// A registered test. +#[derive(Debug)] +pub struct Test { + pub info: TestInfo, + /// The directory under `tests/suites/` holding the test's crate, if any. + pub family: Option<&'static str>, + /// `module_path!()` where the test is defined. + pub module_path: &'static str, + pub ignored: bool, + /// The reason given in `#[ignore = "..."]`. + pub ignore_reason: Option<&'static str>, + /// Runs the test through [`crate::run`]; panics to fail it. + pub run: fn() -> Pin>>, +} + +impl Test { + /// The test's name, as libtest and nextest name it: the module path + /// without the crate, then the function name. + pub fn name(&self) -> String { + match self.module_path.split_once("::") { + Some((_crate, module)) => format!("{module}::{}", self.info.name), + None => self.info.name.to_string(), + } + } + + pub fn family(&self) -> &'static str { + self.family.unwrap_or("-") + } +} + +/// Names a test cannot have at the top level: the program would read them as +/// its own subcommands, so nextest could not run the test by name. +pub const RESERVED_NAMES: &[&str] = &["help", "list", "ready", "run"]; + +/// Check the tests can be told apart by name. +pub fn validate(tests: &[Test]) -> Result<(), String> { + let mut names = BTreeSet::new(); + for test in tests { + let name = test.name(); + if RESERVED_NAMES.contains(&name.as_str()) { + return Err(format!( + "test {name} has the name of a subcommand; rename it or move it into a module" + )); + } + if !names.insert(name.clone()) { + return Err(format!( + "two tests are named {name}; test names must be unique in a program" + )); + } + } + Ok(()) +} + +/// Which tests to run. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Selection { + /// Run tests whose name contains any of these; all tests if empty. + pub filters: Vec, + /// Run `#[ignore]`d tests too, instead of reporting them as skipped. + pub include_ignored: bool, +} + +/// What a selection does with one test. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Disposition { + Run, + /// Selected, but ignored and not asked for. + Skip, +} + +impl Selection { + /// The tests this selects, in name order, with what to do with each. + pub fn apply<'a>(&self, tests: &'a [Test]) -> Vec<(&'a Test, Disposition)> { + let mut selected: Vec<_> = tests + .iter() + .filter_map(|test| { + self.disposition(test) + .map(|disposition| (test, disposition)) + }) + .collect(); + selected.sort_by_cached_key(|(test, _)| test.name()); + selected + } + + fn disposition(&self, test: &Test) -> Option { + let name = test.name(); + if !self.filters.is_empty() + && !self + .filters + .iter() + .any(|filter| name.contains(filter.as_str())) + { + return None; + } + if test.ignored && !self.include_ignored { + Some(Disposition::Skip) + } else { + Some(Disposition::Run) + } + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::{Disposition, Selection, Test, validate}; + use crate::run::TestInfo; + + pub(crate) const fn test(module_path: &'static str, name: &'static str, ignored: bool) -> Test { + Test { + info: TestInfo { + name, + description: "A test.", + }, + family: Some("conformance"), + module_path, + ignored, + ignore_reason: None, + run: || Box::pin(async {}), + } + } + + const TESTS: &[Test] = &[ + test("sandbox::smoke", "create_exec_delete", false), + test("sandbox::lifecycle", "stop_start", false), + test("sandbox::lifecycle", "restart_later", true), + test("smoke", "top_level", false), + ]; + + fn names(selection: &Selection) -> Vec<(String, Disposition)> { + selection + .apply(TESTS) + .into_iter() + .map(|(test, disposition)| (test.name(), disposition)) + .collect() + } + + #[test] + fn names_follow_libtest() { + assert_eq!(TESTS[0].name(), "smoke::create_exec_delete"); + assert_eq!(TESTS[3].name(), "top_level"); + } + + #[test] + fn selection_follows_libtest() { + use Disposition::{Run, Skip}; + let all = Selection::default(); + assert_eq!( + names(&all), + [ + ("lifecycle::restart_later".into(), Skip), + ("lifecycle::stop_start".into(), Run), + ("smoke::create_exec_delete".into(), Run), + ("top_level".into(), Run), + ] + ); + + let filtered = Selection { + filters: vec!["stop".into(), "top".into()], + ..Selection::default() + }; + assert_eq!( + names(&filtered), + [ + ("lifecycle::stop_start".into(), Run), + ("top_level".into(), Run) + ] + ); + + let include = Selection { + filters: vec!["lifecycle".into()], + include_ignored: true, + }; + assert_eq!( + names(&include), + [ + ("lifecycle::restart_later".into(), Run), + ("lifecycle::stop_start".into(), Run) + ] + ); + } + + #[test] + fn validate_rejects_ambiguous_names() { + assert!(validate(TESTS).is_ok()); + let duplicate = [test("a::m", "t", false), test("b::m", "t", false)]; + let error = validate(&duplicate).unwrap_err(); + assert!(error.contains("two tests are named m::t"), "{error}"); + let reserved = [test("suite", "run", false)]; + let error = validate(&reserved).unwrap_err(); + assert!(error.contains("name of a subcommand"), "{error}"); + assert!(validate(&[test("suite::m", "run", false)]).is_ok()); + } +} diff --git a/tests/testkit/testkit/tests/attribute.rs b/tests/testkit/testkit/tests/attribute.rs new file mode 100644 index 0000000000..1f49adea40 --- /dev/null +++ b/tests/testkit/testkit/tests/attribute.rs @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! What `#[testkit::test]` expands to, checked by compiling tests with it in +//! an ordinary libtest target. The tests need a gateway, so they are ignored; +//! the check compares the registry with what libtest lists. + +use std::collections::BTreeSet; +use std::process::Command; + +/// Ignored with a reason. +#[testkit::test] +#[ignore = "needs a gateway"] +async fn ignored_with_reason(_cx: &mut testkit::Context) -> testkit::Result { + Ok(()) +} + +/// Lint attributes reach the body: `cx` is unused. +#[testkit::test] +#[ignore] +#[allow(unused_variables)] +async fn lint_allowed(cx: &mut testkit::Context) -> testkit::Result { + Ok(()) +} + +/// Compiled out, so neither a test nor registered. +#[testkit::test] +#[cfg(any())] +async fn compiled_out(_cx: &mut testkit::Context) -> testkit::Result { + Ok(()) +} + +mod nested { + /// Named with its module. + #[testkit::test] + #[ignore] + async fn in_a_module(_cx: &mut testkit::Context) -> testkit::Result { + Ok(()) + } +} + +/// The names libtest lists for this binary, with `extra` arguments. +fn libtest_list(extra: &[&str]) -> BTreeSet { + let output = Command::new(std::env::current_exe().expect("current exe")) + .args(["--list", "--format", "terse"]) + .args(extra) + .output() + .expect("list the tests"); + assert!(output.status.success(), "{output:?}"); + String::from_utf8(output.stdout) + .expect("utf-8") + .lines() + .filter_map(|line| line.strip_suffix(": test")) + .filter(|name| *name != "registry_matches_libtest") + .map(str::to_string) + .collect() +} + +#[test] +fn registry_matches_libtest() { + let registered: Vec<_> = testkit::__private::TESTS + .iter() + .map(|test| { + ( + test.name(), + test.module_path, + test.family, + test.ignored, + test.ignore_reason, + test.info.description, + ) + }) + .collect::>() + .into_iter() + .collect(); + assert_eq!( + registered, + [ + ( + "ignored_with_reason".to_string(), + "attribute", + None, + true, + Some("needs a gateway"), + "Ignored with a reason." + ), + ( + "lint_allowed".to_string(), + "attribute", + None, + true, + None, + "Lint attributes reach the body: `cx` is unused." + ), + ( + "nested::in_a_module".to_string(), + "attribute::nested", + None, + true, + None, + "Named with its module." + ), + ] + ); + + // libtest runs and ignores the same tests, under the same names. + let names = |ignored_only: bool| -> BTreeSet { + testkit::__private::TESTS + .iter() + .filter(|test| test.ignored || !ignored_only) + .map(testkit::Test::name) + .collect() + }; + assert_eq!(libtest_list(&[]), names(false)); + assert_eq!(libtest_list(&["--ignored"]), names(true)); +} diff --git a/tests/testkit/testkit/tests/program.rs b/tests/testkit/testkit/tests/program.rs new file mode 100644 index 0000000000..c414e8dd27 --- /dev/null +++ b/tests/testkit/testkit/tests/program.rs @@ -0,0 +1,527 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Checks of `testkit::main()`, as a test program. +//! +//! This binary is three programs, chosen by `TESTKIT_FIXTURE`: +//! +//! - unset: the checks below, run by `cargo test` and nextest through the +//! program's libtest interface; +//! - `fixtures`: tests that pass, fail, hang, ignore `SIGTERM`, or leave +//! processes behind, which the checks run through the program's +//! subcommands; +//! - `registered`: `testkit::main()` over the `#[testkit::test]`s below; +//! - `empty`: a program with no tests. + +#[cfg(unix)] +fn main() -> std::process::ExitCode { + match std::env::var("TESTKIT_FIXTURE").as_deref() { + Ok("fixtures") => testkit::program::main_with(&unix::FIXTURES), + Ok("registered") => testkit::main(), + Ok("empty") => testkit::program::main_with(&[]), + _ => testkit::program::main_with(&unix::CHECKS), + } +} + +#[cfg(not(unix))] +fn main() {} + +/// Needs a gateway. +#[testkit::test] +async fn registered_needs_a_gateway(cx: &mut testkit::Context) -> testkit::Result { + cx.cli(["status"]).await?.success()?; + Ok(()) +} + +/// Ignored until later. +#[testkit::test] +#[ignore = "https://github.com/NVIDIA/OpenShell/issues/1"] +async fn registered_ignored(_cx: &mut testkit::Context) -> testkit::Result { + Ok(()) +} + +#[cfg(unix)] +mod unix { + use std::future::Future; + use std::os::unix::process::CommandExt as _; + use std::path::{Path, PathBuf}; + use std::pin::Pin; + use std::process::{Child, Command, Output, Stdio}; + use std::time::{Duration, Instant}; + + use testkit::{Test, TestInfo}; + + const PID_FILE_ENV: &str = "FIXTURE_PID_FILE"; + + type Run = fn() -> Pin>>; + + const fn test(module_path: &'static str, name: &'static str, run: Run) -> Test { + Test { + info: TestInfo { + name, + description: "A fixture.", + }, + family: Some("fixtures"), + module_path, + ignored: false, + ignore_reason: None, + run, + } + } + + const fn fixture(name: &'static str, run: Run) -> Test { + test("program::fixture", name, run) + } + + const fn check(name: &'static str, run: Run) -> Test { + test("program::check", name, run) + } + + fn spawn_helper_and_record_its_pid() { + // A helper in the test's process group, like a CLI process. It is + // never waited on: the supervisor must kill it. + #[allow(clippy::zombie_processes)] + let helper = Command::new("sleep") + .arg("300") + .spawn() + .expect("spawn sleep"); + let pid_file = std::env::var(PID_FILE_ENV).expect("pid file"); + std::fs::write(pid_file, helper.id().to_string()).expect("write pid"); + } + + pub static FIXTURES: [Test; 7] = [ + fixture("passes", || { + Box::pin(async { + println!("passes: stdout"); + eprintln!("passes: stderr"); + }) + }), + fixture("fails", || Box::pin(async { panic!("fixture failure") })), + fixture("hangs", || { + Box::pin(async { + spawn_helper_and_record_its_pid(); + std::thread::sleep(Duration::from_secs(300)); + }) + }), + fixture("leaves_a_helper", || { + Box::pin(async { spawn_helper_and_record_its_pid() }) + }), + fixture("interrupts_after_exit", || { + Box::pin(async { + // Signal the supervisor once this test has exited, while it + // waits for the test's output to close. The helper leaves + // the test's process group so that it survives the test, and + // holds the output open until it has sent the signal. + let supervisor = std::os::unix::process::parent_id(); + #[allow(clippy::zombie_processes)] + let _signaller = Command::new("sh") + .args(["-c", &format!("sleep 1; kill -TERM {supervisor}")]) + .process_group(0) + .spawn() + .expect("spawn the signaller"); + }) + }), + fixture("ignores_sigterm", || { + Box::pin(async { + // SAFETY: plain syscall. + unsafe { + libc::signal(libc::SIGTERM, libc::SIG_IGN); + } + std::thread::sleep(Duration::from_secs(300)); + }) + }), + Test { + ignored: true, + ignore_reason: Some("not today"), + ..fixture("ignored", || Box::pin(async {})) + }, + ]; + + pub static CHECKS: [Test; 12] = [ + check("nextest_interface", || { + Box::pin(async { nextest_interface() }) + }), + check("list_shows_name_and_description", || { + Box::pin(async { list_shows_name_and_description() }) + }), + check("run_reports_every_outcome", || { + Box::pin(async { run_reports_every_outcome() }) + }), + check("run_exit_codes", || Box::pin(async { run_exit_codes() })), + check("a_program_without_tests_fails", || { + Box::pin(async { a_program_without_tests_fails() }) + }), + check("timeout_kills_the_process_group", || { + Box::pin(async { timeout_kills_the_process_group() }) + }), + check("sigterm_is_followed_by_sigkill", || { + Box::pin(async { sigterm_is_followed_by_sigkill() }) + }), + check("interrupt_stops_the_test_and_the_run", || { + Box::pin(async { interrupt_stops_the_test_and_the_run() }) + }), + check("interrupt_between_tests_ends_the_run", || { + Box::pin(async { interrupt_between_tests_ends_the_run() }) + }), + check("a_passing_test_leaves_nothing_behind", || { + Box::pin(async { a_passing_test_leaves_nothing_behind() }) + }), + check("registered_tests_run_through_testkit", || { + Box::pin(async { registered_tests_run_through_testkit() }) + }), + check("ready_fails_without_a_gateway", || { + Box::pin(async { ready_fails_without_a_gateway() }) + }), + ]; + + fn program(fixture: &str, args: &[&str]) -> Command { + let mut command = Command::new(std::env::current_exe().expect("current exe")); + command + .args(args) + .env("TESTKIT_FIXTURE", fixture) + .env_remove("TESTKIT_SUPERVISED") + .stdin(Stdio::null()); + command + } + + fn output(mut command: Command) -> Output { + let output = command.output().expect("run the program"); + eprintln!( + "{command:?}: {}\n--- stdout\n{}--- stderr\n{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + output + } + + fn stdout(output: &Output) -> &str { + std::str::from_utf8(&output.stdout).expect("utf-8") + } + + fn stderr(output: &Output) -> &str { + std::str::from_utf8(&output.stderr).expect("utf-8") + } + + fn junit(dir: &Path) -> (PathBuf, String) { + let path = dir.join("results").join("junit.xml"); + (path.clone(), path.to_str().expect("utf-8 path").to_string()) + } + + /// nextest lists with `--list --format terse [--ignored]` and runs each + /// test with ` --exact --nocapture`. + fn nextest_interface() { + let list = output(program("fixtures", &["--list", "--format", "terse"])); + assert!(list.status.success()); + assert_eq!( + stdout(&list), + "fixture::fails: test\nfixture::hangs: test\nfixture::ignored: test\n\ + fixture::ignores_sigterm: test\nfixture::interrupts_after_exit: test\n\ + fixture::leaves_a_helper: test\nfixture::passes: test\n" + ); + let ignored = output(program( + "fixtures", + &["--list", "--format", "terse", "--ignored"], + )); + assert_eq!(stdout(&ignored), "fixture::ignored: test\n"); + + let pass = output(program( + "fixtures", + &["fixture::passes", "--exact", "--nocapture"], + )); + assert!(pass.status.success()); + assert!(stdout(&pass).contains("passes: stdout")); + assert!(stdout(&pass).contains("1 passed")); + + let fail = output(program( + "fixtures", + &["fixture::fails", "--exact", "--nocapture"], + )); + assert_eq!(fail.status.code(), Some(101)); + assert!(stderr(&fail).contains("fixture failure")); + + // An ignored test runs only when asked for. + let skip = output(program("fixtures", &["fixture::ignored", "--exact"])); + assert!(stdout(&skip).contains("1 ignored")); + let run = output(program( + "fixtures", + &["fixture::ignored", "--exact", "--ignored"], + )); + assert!(stdout(&run).contains("1 passed")); + } + + fn list_shows_name_and_description() { + let list = output(program("registered", &["list"])); + assert!(list.status.success()); + let lines: Vec<_> = stdout(&list).lines().collect(); + assert_eq!(lines.len(), 2, "{lines:?}"); + assert_eq!( + lines[0], + "registered_ignored Ignored until later. \ + [ignored: https://github.com/NVIDIA/OpenShell/issues/1]" + ); + assert_eq!(lines[1], "registered_needs_a_gateway Needs a gateway."); + + let filtered = output(program("fixtures", &["list", "pass"])); + assert_eq!(stdout(&filtered), "fixture::passes A fixture.\n"); + } + + fn run_reports_every_outcome() { + let dir = tempfile::tempdir().expect("temp dir"); + let (path, junit_arg) = junit(dir.path()); + let run = output(program( + "fixtures", + &["run", "passes", "fails", "ignored", "--junit", &junit_arg], + )); + assert_eq!(run.status.code(), Some(1)); + // Test output is passed through. + assert!(stdout(&run).contains("passes: stdout")); + assert!(stderr(&run).contains("passes: stderr")); + assert!(stderr(&run).contains("1 passed, 1 failed, 1 skipped")); + assert!(stderr(&run).contains("FAIL fixture::fails: test failed")); + + let xml = std::fs::read_to_string(path).expect("junit report"); + for expected in [ + r#""#, + r#""#, + r#""#, + "", + "passes: stdout", + "fixture failure", + ] { + assert!(xml.contains(expected), "missing {expected} in\n{xml}"); + } + } + + fn run_exit_codes() { + let pass = output(program("fixtures", &["run", "passes"])); + assert_eq!(pass.status.code(), Some(0)); + let ignored = output(program( + "fixtures", + &["run", "ignored", "--include-ignored"], + )); + assert_eq!(ignored.status.code(), Some(0)); + assert!(stderr(&ignored).contains("1 passed, 0 failed")); + // Selecting nothing to run is not a pass. + for args in [&["run", "nothing-matches"][..], &["run", "ignored"]] { + let none = output(program("fixtures", args)); + assert_eq!(none.status.code(), Some(2), "{args:?}"); + assert!(stderr(&none).contains("no tests to run")); + } + let usage = output(program("fixtures", &["run", "--timeout", "0"])); + assert_eq!(usage.status.code(), Some(2)); + } + + /// A program whose suites were not linked fails rather than passing with + /// no tests, under every interface. + fn a_program_without_tests_fails() { + for args in [ + &["list"][..], + &["run"], + &["--list", "--format", "terse"], + &[], + ] { + let empty = output(program("empty", args)); + assert_eq!(empty.status.code(), Some(2), "{args:?}"); + assert!( + stderr(&empty).contains("no tests are linked into this program"), + "{args:?}" + ); + } + } + + fn is_alive(pid: libc::pid_t) -> bool { + // SAFETY: signal 0 only checks the process exists. + unsafe { libc::kill(pid, 0) == 0 } + } + + /// Wait for a killed helper to go: once its parent is gone, init reaps it. + fn assert_gone(pid: libc::pid_t) { + let deadline = Instant::now() + Duration::from_secs(10); + while is_alive(pid) { + assert!(Instant::now() < deadline, "helper {pid} outlived its test"); + std::thread::sleep(Duration::from_millis(50)); + } + } + + fn read_pid(path: &Path) -> libc::pid_t { + let deadline = Instant::now() + Duration::from_secs(30); + loop { + if let Ok(pid) = std::fs::read_to_string(path) + && let Ok(pid) = pid.trim().parse() + { + return pid; + } + assert!(Instant::now() < deadline, "the hanging test never started"); + std::thread::sleep(Duration::from_millis(20)); + } + } + + fn timeout_kills_the_process_group() { + let dir = tempfile::tempdir().expect("temp dir"); + let pid_file = dir.path().join("pid"); + let (path, junit_arg) = junit(dir.path()); + let mut command = program( + "fixtures", + &[ + "run", + "hangs", + "--timeout", + "1", + "--grace", + "5", + "--junit", + &junit_arg, + ], + ); + command.env(PID_FILE_ENV, &pid_file); + let started = Instant::now(); + let run = output(command); + assert_eq!(run.status.code(), Some(1)); + // `SIGTERM` stops the test well inside the grace period. + assert!( + started.elapsed() < Duration::from_secs(5), + "{:?}", + started.elapsed() + ); + assert!(stderr(&run).contains("TIMEOUT fixture::hangs: timed out after 1s")); + assert_gone(read_pid(&pid_file)); + let xml = std::fs::read_to_string(path).expect("junit report"); + assert!(xml.contains(r#"type="timeout""#), "{xml}"); + } + + fn sigterm_is_followed_by_sigkill() { + let started = Instant::now(); + let run = output(program( + "fixtures", + &[ + "run", + "ignores_sigterm", + "--timeout", + "0.2", + "--grace", + "0.5", + ], + )); + assert_eq!(run.status.code(), Some(1)); + let elapsed = started.elapsed(); + assert!(elapsed >= Duration::from_millis(700), "{elapsed:?}"); + assert!(elapsed < Duration::from_secs(10), "{elapsed:?}"); + assert!(stderr(&run).contains("TIMEOUT fixture::ignores_sigterm")); + } + + fn interrupt_stops_the_test_and_the_run() { + let dir = tempfile::tempdir().expect("temp dir"); + let pid_file = dir.path().join("pid"); + let (path, junit_arg) = junit(dir.path()); + let mut command = program( + "fixtures", + &[ + "run", "hangs", "passes", "--grace", "5", "--junit", &junit_arg, + ], + ); + command + .env(PID_FILE_ENV, &pid_file) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let child: Child = command.spawn().expect("start the run"); + let helper = read_pid(&pid_file); + // SAFETY: plain syscall to our own child, not yet reaped. + unsafe { + libc::kill( + libc::pid_t::try_from(child.id()).expect("pid"), + libc::SIGTERM, + ); + } + let run = child.wait_with_output().expect("wait for the run"); + eprintln!("{}", String::from_utf8_lossy(&run.stderr)); + assert_eq!(run.status.code(), Some(2)); + assert!(stderr(&run).contains("ERROR fixture::hangs: interrupted")); + assert_gone(helper); + let xml = std::fs::read_to_string(path).expect("junit report"); + assert!(xml.contains(r#"type="interrupted""#), "{xml}"); + assert!(!xml.contains("fixture::passes"), "{xml}"); + } + + fn interrupt_between_tests_ends_the_run() { + let dir = tempfile::tempdir().expect("temp dir"); + let (path, junit_arg) = junit(dir.path()); + let run = output(program( + "fixtures", + &[ + "run", + "interrupts_after_exit", + "passes", + "--junit", + &junit_arg, + ], + )); + assert_eq!(run.status.code(), Some(2)); + assert!(stderr(&run).contains("<== PASS fixture::interrupts_after_exit")); + assert!(stderr(&run).contains("interrupted; 1 later test did not run")); + assert!(!stdout(&run).contains("passes: stdout")); + let xml = std::fs::read_to_string(path).expect("junit report"); + assert!(!xml.contains("fixture::passes"), "{xml}"); + } + + fn a_passing_test_leaves_nothing_behind() { + let dir = tempfile::tempdir().expect("temp dir"); + let pid_file = dir.path().join("pid"); + let mut command = program("fixtures", &["run", "leaves_a_helper"]); + command.env(PID_FILE_ENV, &pid_file); + let run = output(command); + assert_eq!(run.status.code(), Some(0)); + assert_gone(read_pid(&pid_file)); + } + + /// Point the gateway lookup at a closed port, away from any registered + /// gateways. + fn without_gateway(mut command: Command, config: &Path) -> Command { + let closed = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); + let endpoint = format!("http://{}", closed.local_addr().expect("address")); + drop(closed); + command + .env("XDG_CONFIG_HOME", config) + .env("OPENSHELL_GATEWAY_ENDPOINT", endpoint) + .env_remove("OPENSHELL_GATEWAY") + .env("OPENSHELL_BIN", config.join("no-openshell")); + command + } + + fn registered_tests_run_through_testkit() { + let dir = tempfile::tempdir().expect("temp dir"); + let (path, junit_arg) = junit(dir.path()); + let run = output(without_gateway( + program("registered", &["run", "--junit", &junit_arg]), + dir.path(), + )); + assert_eq!(run.status.code(), Some(1)); + // `testkit::run` logs the description and the gateway. + assert!(stderr(&run).contains("[registered_needs_a_gateway] Needs a gateway.")); + assert!(stderr(&run).contains("1 failed, 1 skipped")); + let xml = std::fs::read_to_string(path).expect("junit report"); + assert!( + xml.contains(r#""#), + "{xml}" + ); + } + + fn ready_fails_without_a_gateway() { + let dir = tempfile::tempdir().expect("temp dir"); + let started = Instant::now(); + let ready = output(without_gateway( + program("registered", &["ready", "--timeout", "1"]), + dir.path(), + )); + assert_eq!(ready.status.code(), Some(2)); + assert!(stderr(&ready).contains("not ready after 1s")); + assert!(started.elapsed() < Duration::from_secs(10)); + } +}