feat(conformance): add the openshell-conformance program

.github/workflows/branch-checks.yml

@@ -389,6 +389,7 @@ jobs:
           cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/testkit/Cargo.toml
           cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/tmachine/Cargo.toml
           cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/runner/Cargo.toml
+          tasks/scripts/check-conformance-program.sh
 
   rust-postgres-test:
     name: Rust PostgreSQL tests

CI.md

@@ -214,7 +214,7 @@ nextest filterset:
 nix run .#openshell-test-runner -- \
   --environment ubuntu-docker-rootful --installer deb \
   --family conformance --package conformance \
-  --filter 'binary_id(=conformance::policy_advisor)'
+  --filter 'test(/^policy_advisor::/)'
 ```
 
 A local run does not replace the required PR E2E gate.

TESTING.md

@@ -181,6 +181,12 @@ by `OPENSHELL_BIN`. See
 [tests/testkit/README.md](tests/testkit/README.md) for how to write and run
 tests.
 
+The conformance crate also builds `openshell-conformance`, a program that
+runs the conformance suite against a gateway without cargo or nextest
+(`openshell-conformance list`, `ready`, `run`). It covers only the
+conformance suite. See
+[tests/suites/conformance/README.md](tests/suites/conformance/README.md).
+
 Suites live under `tests/suites/<family>/`. Each family is a Cargo workspace
 with its own lockfile. Run the `conformance` suites against a local Docker
 gateway with:
@@ -414,7 +420,9 @@ Suites:
 - Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests.
 - Conformance (`tests/suites/conformance`) - the testkit `conformance` crate:
   lifecycle, the CLI, persistent workspaces, file transfer, and the policy
-  advisor. Driver E2E runs it with nextest against the lane's gateway. The
+  advisor. Driver E2E runs it with `openshell-conformance` against the lane's
+  gateway and writes a JUnit report under `results/`; set
+  `OPENSHELL_CONFORMANCE_BIN` to use a prebuilt program. The
   installed-artifact conformance lanes run the same crate through
   `openshell-test-runner` (see above).
 - Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or

e2e/policy-advisor/README.md

@@ -69,7 +69,7 @@ test runner:
 nix run .#openshell-test-runner -- \
   --environment ubuntu-docker-rootful --installer deb \
   --family conformance --package conformance \
-  --filter 'binary_id(=conformance::policy_advisor)'
+  --filter 'test(/^policy_advisor::/)'
 ```
 
 The GitHub write test above and the regressions below still exercise distinct

e2e/rust/e2e-kubernetes.sh

@@ -79,7 +79,8 @@ run_conformance() {
     return 0
   fi
 
-  e2e_run_openshell_conformance "Kubernetes"
+  OPENSHELL_CONFORMANCE_JUNIT="${OPENSHELL_CONFORMANCE_JUNIT:-${ROOT}/results/${OPENSHELL_E2E_REPORT_NAME:-e2e-kubernetes}-conformance.xml}" \
+    e2e_run_openshell_conformance "Kubernetes"
 }
 
 # `OPENSHELL_E2E_REPORT_NAME` (default `e2e-kubernetes`) names the report

e2e/support/conformance.sh

@@ -16,30 +16,55 @@ e2e_require_openshell_bin() {
     return 2
   fi
 
-  # Cargo runs each test from its package directory, so resolve the candidate
-  # relative to the caller's working directory before passing it to the tests.
+  # Tests may run from another directory, so resolve the candidate relative
+  # to the caller's working directory before passing it to them.
   local binary_dir
   binary_dir="$(cd "$(dirname "${OPENSHELL_BIN}")" && pwd -P)" || return
   export OPENSHELL_BIN="${binary_dir}/$(basename "${OPENSHELL_BIN}")"
 }
 
+# Print the path of the openshell-conformance program: OPENSHELL_CONFORMANCE_BIN
+# if set, else a debug build from this checkout.
+e2e_openshell_conformance_bin() {
+  local root=$1
+
+  if [ -n "${OPENSHELL_CONFORMANCE_BIN:-}" ]; then
+    if [ ! -x "${OPENSHELL_CONFORMANCE_BIN}" ]; then
+      echo "ERROR: openshell-conformance is not executable: ${OPENSHELL_CONFORMANCE_BIN}" >&2
+      return 2
+    fi
+    echo "${OPENSHELL_CONFORMANCE_BIN}"
+    return
+  fi
+
+  cargo build \
+    --locked \
+    --manifest-path "${root}/tests/suites/conformance/Cargo.toml" \
+    --bin openshell-conformance >&2 || return
+  # Cargo resolves a relative CARGO_TARGET_DIR against the working directory.
+  local target_dir
+  target_dir="$(cd "${CARGO_TARGET_DIR:-${root}/tests/suites/conformance/target}" && pwd)" || return
+  echo "${target_dir}/debug/openshell-conformance"
+}
+
+# Run the conformance suite with openshell-conformance. The JUnit report goes
+# to OPENSHELL_CONFORMANCE_JUNIT, else results/conformance-<gateway>.xml.
 e2e_run_openshell_conformance() {
   local gateway_label=${1:-OpenShell}
-  local root
+  local root program junit
   root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 
   e2e_require_openshell_bin || return
+  program="$(e2e_openshell_conformance_bin "${root}")" || return
+  junit="${OPENSHELL_CONFORMANCE_JUNIT:-${root}/results/conformance-$(echo "${gateway_label}" | tr '[:upper:]' '[:lower:]').xml}"
+  mkdir -p "$(dirname "${junit}")" || return
 
-  # The tests find the gateway as the CLI does: OPENSHELL_GATEWAY_ENDPOINT,
+  # The program finds the gateway as the CLI does: OPENSHELL_GATEWAY_ENDPOINT,
   # else OPENSHELL_GATEWAY, else the active gateway. The gateway wrappers set
   # these for the gateway they started.
   echo "==> Running conformance suites against the ${gateway_label} gateway"
-  cargo nextest run \
-    --locked \
-    --manifest-path "${root}/tests/suites/conformance/Cargo.toml" \
-    --test-threads 1 \
-    --no-fail-fast \
-    --no-tests fail
+  "${program}" ready || return
+  "${program}" run --junit "${junit}"
 }
 
 if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then

tasks/scripts/check-conformance-program.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Check that `openshell-conformance list` names the same tests that nextest
+# lists for the conformance crate, so the program runs the whole suite.
+# Neither command needs a gateway.
+
+set -euo pipefail
+
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+manifest="${root}/tests/suites/conformance/Cargo.toml"
+work="$(mktemp -d)"
+trap 'rm -rf "${work}"' EXIT
+
+cargo run --quiet --locked --manifest-path "${manifest}" --bin openshell-conformance -- list \
+    | awk '{print $1}' | sort > "${work}/program"
+cargo nextest list --locked --color never --manifest-path "${manifest}" --message-format oneline \
+    | awk '{print $2}' | sort > "${work}/nextest"
+
+if ! diff -u "${work}/nextest" "${work}/program"; then
+    echo "error: openshell-conformance list differs from nextest's list (-nextest +program)" >&2
+    exit 1
+fi
+if [[ ! -s "${work}/program" ]]; then
+    echo "error: openshell-conformance lists no tests" >&2
+    exit 1
+fi
+echo "openshell-conformance lists the same $(wc -l < "${work}/program" | tr -d ' ') tests as nextest"

tasks/test.toml

@@ -127,6 +127,7 @@ run = [
   "cargo nextest run --config-file .config/nextest.toml --manifest-path tests/testkit/Cargo.toml",
   "cargo nextest run --config-file .config/nextest.toml --manifest-path tests/tmachine/Cargo.toml",
   "cargo nextest run --config-file .config/nextest.toml --manifest-path tests/runner/Cargo.toml",
+  "tasks/scripts/check-conformance-program.sh",
 ]
 run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 test-precommit native"
 hide = true

tests/runner/src/main.rs

@@ -37,7 +37,7 @@ struct Cli {
     #[arg(long = "package", short = 'p', value_name = "PACKAGE")]
     packages: Vec<String>,
     /// A nextest filterset narrowing the tests that run, e.g.
-    /// `binary(smoke)`.
+    /// `test(/^smoke::/)`.
     #[arg(long, value_name = "FILTERSET")]
     filter: Option<String>,
     /// The gateway name the installer registers with the CLI. The packaged