From 46d0b471f8785a32946e9ef472b382e484aed354 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 6 Nov 2026 15:24:42 +0100 Subject: [PATCH 1/2] ci(e2e): run the tmachine conformance rows with the prebuilt openshell-conformance The host lanes already share one openshell-conformance build per architecture, but the tmachine conformance rows each cross-built a nextest archive. Add --program to openshell-test-runner to run a prebuilt testkit program in the guest, and have Branch E2E pass the build-conformance artifact to the conformance rows of e2e-suites.yml. Signed-off-by: Evan Lezar --- .github/workflows/branch-e2e.yml | 3 +- .github/workflows/e2e-suites.yml | 36 ++++++++-- TESTING.md | 8 ++- tests/runner/src/bundle.rs | 54 ++++++++++++++- tests/runner/src/execute.rs | 112 +++++++++++++++++++++---------- tests/runner/src/main.rs | 47 ++++++++----- 6 files changed, 197 insertions(+), 63 deletions(-) diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 0f59ab2a81..fa23b89dac 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -296,7 +296,7 @@ jobs: # Run driver-independent conformance tests. conformance-integration: - needs: prepare-integration + needs: [prepare-integration, build-conformance] permissions: actions: read contents: read @@ -304,6 +304,7 @@ jobs: uses: ./.github/workflows/e2e-suites.yml with: family: conformance + prebuilt-conformance: true source-sha: ${{ needs.prepare-integration.outputs.source_sha }} integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} secrets: diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 791761befb..2fef301cae 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -7,9 +7,10 @@ # optionally, the package in it, the gateway name the installer registers, and # the runner label for the environment's architecture. `openshell-test-runner` # builds the tests, boots the environment, runs them inside it and checks the -# results. The `family` input selects the rows of one family, including the -# workspaces under it (drivers selects drivers/podman); leave it empty to run -# every row. +# results; a conformance row can run the prebuilt `openshell-conformance` +# instead (`prebuilt-conformance`). The `family` input selects the rows of one +# family, including the workspaces under it (drivers selects drivers/podman); +# leave it empty to run every row. name: E2E Suites @@ -29,6 +30,11 @@ on: description: Immutable ID of the prepared integration inputs artifact required: true type: string + prebuilt-conformance: + description: Run the conformance rows with the openshell-conformance-x86_64-unknown-linux-musl artifact of this run instead of building the suites in every job + required: false + type: boolean + default: false secrets: CACHIX_AUTH_TOKEN: description: Token used to push Nix builds, such as the test runner, to Cachix @@ -161,8 +167,18 @@ jobs: artifact-ids: ${{ inputs['integration-inputs-artifact-id'] }} path: artifacts + # The program is built once per run (see build-conformance in + # branch-e2e.yml), so the conformance rows need not each build the suites. + - name: Download prebuilt openshell-conformance + if: inputs.prebuilt-conformance && matrix.family == 'conformance' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: openshell-conformance-x86_64-unknown-linux-musl + path: artifacts/prebuilt-conformance + - name: Run the suites in this environment env: + PREBUILT_CONFORMANCE: ${{ inputs.prebuilt-conformance && matrix.family == 'conformance' }} ENVIRONMENT: ${{ matrix.environment }} INSTALLER: ${{ matrix.installer }} FAMILY: ${{ matrix.family }} @@ -172,15 +188,21 @@ jobs: # Retry dependency preparation, then run the suites once. The runner # builds the suites for the guest, so it needs no prepared archives. nix build --no-link .#openshell-test-runner || nix build --no-link .#openshell-test-runner - packages=() - if [ -n "${PACKAGE}" ]; then - packages=(--package "${PACKAGE}") + # A prebuilt program replaces the build, and so the package choice. + selection=() + if [ "${PREBUILT_CONFORMANCE}" = "true" ]; then + program="artifacts/prebuilt-conformance/openshell-conformance" + chmod +x "${program}" + "${program}" list > /dev/null + selection=(--program "${program}") + elif [ -n "${PACKAGE}" ]; then + selection=(--package "${PACKAGE}") fi nix run .#openshell-test-runner -- \ --environment "${ENVIRONMENT}" \ --installer "${INSTALLER}" \ --family "${FAMILY}" \ - "${packages[@]}" \ + "${selection[@]}" \ --gateway "${GATEWAY}" \ --results artifacts/test-results diff --git a/TESTING.md b/TESTING.md index a3936ad649..636bb12e3c 100644 --- a/TESTING.md +++ b/TESTING.md @@ -248,7 +248,13 @@ complete, for example because the environment didn't come up. It writes `artifacts/test-results/` (`--results`), even when the run timed out. `--filter ` narrows the tests with a nextest filterset, and -`--keep` leaves the VM running until you press Enter. Two runs on one host +`--keep` leaves the VM running until you press Enter. `--program ` runs a +prebuilt testkit program, such as a static `openshell-conformance` for the +guest's architecture, instead of building the family's tests; it can't be +combined with `--package` or `--filter`. The runner runs the program's `ready` +and `run` commands in the guest and still writes `junit.xml` and `nextest.log` +(the program's output) to the results directory. Branch E2E builds +`openshell-conformance` once and passes it to every conformance row this way. Two runs on one host need different `TMACHINE_SSH_PORT` values (default 2222). The installer inputs, such as the Debian package and the images, come from `artifacts/`, built by `nix run .#build-artifacts` or downloaded by CI. diff --git a/tests/runner/src/bundle.rs b/tests/runner/src/bundle.rs index 08e66738c6..4bc5da475c 100644 --- a/tests/runner/src/bundle.rs +++ b/tests/runner/src/bundle.rs @@ -8,6 +8,10 @@ //! `nextest` config with the runner's `JUnit` profile, and `testkit-ready`, //! which waits for the gateway before the tests run. The guest needs only the //! `cargo-nextest` binary to run it. +//! +//! A family that ships a prebuilt test program, such as `openshell-conformance`, +//! can run it instead: the bundle then holds that program alone, built once +//! elsewhere for the guest target, and the runner builds nothing. use std::path::{Path, PathBuf}; use std::process::Command; @@ -34,13 +38,44 @@ slow-timeout = { period = "30s", terminate-after = 4 } path = "junit.xml" "#; +/// The prebuilt program's name in a program bundle. +pub const PROGRAM_BINARY: &str = "test-program"; + +/// What a bundle runs the tests with. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Kind { + /// A nextest archive, run with `cargo-nextest`. + Nextest, + /// A prebuilt testkit program (`list`, `ready` and `run`), run directly. + Program, +} + /// The bundle directory, removed when this is dropped. -pub struct Bundle(TempDir); +pub struct Bundle(TempDir, Kind); impl Bundle { pub fn path(&self) -> &Path { self.0.path() } + + pub const fn kind(&self) -> Kind { + self.1 + } +} + +/// A bundle holding only `program`, a testkit program already built for the +/// guest target. `program` must be a static binary, as the guest's libc is +/// not known. +pub fn program(program: &Path) -> Result { + ensure!( + program.is_file(), + "--program {} is not a file", + program.display() + ); + let dir = tempfile::tempdir().context("create bundle directory")?; + std::fs::copy(program, dir.path().join(PROGRAM_BINARY)) + .with_context(|| format!("copy {}", program.display()))?; + Ok(Bundle(dir, Kind::Program)) } /// The Rust target for guests. tmachine only boots guests of the host's @@ -137,7 +172,7 @@ pub fn build(workspace: &Path, packages: &[String], target: &str, ready: &Path) std::fs::write(stage.join("nextest.toml"), NEXTEST_CONFIG)?; std::fs::copy(ready, stage.join(READY_BINARY)) .with_context(|| format!("copy {}", ready.display()))?; - Ok(Bundle(dir)) + Ok(Bundle(dir, Kind::Nextest)) } /// Copy every `Cargo.toml` under `workspace` into `stage`, keeping relative @@ -176,6 +211,21 @@ fn run(command: &mut Command) -> Result<()> { mod tests { use super::*; + #[test] + fn program_bundle_holds_only_the_program() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("openshell-conformance"); + std::fs::write(&path, b"binary").unwrap(); + let bundle = program(&path).unwrap(); + assert_eq!(bundle.kind(), Kind::Program); + let names: Vec<_> = std::fs::read_dir(bundle.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert_eq!(names, [PROGRAM_BINARY]); + assert!(program(&dir.path().join("missing")).is_err()); + } + #[test] fn built_executable_finds_the_named_binary() { let messages = br#"{"reason":"compiler-artifact","target":{"name":"testkit"},"executable":null} diff --git a/tests/runner/src/execute.rs b/tests/runner/src/execute.rs index 8c46dbe103..bbac5f8977 100644 --- a/tests/runner/src/execute.rs +++ b/tests/runner/src/execute.rs @@ -20,11 +20,12 @@ use anyhow::{Context, Result, ensure}; use tmachine::Session; use tmachine::config::Config; -use crate::bundle::{Bundle, PROFILE, READY_BINARY}; +use crate::bundle::{Bundle, Kind, PROFILE, PROGRAM_BINARY, READY_BINARY}; use crate::test_env; const GUEST_DIR: &str = "/var/tmp/openshell-tests"; -/// The nextest log's name, in the guest directory and in the results. +/// The test log's name, in the guest directory and in the results. A program +/// bundle's log keeps the name, so consumers of the results see one layout. const NEXTEST_LOG: &str = "nextest.log"; /// How long the gateway has to become healthy after the environment is up. // Matches the installers' own wait: an HA k3s gateway booted from a cached @@ -41,6 +42,9 @@ const READY_SLACK: Duration = Duration::from_mins(1); const NEXTEST_TEST_FAILED: i32 = 100; /// nextest's exit code when `--no-tests fail` found no tests to run. const NEXTEST_NO_TESTS_RUN: i32 = 4; +/// A testkit program's exit code when a test failed. It exits 2 when tests +/// could not run. +const PROGRAM_TEST_FAILED: i32 = 1; /// How much of the end of the nextest log to print after a failure. nextest /// ends its output with a summary that names the failed tests. const LOG_TAIL_LINES: usize = 40; @@ -91,6 +95,18 @@ impl Status { )), } } + + /// The status of a testkit program's `run` that exited with `status`. + fn of_program(status: ExitStatus, log: &Path) -> Self { + match status.code() { + Some(0) => Self::Passed, + Some(PROGRAM_TEST_FAILED) => Self::Failed, + _ => Self::Error(format!( + "the test program exited with {status}; see {}", + log.display() + )), + } + } } impl fmt::Display for Status { @@ -112,7 +128,8 @@ pub struct Run<'a> { /// The gateway name the installer registers with the CLI. pub gateway: &'a str, pub bundle: &'a Bundle, - /// A nextest filterset (`-E`) narrowing the tests in the bundle. + /// A nextest filterset (`-E`) narrowing the tests in the bundle. Unused + /// with a program bundle. pub filter: Option<&'a str>, /// Where `junit.xml`, `nextest.log` and, after a failure, the guest's /// `diagnostics/` go. @@ -244,15 +261,20 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { .exec(&["mkdir", "-p", &format!("{guest}/extract")]) .await?, )?; + let kind = run.bundle.kind(); // A booted guest restarts its gateway, so wait for it once here rather // than in every test. Failing here is an environment error, not a test // failure. + let program = format!("{guest}/{PROGRAM_BINARY}"); let ready_binary = format!("{guest}/{READY_BINARY}"); let ready_timeout = READY_TIMEOUT_SECS.to_string(); let mut ready = vec!["env"]; ready.extend(env.iter().map(String::as_str)); - ready.extend([ready_binary.as_str(), ready_timeout.as_str()]); + match kind { + Kind::Nextest => ready.extend([ready_binary.as_str(), ready_timeout.as_str()]), + Kind::Program => ready.extend([program.as_str(), "ready", "--timeout", &ready_timeout]), + } checked( "wait for the gateway", session @@ -282,36 +304,44 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { )?; } + let junit = format!("{guest}/junit.xml"); let archive = format!("{guest}/tests.tar.zst"); let extract = format!("{guest}/extract"); let config = format!("{guest}/nextest.toml"); - let mut nextest = vec!["env"]; - nextest.extend(env.iter().map(String::as_str)); - nextest.extend([ - "cargo-nextest", - "nextest", - "run", - "--archive-file", - &archive, - "--workspace-remap", - guest, - "--extract-to", - &extract, - "--extract-overwrite", - "--config-file", - &config, - "--profile", - PROFILE, - "--no-capture", - // Tests share one gateway and the guest has 4 GiB; run them serially - // whatever --no-capture implies. - "--test-threads", - "1", - "--no-tests", - "fail", - ]); - if let Some(filter) = run.filter { - nextest.extend(["-E", filter]); + let mut tests = vec!["env"]; + tests.extend(env.iter().map(String::as_str)); + match kind { + Kind::Nextest => { + tests.extend([ + "cargo-nextest", + "nextest", + "run", + "--archive-file", + &archive, + "--workspace-remap", + guest, + "--extract-to", + &extract, + "--extract-overwrite", + "--config-file", + &config, + "--profile", + PROFILE, + "--no-capture", + // Tests share one gateway and the guest has 4 GiB; run them + // serially whatever --no-capture implies. + "--test-threads", + "1", + "--no-tests", + "fail", + ]); + if let Some(filter) = run.filter { + tests.extend(["-E", filter]); + } + } + // The program runs its tests one at a time, and writes the report + // where `collect_results` looks for it. + Kind::Program => tests.extend([program.as_str(), "run", "--junit", &junit]), } // nextest writes its output to a file in the guest, so a run that times // out or loses SSH still leaves a log for `collect_results`. The guest @@ -323,12 +353,13 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { test_env::export_script(test_env::ENV_DIR) ); let mut command = vec!["sh", "-c", &script, &log]; - command.extend(nextest); + command.extend(tests); let output = session.exec_timeout(&command, RUN_TIMEOUT).await?; - Ok(Status::of_nextest( - output.status, - &run.results.join(NEXTEST_LOG), - )) + let log = run.results.join(NEXTEST_LOG); + Ok(match kind { + Kind::Nextest => Status::of_nextest(output.status, &log), + Kind::Program => Status::of_program(output.status, &log), + }) } /// Copy the nextest log and the JUnit report out of the guest, whether or not @@ -488,6 +519,15 @@ mod tests { assert!(matches!(status(1), Status::Error(_))); } + #[test] + fn status_follows_the_program_exit_code() { + let status = |code| Status::of_program(ExitStatus::from_raw(code << 8), Path::new("log")); + assert_eq!(status(0), Status::Passed); + assert_eq!(status(PROGRAM_TEST_FAILED), Status::Failed); + // Exit 2: tests could not run. + assert!(matches!(status(2), Status::Error(_))); + } + #[test] fn k3s_diagnostics_keep_everything_the_old_playbook_collected() { for wanted in [ diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index 9bac397824..3de08b5f2c 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -35,12 +35,22 @@ struct Cli { #[arg(long)] family: String, /// Only build these packages of the family [default: every package]. - #[arg(long = "package", short = 'p', value_name = "PACKAGE")] + #[arg( + long = "package", + short = 'p', + value_name = "PACKAGE", + conflicts_with = "program" + )] packages: Vec, /// A nextest filterset narrowing the tests that run, e.g. /// `test(/^smoke::/)`. - #[arg(long, value_name = "FILTERSET")] + #[arg(long, value_name = "FILTERSET", conflicts_with = "program")] filter: Option, + /// Run this prebuilt testkit program, such as `openshell-conformance`, + /// instead of building the family's tests. It must be a static binary for + /// the guest's architecture (a `*-unknown-linux-musl` build). + #[arg(long, value_name = "PATH")] + program: Option, /// The gateway name the installer registers with the CLI. The packaged /// installers register `openshell`. #[arg(long, default_value = "openshell")] @@ -78,20 +88,25 @@ async fn run(cli: Cli) -> Result { let results = std::path::absolute(&cli.results) .with_context(|| format!("resolve {}", cli.results.display()))?; - let target = bundle::guest_target()?; - // The builds block, so they run off the runtime thread to keep the - // signal handlers responsive. - let bundle = tokio::task::spawn_blocking({ - let (root, family, packages) = (root.clone(), cli.family.clone(), cli.packages.clone()); - move || { - eprintln!("==> building {} for {target}", bundle::READY_BINARY); - let ready = bundle::build_ready(&root, target)?; - eprintln!("==> building the {family} tests for {target}"); - bundle::build(&workspace, &packages, target, &ready) - } - }) - .await - .context("build task panicked")??; + let bundle = if let Some(program) = &cli.program { + eprintln!("==> using the prebuilt {}", program.display()); + bundle::program(program)? + } else { + let target = bundle::guest_target()?; + // The builds block, so they run off the runtime thread to keep the + // signal handlers responsive. + tokio::task::spawn_blocking({ + let (root, family, packages) = (root.clone(), cli.family.clone(), cli.packages.clone()); + move || { + eprintln!("==> building {} for {target}", bundle::READY_BINARY); + let ready = bundle::build_ready(&root, target)?; + eprintln!("==> building the {family} tests for {target}"); + bundle::build(&workspace, &packages, target, &ready) + } + }) + .await + .context("build task panicked")?? + }; let tmachine = load_tmachine(&root)?; let status = execute::run( From 4146283782b4be7d4ab0c7ed3d228a93e2f1400f Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 6 Nov 2026 15:31:00 +0100 Subject: [PATCH 2/2] ci(e2e): apply runner filters to a prebuilt test program Signed-off-by: Evan Lezar --- TESTING.md | 4 +++- tests/runner/src/execute.rs | 18 +++++++++++++----- tests/runner/src/main.rs | 12 +++++++----- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/TESTING.md b/TESTING.md index 636bb12e3c..7443e1380f 100644 --- a/TESTING.md +++ b/TESTING.md @@ -251,7 +251,9 @@ complete, for example because the environment didn't come up. It writes `--keep` leaves the VM running until you press Enter. `--program ` runs a prebuilt testkit program, such as a static `openshell-conformance` for the guest's architecture, instead of building the family's tests; it can't be -combined with `--package` or `--filter`. The runner runs the program's `ready` +combined with `--package`. With it, `--filter` (repeatable) takes parts of test +names, as the program does, such as `--filter policy_advisor::`, rather than a +nextest filterset. The runner runs the program's `ready` and `run` commands in the guest and still writes `junit.xml` and `nextest.log` (the program's output) to the results directory. Branch E2E builds `openshell-conformance` once and passes it to every conformance row this way. Two runs on one host diff --git a/tests/runner/src/execute.rs b/tests/runner/src/execute.rs index bbac5f8977..2f477134f9 100644 --- a/tests/runner/src/execute.rs +++ b/tests/runner/src/execute.rs @@ -128,9 +128,9 @@ pub struct Run<'a> { /// The gateway name the installer registers with the CLI. pub gateway: &'a str, pub bundle: &'a Bundle, - /// A nextest filterset (`-E`) narrowing the tests in the bundle. Unused - /// with a program bundle. - pub filter: Option<&'a str>, + /// Narrowing the tests in the bundle: nextest filtersets (`-E`) for a + /// nextest bundle, parts of test names for a program bundle. + pub filters: &'a [String], /// Where `junit.xml`, `nextest.log` and, after a failure, the guest's /// `diagnostics/` go. pub results: &'a Path, @@ -335,13 +335,21 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { "--no-tests", "fail", ]); - if let Some(filter) = run.filter { + for filter in run.filters { tests.extend(["-E", filter]); } } // The program runs its tests one at a time, and writes the report // where `collect_results` looks for it. - Kind::Program => tests.extend([program.as_str(), "run", "--junit", &junit]), + Kind::Program => { + tests.extend([program.as_str(), "run", "--junit", &junit]); + // `run` takes name filters as arguments, after `--` so that one + // starting with a dash isn't read as an option. + if !run.filters.is_empty() { + tests.push("--"); + tests.extend(run.filters.iter().map(String::as_str)); + } + } } // nextest writes its output to a file in the guest, so a run that times // out or loses SSH still leaves a log for `collect_results`. The guest diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index 3de08b5f2c..e0a1f8c7cb 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -42,10 +42,12 @@ struct Cli { conflicts_with = "program" )] packages: Vec, - /// A nextest filterset narrowing the tests that run, e.g. - /// `test(/^smoke::/)`. - #[arg(long, value_name = "FILTERSET", conflicts_with = "program")] - filter: Option, + /// Narrow the tests that run; repeat to run the tests that match any + /// filter. Without `--program` a filter is a nextest filterset, e.g. + /// `test(/^smoke::/)`. With it, a filter is a part of a test's name, e.g. + /// `smoke::`, as the program takes them. + #[arg(long, value_name = "FILTER")] + filter: Vec, /// Run this prebuilt testkit program, such as `openshell-conformance`, /// instead of building the family's tests. It must be a static binary for /// the guest's architecture (a `*-unknown-linux-musl` build). @@ -116,7 +118,7 @@ async fn run(cli: Cli) -> Result { installer: &cli.installer, gateway: &cli.gateway, bundle: &bundle, - filter: cli.filter.as_deref(), + filters: &cli.filter, results: &results, keep: cli.keep, },