Skip to content

Commit 90cbdec

Browse files
feat(runner): add a test runner that runs a test family on tmachine
`openshell-test-runner` builds a family's packages (a workspace under tests/suites/) on the host as a nextest archive for the guest's musl target, boots one tmachine environment by name with the named installer through the Session API, runs nextest in it and brings back the JUnit report and log. No per-suite Ansible playbook is involved: nix run .#openshell-test-runner -- --environment ubuntu-docker-rootful \ --installer deb --family conformance --package sandbox The runner points the tests at the system under test with environment variables: OPENSHELL_GATEWAY (the gateway the installer registered, `--gateway`, default `openshell`) and OPENSHELL_BIN (the installed CLI). It waits for the gateway with `testkit-ready` before the tests start. Nextest runs with `--test-threads 1` and a slow-timeout that terminates a hung test. A run passes if nextest succeeds and its JUnit report shows at least one test and no failures. `--filter` narrows the tests with a nextest filterset, and `--keep` leaves the VM up for debugging. Results from an earlier run are removed first, and the guest's diagnostics are collected after a failure. Which tests run in which environments is left to the caller for now; there are no plans, environment registry or per-test requirements, and a broken test is `#[ignore]`d. Most of this comes from #4629, adapted to testkit finding the gateway the way the CLI does instead of reading a target file. Co-authored-by: Evan Lezar <elezar@nvidia.com> Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>
1 parent 0f1bb94 commit 90cbdec

8 files changed

Lines changed: 1673 additions & 0 deletions

File tree

‎tests/runner/Cargo.lock‎

Lines changed: 682 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tests/runner/Cargo.toml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
[package]
5+
name = "openshell-test-runner"
6+
description = "Runs an OpenShell test family in a tmachine environment"
7+
version = "0.0.0"
8+
edition = "2024"
9+
rust-version = "1.94"
10+
license = "Apache-2.0"
11+
repository = "https://github.com/NVIDIA/OpenShell"
12+
publish = false
13+
14+
[[bin]]
15+
name = "openshell-test-runner"
16+
path = "src/main.rs"
17+
18+
[dependencies]
19+
anyhow = "1"
20+
clap = { version = "4", features = ["derive"] }
21+
serde = { version = "1", features = ["derive"] }
22+
serde_json = "1"
23+
tempfile = "3"
24+
tmachine = { path = "../tmachine" }
25+
tokio = { version = "1", features = ["macros", "rt"] }
26+
27+
[workspace]

‎tests/runner/src/bundle.rs‎

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//! Building a family's test bundle for the guest, on the host.
5+
//!
6+
//! A bundle is a tar holding a nextest archive built for the guest target,
7+
//! the family workspace's `Cargo.toml` files (for `--workspace-remap`), a
8+
//! `nextest` config with the runner's `JUnit` profile, and `testkit-ready`,
9+
//! which waits for the gateway before the tests run. The guest needs only the
10+
//! `cargo-nextest` binary to run it.
11+
//!
12+
//! Building is separate from running, so a prebuilt bundle can be passed in
13+
//! later instead.
14+
//!
15+
//! Adapted from the bundle builder in Evan Lezar's NVIDIA/OpenShell#4629.
16+
17+
use std::path::{Path, PathBuf};
18+
use std::process::Command;
19+
20+
use anyhow::{Context, Result, bail, ensure};
21+
use serde::Deserialize;
22+
use tempfile::TempDir;
23+
24+
/// The nextest profile the runner uses in the guest.
25+
pub const PROFILE: &str = "openshell";
26+
27+
/// The readiness check every bundle carries, at the bundle root.
28+
pub const READY_BINARY: &str = "testkit-ready";
29+
30+
/// Nextest flags a test as slow after the period and terminates it after
31+
/// `terminate-after` periods, so a hung test fails after two minutes instead
32+
/// of holding the lane until the job times out.
33+
const NEXTEST_CONFIG: &str = r#"# Written by the OpenShell test runner.
34+
[profile.openshell]
35+
fail-fast = false
36+
slow-timeout = { period = "30s", terminate-after = 4 }
37+
38+
[profile.openshell.junit]
39+
path = "junit.xml"
40+
"#;
41+
42+
pub struct Bundle {
43+
pub path: PathBuf,
44+
_dir: TempDir,
45+
}
46+
47+
/// The Rust target for guests. tmachine only boots guests of the host's
48+
/// architecture, and static musl binaries don't depend on the guest's libc.
49+
pub fn guest_target() -> Result<&'static str> {
50+
if cfg!(target_arch = "x86_64") {
51+
Ok("x86_64-unknown-linux-musl")
52+
} else if cfg!(target_arch = "aarch64") {
53+
Ok("aarch64-unknown-linux-musl")
54+
} else {
55+
bail!("tmachine can't boot guests on this host's architecture")
56+
}
57+
}
58+
59+
/// Build `testkit-ready` for the guest and return its path.
60+
pub fn build_ready(root: &Path, target: &str) -> Result<PathBuf> {
61+
let output = Command::new("cargo")
62+
.args(["build", "--locked", "--manifest-path"])
63+
.arg(root.join("tests/testkit/Cargo.toml"))
64+
.args(["-p", "openshell-testkit", "--bin", READY_BINARY])
65+
.args([
66+
"--target",
67+
target,
68+
"--message-format",
69+
"json-render-diagnostics",
70+
])
71+
.stderr(std::process::Stdio::inherit())
72+
.output()
73+
.context("run cargo build")?;
74+
ensure!(
75+
output.status.success(),
76+
"build {READY_BINARY} exited with {}",
77+
output.status
78+
);
79+
built_executable(&output.stdout, READY_BINARY)
80+
.with_context(|| format!("cargo build did not report a {READY_BINARY} executable"))
81+
}
82+
83+
/// The executable cargo reports building for the binary target `name`.
84+
fn built_executable(messages: &[u8], name: &str) -> Option<PathBuf> {
85+
#[derive(Deserialize)]
86+
struct Message {
87+
reason: String,
88+
target: Option<Target>,
89+
executable: Option<PathBuf>,
90+
}
91+
#[derive(Deserialize)]
92+
struct Target {
93+
name: String,
94+
}
95+
96+
serde_json::Deserializer::from_slice(messages)
97+
.into_iter::<Message>()
98+
.filter_map(std::result::Result::ok)
99+
.filter(|message| message.reason == "compiler-artifact")
100+
.filter(|message| {
101+
message
102+
.target
103+
.as_ref()
104+
.is_some_and(|target| target.name == name)
105+
})
106+
.find_map(|message| message.executable)
107+
}
108+
109+
/// Build the bundle for a family workspace (`tests/suites/<family>`). With no
110+
/// `packages`, the archive holds every package in the workspace. `ready` is
111+
/// the guest's `testkit-ready`.
112+
pub fn build(workspace: &Path, packages: &[String], target: &str, ready: &Path) -> Result<Bundle> {
113+
let dir = tempfile::tempdir().context("create bundle directory")?;
114+
let stage = dir.path().join("stage");
115+
std::fs::create_dir(&stage)?;
116+
let manifest = workspace.join("Cargo.toml");
117+
ensure!(
118+
manifest.is_file(),
119+
"{} is not a test family workspace",
120+
workspace.display()
121+
);
122+
let archive = stage.join("tests.tar.zst");
123+
124+
let mut command = Command::new("cargo");
125+
command
126+
.args(["nextest", "archive", "--locked", "--manifest-path"])
127+
.arg(&manifest)
128+
.args(["--target", target, "--archive-file"])
129+
.arg(&archive);
130+
if packages.is_empty() {
131+
command.arg("--workspace");
132+
}
133+
for package in packages {
134+
command.args(["-p", package]);
135+
}
136+
run(&mut command).with_context(|| format!("build the {} test archive", workspace.display()))?;
137+
138+
copy_manifests(workspace, &stage)?;
139+
std::fs::write(stage.join("nextest.toml"), NEXTEST_CONFIG)?;
140+
std::fs::copy(ready, stage.join(READY_BINARY))
141+
.with_context(|| format!("copy {}", ready.display()))?;
142+
143+
let path = dir.path().join("tests.tar");
144+
run(Command::new("tar")
145+
.arg("-C")
146+
.arg(&stage)
147+
.arg("-cf")
148+
.arg(&path)
149+
.arg("."))
150+
.context("pack test bundle")?;
151+
Ok(Bundle { path, _dir: dir })
152+
}
153+
154+
/// Copy every `Cargo.toml` under `workspace` into `stage`, keeping relative
155+
/// paths and skipping build output.
156+
fn copy_manifests(workspace: &Path, stage: &Path) -> Result<()> {
157+
fn walk(root: &Path, dir: &Path, stage: &Path) -> Result<()> {
158+
for entry in std::fs::read_dir(dir).with_context(|| format!("read {}", dir.display()))? {
159+
let path = entry?.path();
160+
let name = path.file_name().unwrap_or_default();
161+
if path.is_dir() {
162+
if name != "target" && name != ".git" {
163+
walk(root, &path, stage)?;
164+
}
165+
} else if name == "Cargo.toml" {
166+
let dest = stage.join(path.strip_prefix(root)?);
167+
std::fs::create_dir_all(dest.parent().unwrap_or(stage))?;
168+
std::fs::copy(&path, &dest).with_context(|| format!("copy {}", path.display()))?;
169+
}
170+
}
171+
Ok(())
172+
}
173+
walk(workspace, workspace, stage)
174+
}
175+
176+
fn run(command: &mut Command) -> Result<()> {
177+
let status = command
178+
.status()
179+
.with_context(|| format!("run {command:?}"))?;
180+
if !status.success() {
181+
bail!("{command:?} exited with {status}");
182+
}
183+
Ok(())
184+
}
185+
186+
#[cfg(test)]
187+
mod tests {
188+
use super::*;
189+
190+
#[test]
191+
fn built_executable_finds_the_named_binary() {
192+
let messages = br#"{"reason":"compiler-artifact","target":{"name":"testkit"},"executable":null}
193+
{"reason":"compiler-artifact","target":{"name":"testkit-ready"},"executable":"/t/debug/testkit-ready"}
194+
{"reason":"build-finished","success":true}
195+
"#;
196+
assert_eq!(
197+
built_executable(messages, "testkit-ready"),
198+
Some(PathBuf::from("/t/debug/testkit-ready"))
199+
);
200+
assert_eq!(built_executable(messages, "other"), None);
201+
}
202+
}

0 commit comments

Comments
 (0)