From 76953c85bf6c90b6a94480023cced739cc5038f0 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 4 Nov 2026 12:35:49 -0800 Subject: [PATCH 1/9] test(tmachine): load the Python workload image and set the k3s host gateway Every installer loads openshell/e2e-python:dev next to the runtime images, so tests can create sandboxes that run python3, curl, git or openssl. The k3s installers set the chart's server.hostGatewayIP to the node's InternalIP. Tests run on the node, so host.openshell.internal now resolves to it on k3s and k3s-ha. Refs #5275 Signed-off-by: Matthew Grossman --- .../ansible/playbooks/openshell-k3s.yaml | 23 +++++++++++++++++++ .../roles/openshell_gateway/tasks/docker.yaml | 9 ++++++-- .../roles/openshell_gateway/tasks/podman.yaml | 9 ++++++-- tests/tmachine/config.nix | 8 ++++++- 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/tests/tmachine/ansible/playbooks/openshell-k3s.yaml b/tests/tmachine/ansible/playbooks/openshell-k3s.yaml index 2f03f9d7a8..abebd8fc39 100644 --- a/tests/tmachine/ansible/playbooks/openshell-k3s.yaml +++ b/tests/tmachine/ansible/playbooks/openshell-k3s.yaml @@ -34,6 +34,9 @@ src: "{{ openshell_sandbox_image }}" - name: supervisor src: "{{ openshell_supervisor_image }}" + # The testkit's python_image() sandbox workload. + - name: e2e-python + src: "{{ openshell_e2e_python_image }}" - name: Import OpenShell images into K3s ansible.builtin.command: @@ -49,6 +52,7 @@ - gateway - sandbox - supervisor + - e2e-python - name: Copy OpenShell Helm chart ansible.builtin.copy: @@ -56,6 +60,24 @@ dest: /var/lib/openshell/artifacts/helm-chart.tgz mode: "0600" + # Tests run in the guest, so host.openshell.internal must resolve to the + # node itself. QEMU user networking gives every boot the same guest + # address (10.0.2.15), so the value stays valid when the disk is reused. + - name: Discover the K3s node address + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - --request-timeout=10s + - get + - nodes + - --output=jsonpath={.items[0].status.addresses[?(@.type=="InternalIP")].address} + register: k3s_node_address + changed_when: false + until: k3s_node_address.rc == 0 and k3s_node_address.stdout is match('^[0-9.]+$') + retries: 30 + delay: 2 + - name: Write OpenShell Helm values ansible.builtin.copy: dest: /var/lib/openshell/artifacts/values.yaml @@ -85,6 +107,7 @@ auth: allowUnauthenticatedUsers: true disableTls: true + hostGatewayIP: "{{ k3s_node_address.stdout }}" telemetryEnabled: false - name: Write OpenShell Helm values overlay diff --git a/tests/tmachine/ansible/roles/openshell_gateway/tasks/docker.yaml b/tests/tmachine/ansible/roles/openshell_gateway/tasks/docker.yaml index 3585c1d33f..11b033972a 100644 --- a/tests/tmachine/ansible/roles/openshell_gateway/tasks/docker.yaml +++ b/tests/tmachine/ansible/roles/openshell_gateway/tasks/docker.yaml @@ -2,7 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 --- -- name: Copy OpenShell runtime images +# Tests run sandboxes from the e2e-python workload image (the testkit's +# python_image()), so load it next to the runtime images. +- name: Copy OpenShell runtime and test workload images become: true ansible.builtin.copy: src: "{{ item.src }}" @@ -15,8 +17,10 @@ src: "{{ openshell_sandbox_image }}" - name: supervisor src: "{{ openshell_supervisor_image }}" + - name: e2e-python + src: "{{ openshell_e2e_python_image }}" -- name: Load OpenShell runtime images +- name: Load OpenShell runtime and test workload images become: true ansible.builtin.command: argv: @@ -27,3 +31,4 @@ loop: - sandbox - supervisor + - e2e-python diff --git a/tests/tmachine/ansible/roles/openshell_gateway/tasks/podman.yaml b/tests/tmachine/ansible/roles/openshell_gateway/tasks/podman.yaml index 784c56f6cb..b85361b769 100644 --- a/tests/tmachine/ansible/roles/openshell_gateway/tasks/podman.yaml +++ b/tests/tmachine/ansible/roles/openshell_gateway/tasks/podman.yaml @@ -2,7 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 --- -- name: Copy OpenShell runtime images +# Tests run sandboxes from the e2e-python workload image (the testkit's +# python_image()), so load it next to the runtime images. +- name: Copy OpenShell runtime and test workload images become: true ansible.builtin.copy: src: "{{ item.src }}" @@ -15,8 +17,10 @@ src: "{{ openshell_sandbox_image }}" - name: supervisor src: "{{ openshell_supervisor_image }}" + - name: e2e-python + src: "{{ openshell_e2e_python_image }}" -- name: Load OpenShell runtime images +- name: Load OpenShell runtime and test workload images become: true become_user: "{{ openshell_gateway_user }}" ansible.builtin.command: @@ -28,6 +32,7 @@ loop: - sandbox - supervisor + - e2e-python environment: HOME: "{{ openshell_gateway_home }}" XDG_RUNTIME_DIR: "/run/user/{{ openshell_gateway_uid }}" diff --git a/tests/tmachine/config.nix b/tests/tmachine/config.nix index ac1aa28d6d..24206097f9 100644 --- a/tests/tmachine/config.nix +++ b/tests/tmachine/config.nix @@ -23,9 +23,12 @@ let OVMF = firmwarePkgs.OVMF; }; qemu = qemuPkgs.qemu.override { hostCpuOnly = true; }; + # Every installer loads the sandbox workload image tests run python3 in. + e2ePythonImage = "../../artifacts/images/openshell-e2e-python-dev.tar"; k3sInputs = { agent_sandbox_version = "0.5.0"; openshell_cli_binary = "../../artifacts/binaries/${muslTarget}/openshell"; + openshell_e2e_python_image = e2ePythonImage; openshell_gateway_image = "../../artifacts/images/openshell-gateway-tmachine.tar"; openshell_helm_chart = "../../artifacts/helm/helm-chart-0.0.0.tgz"; openshell_sandbox_image = "../../artifacts/images/openshell-sandbox-tmachine.tar"; @@ -204,6 +207,7 @@ let inputs = { openshell_cli_binary = "../../artifacts/binaries/${muslTarget}/openshell"; openshell_gateway_binary = "../../artifacts/binaries/${gnuTarget}/openshell-gateway"; + openshell_e2e_python_image = e2ePythonImage; openshell_supervisor_image = "../../artifacts/images/openshell-supervisor-tmachine.tar"; openshell_sandbox_image = "../../artifacts/images/openshell-sandbox-tmachine.tar"; }; @@ -216,6 +220,7 @@ let ]; inputs = { openshell_deb = "../../artifacts/packages/openshell.deb"; + openshell_e2e_python_image = e2ePythonImage; openshell_supervisor_image = "../../artifacts/images/openshell-supervisor-tmachine.tar"; openshell_sandbox_image = "../../artifacts/images/openshell-sandbox-tmachine.tar"; }; @@ -227,6 +232,7 @@ let inputs = { openshell_rpm = "../../artifacts/packages/rpm/openshell.rpm"; openshell_gateway_rpm = "../../artifacts/packages/rpm/openshell-gateway.rpm"; + openshell_e2e_python_image = e2ePythonImage; openshell_supervisor_image = "../../artifacts/images/openshell-supervisor-tmachine.tar"; openshell_sandbox_image = "../../artifacts/images/openshell-sandbox-tmachine.tar"; }; @@ -245,7 +251,7 @@ let playbooks = [ "ansible/playbooks/drivers/podman/e2e.yaml" ]; inputs = { openshell_podman_e2e_test_bundle = "../../artifacts/test-archives/${muslTarget}/openshell-podman-e2e-tests.tar"; - openshell_podman_e2e_workload_image = "../../artifacts/images/openshell-e2e-python-dev.tar"; + openshell_podman_e2e_workload_image = e2ePythonImage; }; } ]; From 283b9ff5f8a626937c2d9c32f1e45878124a0a03 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 4 Nov 2026 12:35:59 -0800 Subject: [PATCH 2/9] test(testkit): add python_image() for the Python workload image OPENSHELL_TEST_PYTHON_IMAGE overrides the default openshell/e2e-python:dev, next to OPENSHELL_TEST_BASE_IMAGE. Refs #5275 Signed-off-by: Matthew Grossman --- tests/testkit/README.md | 4 ++++ tests/testkit/testkit/src/image.rs | 23 ++++++++++++++++++++--- tests/testkit/testkit/src/lib.rs | 5 ++++- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/tests/testkit/README.md b/tests/testkit/README.md index 1aea174049..1749fad6e7 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -75,6 +75,10 @@ fails. A killed test process skips cleanup, so names are unique to the run. compute driver. Driver suites call it first. - `testkit::base_image()` for the image to create sandboxes from or build on: `OPENSHELL_TEST_BASE_IMAGE`, or OpenShell's default sandbox image. +- `testkit::python_image()` for sandboxes that run `python3`, curl, git, or + openssl: `OPENSHELL_TEST_PYTHON_IMAGE`, or `openshell/e2e-python:dev` + (`e2e/python/Dockerfile.workload`). The tmachine installers load it into + every environment; `mise run e2e:workload:build` builds it locally. - `cx.cli([...])` to run the installed CLI against the same gateway, with `success()`, `failure()`, and `json()` on the output. `cx.cli_handle()` returns that CLI for code that can't borrow `cx`, such as a cleanup hook. diff --git a/tests/testkit/testkit/src/image.rs b/tests/testkit/testkit/src/image.rs index 4e398800b1..06952a82cf 100644 --- a/tests/testkit/testkit/src/image.rs +++ b/tests/testkit/testkit/src/image.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! The base image tests create sandboxes from or build their images on. +//! The images tests create sandboxes from or build their images on. /// Overrides [`DEFAULT_BASE_IMAGE`], e.g. with a mirror. pub const BASE_IMAGE_ENV: &str = "OPENSHELL_TEST_BASE_IMAGE"; @@ -9,10 +9,27 @@ pub const BASE_IMAGE_ENV: &str = "OPENSHELL_TEST_BASE_IMAGE"; /// nvcr.io. pub const DEFAULT_BASE_IMAGE: &str = "nvcr.io/nvidia/base/ubuntu:24.04"; +/// Overrides [`DEFAULT_PYTHON_IMAGE`]. +pub const PYTHON_IMAGE_ENV: &str = "OPENSHELL_TEST_PYTHON_IMAGE"; +/// The workload image for tests that run `python3`, curl, git or openssl in a +/// sandbox, built from `e2e/python/Dockerfile.workload`. It isn't published: +/// the tmachine installers load it into every environment, and +/// `mise run e2e:workload:build` builds it locally. +pub const DEFAULT_PYTHON_IMAGE: &str = "openshell/e2e-python:dev"; + /// The image named by [`BASE_IMAGE_ENV`], or [`DEFAULT_BASE_IMAGE`]. pub fn base_image() -> String { - std::env::var(BASE_IMAGE_ENV) + image_from_env(BASE_IMAGE_ENV, DEFAULT_BASE_IMAGE) +} + +/// The image named by [`PYTHON_IMAGE_ENV`], or [`DEFAULT_PYTHON_IMAGE`]. +pub fn python_image() -> String { + image_from_env(PYTHON_IMAGE_ENV, DEFAULT_PYTHON_IMAGE) +} + +fn image_from_env(var: &str, default: &str) -> String { + std::env::var(var) .ok() .filter(|image| !image.trim().is_empty()) - .unwrap_or_else(|| DEFAULT_BASE_IMAGE.to_string()) + .unwrap_or_else(|| default.to_string()) } diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 10db7d0e33..7b5896349e 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -39,7 +39,10 @@ pub use context::{ }; pub use error::{Error, Result, ResultExt}; pub use gateway::Gateway; -pub use image::{BASE_IMAGE_ENV, DEFAULT_BASE_IMAGE, base_image}; +pub use image::{ + BASE_IMAGE_ENV, DEFAULT_BASE_IMAGE, DEFAULT_PYTHON_IMAGE, PYTHON_IMAGE_ENV, base_image, + python_image, +}; pub use openshell_sdk as sdk; pub use poll::{Poll, poll_until}; pub use run::{TestInfo, run}; From 8441388e50a6dbac153b5c9b4a8b18c804bfc9d0 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 4 Nov 2026 12:35:59 -0800 Subject: [PATCH 3/9] test(runner): honor family nextest config and add --test-threads The runner passes its openshell profile as a nextest tool config and the bundle carries tests/suites//.config/nextest.toml into the remapped workspace, so a family can set per-test timeouts and test groups, or override the profile. --test-threads defaults to 1, which keeps the serial --no-capture run. Refs #5275 Signed-off-by: Matthew Grossman --- TESTING.md | 18 +++++++ tests/runner/src/bundle.rs | 59 +++++++++++++++++++--- tests/runner/src/execute.rs | 98 +++++++++++++++++++++++++------------ tests/runner/src/main.rs | 7 +++ 4 files changed, 146 insertions(+), 36 deletions(-) diff --git a/TESTING.md b/TESTING.md index 5e74f53c72..4e8d120455 100644 --- a/TESTING.md +++ b/TESTING.md @@ -247,6 +247,24 @@ need different `TMACHINE_SSH_PORT` values (default 2222). The installer inputs, such as the Debian package and the images, come from `artifacts/`, built by `nix run .#build-artifacts` or downloaded by CI. +The runner's nextest profile, `openshell`, runs tests one at a time and +fails a test that runs longer than two minutes (`slow-timeout` 30 s × 4). A +family that needs other settings puts them in +`tests/suites//.config/nextest.toml`, which takes priority over the +runner's profile: + +- `[[profile.default.overrides]]` for some tests, such as a longer + `slow-timeout` or a `test-group` from `[test-groups]` that keeps tests from + running at the same time. These also apply when you run the suite with + `cargo nextest run`. +- `[profile.openshell]` for every test of the family in the runner, such as + its `slow-timeout`. + +`--test-threads ` runs N tests at once. Tests share the gateway and the +guest, so it defaults to 1; raise it only for tests that tolerate that, and +put the ones that don't in a test group. A serial run streams test output into +`nextest.log`; a parallel run shows a test's output only when it fails. + A flaky or broken test is marked `#[ignore = ""]`, which takes it out of every environment until it's fixed. diff --git a/tests/runner/src/bundle.rs b/tests/runner/src/bundle.rs index 08e66738c6..d5eba208ab 100644 --- a/tests/runner/src/bundle.rs +++ b/tests/runner/src/bundle.rs @@ -4,10 +4,16 @@ //! Building a family's test bundle for the guest, on the host. //! //! A bundle is a directory holding a nextest archive built for the guest target, -//! the family workspace's `Cargo.toml` files (for `--workspace-remap`), a -//! `nextest` config with the runner's `JUnit` profile, and `testkit-ready`, -//! which waits for the gateway before the tests run. The guest needs only the -//! `cargo-nextest` binary to run it. +//! the family workspace's `Cargo.toml` files (for `--workspace-remap`), the +//! runner's nextest profile, and `testkit-ready`, which waits for the gateway +//! before the tests run. The guest needs only the `cargo-nextest` binary to run +//! it. +//! +//! The runner's profile is a nextest tool config, below the repository config +//! in priority. A family that needs other settings, such as longer timeouts for +//! some tests or a test group that runs serially, puts them in +//! `tests/suites//.config/nextest.toml`; the bundle carries it to the +//! same place in the remapped workspace, where nextest reads it. use std::path::{Path, PathBuf}; use std::process::Command; @@ -22,9 +28,20 @@ pub const PROFILE: &str = "openshell"; /// The readiness check every bundle carries, at the bundle root. pub const READY_BINARY: &str = "testkit-ready"; +/// The runner's nextest config, at the bundle root. It is passed as the tool +/// config of [`TOOL_NAME`]. +pub const TOOL_CONFIG: &str = "runner-nextest.toml"; + +/// The tool name nextest gives the runner's config. +pub const TOOL_NAME: &str = "openshell-test-runner"; + +/// A family's nextest config, relative to its workspace and the bundle root. +const FAMILY_CONFIG: &str = ".config/nextest.toml"; + /// Nextest flags a test as slow after the period and terminates it after /// `terminate-after` periods, so a hung test fails after two minutes instead -/// of holding the lane until the job times out. +/// of holding the lane until the job times out. A family's own config takes +/// priority over this. const NEXTEST_CONFIG: &str = r#"# Written by the OpenShell test runner. [profile.openshell] fail-fast = false @@ -134,12 +151,26 @@ pub fn build(workspace: &Path, packages: &[String], target: &str, ready: &Path) run(&mut command).with_context(|| format!("build the {} test archive", workspace.display()))?; copy_manifests(workspace, stage)?; - std::fs::write(stage.join("nextest.toml"), NEXTEST_CONFIG)?; + copy_family_config(workspace, stage)?; + std::fs::write(stage.join(TOOL_CONFIG), NEXTEST_CONFIG)?; std::fs::copy(ready, stage.join(READY_BINARY)) .with_context(|| format!("copy {}", ready.display()))?; Ok(Bundle(dir)) } +/// Copy the family's nextest config, if it has one, to the same place in +/// `stage`. +fn copy_family_config(workspace: &Path, stage: &Path) -> Result<()> { + let config = workspace.join(FAMILY_CONFIG); + if !config.is_file() { + return Ok(()); + } + let dest = stage.join(FAMILY_CONFIG); + std::fs::create_dir_all(dest.parent().unwrap_or(stage))?; + std::fs::copy(&config, &dest).with_context(|| format!("copy {}", config.display()))?; + Ok(()) +} + /// Copy every `Cargo.toml` under `workspace` into `stage`, keeping relative /// paths and skipping build output. fn copy_manifests(workspace: &Path, stage: &Path) -> Result<()> { @@ -188,4 +219,20 @@ mod tests { ); assert_eq!(built_executable(messages, "other"), None); } + + #[test] + fn family_config_lands_where_nextest_reads_it() { + let workspace = tempfile::tempdir().unwrap(); + let stage = tempfile::tempdir().unwrap(); + copy_family_config(workspace.path(), stage.path()).unwrap(); + assert!(!stage.path().join(".config").exists()); + + std::fs::create_dir(workspace.path().join(".config")).unwrap(); + std::fs::write(workspace.path().join(FAMILY_CONFIG), "[test-groups]\n").unwrap(); + copy_family_config(workspace.path(), stage.path()).unwrap(); + assert_eq!( + std::fs::read_to_string(stage.path().join(".config/nextest.toml")).unwrap(), + "[test-groups]\n" + ); + } } diff --git a/tests/runner/src/execute.rs b/tests/runner/src/execute.rs index 8c46dbe103..10353eb177 100644 --- a/tests/runner/src/execute.rs +++ b/tests/runner/src/execute.rs @@ -12,6 +12,7 @@ //! [`crate::test_env`]). use std::fmt; +use std::num::NonZeroUsize; use std::path::Path; use std::process::{ExitCode, ExitStatus, Output}; use std::time::Duration; @@ -20,7 +21,7 @@ use anyhow::{Context, Result, ensure}; use tmachine::Session; use tmachine::config::Config; -use crate::bundle::{Bundle, PROFILE, READY_BINARY}; +use crate::bundle::{Bundle, PROFILE, READY_BINARY, TOOL_CONFIG, TOOL_NAME}; use crate::test_env; const GUEST_DIR: &str = "/var/tmp/openshell-tests"; @@ -114,6 +115,8 @@ pub struct Run<'a> { pub bundle: &'a Bundle, /// A nextest filterset (`-E`) narrowing the tests in the bundle. pub filter: Option<&'a str>, + /// How many tests run at once. + pub test_threads: NonZeroUsize, /// Where `junit.xml`, `nextest.log` and, after a failure, the guest's /// `diagnostics/` go. pub results: &'a Path, @@ -282,37 +285,10 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { )?; } - let archive = format!("{guest}/tests.tar.zst"); - let extract = format!("{guest}/extract"); - let config = format!("{guest}/nextest.toml"); let mut nextest = vec!["env"]; nextest.extend(env.iter().map(String::as_str)); - nextest.extend([ - "cargo-nextest", - "nextest", - "run", - "--archive-file", - &archive, - "--workspace-remap", - guest, - "--extract-to", - &extract, - "--extract-overwrite", - "--config-file", - &config, - "--profile", - PROFILE, - "--no-capture", - // Tests share one gateway and the guest has 4 GiB; run them serially - // whatever --no-capture implies. - "--test-threads", - "1", - "--no-tests", - "fail", - ]); - if let Some(filter) = run.filter { - nextest.extend(["-E", filter]); - } + let args = nextest_args(guest, run.test_threads, run.filter); + nextest.extend(args.iter().map(String::as_str)); // nextest writes its output to a file in the guest, so a run that times // out or loses SSH still leaves a log for `collect_results`. The guest // shell exports the published variables itself, so their values, which @@ -331,6 +307,47 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { )) } +/// The `cargo-nextest` command line that runs the bundle unpacked at `guest`. +/// +/// The runner's profile is a tool config, so the family's +/// `.config/nextest.toml` in the remapped workspace overrides it. Tests share +/// one gateway and the guest has 4 GiB, so they run one at a time unless the +/// run asks for more threads. A serial run uses `--no-capture`, which runs +/// tests one at a time and streams their output into the log. A parallel run +/// can't, so nextest captures each test's output and shows it when the test +/// fails. +fn nextest_args(guest: &str, test_threads: NonZeroUsize, filter: Option<&str>) -> Vec { + let mut args: Vec = [ + "cargo-nextest", + "nextest", + "run", + "--archive-file", + &format!("{guest}/tests.tar.zst"), + "--workspace-remap", + guest, + "--extract-to", + &format!("{guest}/extract"), + "--extract-overwrite", + "--tool-config-file", + &format!("{TOOL_NAME}:{guest}/{TOOL_CONFIG}"), + "--profile", + PROFILE, + "--no-tests", + "fail", + ] + .map(String::from) + .into(); + if test_threads.get() == 1 { + args.push("--no-capture".to_string()); + } else { + args.extend(["--test-threads".to_string(), test_threads.to_string()]); + } + if let Some(filter) = filter { + args.extend(["-E".to_string(), filter.to_string()]); + } + args +} + /// Copy the nextest log and the JUnit report out of the guest, whether or not /// the run finished. Either may be missing, for example when the gateway never /// became ready. @@ -525,6 +542,27 @@ mod tests { } } + #[test] + fn nextest_runs_serially_with_output_unless_asked_for_threads() { + let one = NonZeroUsize::MIN; + let serial = nextest_args("/g", one, None); + assert!(serial.contains(&"--no-capture".to_string())); + // nextest ignores --test-threads with --no-capture, and warns. + assert!(!serial.contains(&"--test-threads".to_string())); + assert!(serial.windows(2).any(|w| w + == [ + "--tool-config-file", + "openshell-test-runner:/g/runner-nextest.toml" + ])); + assert!(!serial.contains(&"--config-file".to_string())); + assert!(!serial.contains(&"-E".to_string())); + + let parallel = nextest_args("/g", NonZeroUsize::new(4).unwrap(), Some("test(a)")); + assert!(!parallel.contains(&"--no-capture".to_string())); + assert!(parallel.windows(2).any(|w| w == ["--test-threads", "4"])); + assert!(parallel.ends_with(&["-E".to_string(), "test(a)".to_string()])); + } + #[test] fn exit_code_separates_test_failures_from_errors() { assert_eq!(Status::Passed.exit_code(), ExitCode::SUCCESS); diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index 1503a4e0d7..b55c487c70 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -7,6 +7,7 @@ //! provides the guest cross-compilation toolchain, QEMU, Ansible, and the //! tmachine configuration. +use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; use std::process::ExitCode; @@ -41,6 +42,11 @@ struct Cli { /// `binary(smoke)`. #[arg(long, value_name = "FILTERSET")] filter: Option, + /// How many tests run at once. Tests share the gateway and the guest, so + /// raise it only for tests that tolerate that; nextest then captures each + /// test's output and shows it when the test fails. + #[arg(long, default_value = "1", value_name = "N")] + test_threads: NonZeroUsize, /// The gateway name the installer registers with the CLI. The packaged /// installers register `openshell`. #[arg(long, default_value = "openshell")] @@ -102,6 +108,7 @@ async fn run(cli: Cli) -> Result { gateway: &cli.gateway, bundle: &bundle, filter: cli.filter.as_deref(), + test_threads: cli.test_threads, results: &results, keep: cli.keep, }, From 061ae1ee9a155984d381f66f28a166da70b8977c Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 4 Nov 2026 12:36:00 -0800 Subject: [PATCH 4/9] test(features): check host.openshell.internal from a Python sandbox The host-gateway suite starts a server on the host and checks that a sandbox from the Python workload image reaches it through host.openshell.internal. CI runs it on Docker rootful, Podman rootful and rootless, k3s and k3s-ha. Closes #5275 Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 5 + TESTING.md | 7 + tests/suites/features/Cargo.lock | 9 + tests/suites/features/Cargo.toml | 2 +- tests/suites/features/host-gateway/Cargo.toml | 17 ++ .../host-gateway/tests/host_gateway.rs | 203 ++++++++++++++++++ 6 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 tests/suites/features/host-gateway/Cargo.toml create mode 100644 tests/suites/features/host-gateway/tests/host_gateway.rs diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 04bd61ef37..791761befb 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -77,6 +77,11 @@ jobs: {"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"oci-image","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful","installer":"rpm","family":"features","package":"oci-image","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"oci-image","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"host-gateway","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"fedora-podman-rootful","installer":"rpm","family":"features","package":"host-gateway","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"host-gateway","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-k3s","installer":"k3s","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootless-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful-userns-default","installer":"binaries","family":"drivers/podman","gateway":"tmachine","runner":"ubuntu-24.04"}, diff --git a/TESTING.md b/TESTING.md index 4e8d120455..a697ffd0fb 100644 --- a/TESTING.md +++ b/TESTING.md @@ -595,6 +595,13 @@ mise run e2e:oci-image:podman Both tasks set `OPENSHELL_TEST_CONTAINER_ENGINE`, the command that builds test images into the gateway's image store. +The `host-gateway` feature suite (`tests/suites/features/host-gateway`) checks +that a sandbox from the Python test image reaches a server the test runs on the +host through `host.openshell.internal`. CI runs it on Docker rootful, Podman +rootful and rootless, and the `k3s` and `k3s-ha` installers. The tests run on +the K3s node, so the K3s installers set the chart's `server.hostGatewayIP` to +the node's address. + Run the VM-backed Rust CLI e2e suite: ```shell diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 4362c9cacd..e461cb5312 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -1062,6 +1062,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "host-gateway" +version = "0.0.0" +dependencies = [ + "openshell-testkit", + "tempfile", + "tokio", +] + [[package]] name = "hostname" version = "0.4.2" diff --git a/tests/suites/features/Cargo.toml b/tests/suites/features/Cargo.toml index 5912e9176f..af907b4b15 100644 --- a/tests/suites/features/Cargo.toml +++ b/tests/suites/features/Cargo.toml @@ -3,4 +3,4 @@ [workspace] resolver = "2" -members = ["keycloak-provider-refresh", "kubernetes-updates", "oci-image"] +members = ["host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image"] diff --git a/tests/suites/features/host-gateway/Cargo.toml b/tests/suites/features/host-gateway/Cargo.toml new file mode 100644 index 0000000000..d81b6cafb2 --- /dev/null +++ b/tests/suites/features/host-gateway/Cargo.toml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "host-gateway" +description = "Features suite: sandboxes reach a host server through host.openshell.internal" +version = "0.0.0" +edition = "2024" +rust-version = "1.94" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA/OpenShell" +publish = false + +[dependencies] +openshell-testkit = { path = "../../../testkit/testkit" } +tempfile = "3" +tokio = { version = "1.43", features = ["io-util", "net", "rt"] } diff --git a/tests/suites/features/host-gateway/tests/host_gateway.rs b/tests/suites/features/host-gateway/tests/host_gateway.rs new file mode 100644 index 0000000000..66af363071 --- /dev/null +++ b/tests/suites/features/host-gateway/tests/host_gateway.rs @@ -0,0 +1,203 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `host.openshell.internal` names the host from inside a sandbox, on every +//! compute driver. +//! +//! The test starts an HTTP server where the tests run, which is the host the +//! gateway runs its sandboxes on, and creates a sandbox from the Python test +//! image ([`testkit::python_image`]) whose main process requests a page from +//! that server through the name. + +use testkit::{Context, Error, Result, ResultExt as _}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::task::JoinHandle; + +/// The Python test image's virtual environment, which links to a uv-managed +/// Python under `/sandbox/.uv/python`. +const VENV_PYTHON: &str = "/sandbox/.venv/bin/python3"; + +/// Prints the interpreter's prefix, then requests `/` from +/// `host.openshell.internal` on the port in `argv[1]` and prints the +/// response. A failed connection is retried for 20 seconds, so a slow +/// start doesn't fail the test, and reported on stderr. +const CLIENT: &str = r#" +import socket, sys, time + +print(f"prefix={sys.prefix}", flush=True) +host, port = "host.openshell.internal", int(sys.argv[1]) +deadline = time.monotonic() + 20 +while True: + try: + conn = socket.create_connection((host, port), timeout=10) + break + except OSError as error: + print(f"connect {host}:{port}: {error}", file=sys.stderr, flush=True) + if time.monotonic() > deadline: + sys.exit(1) + time.sleep(1) +with conn: + conn.sendall(f"GET / HTTP/1.1\r\nHost: {host}:{port}\r\nConnection: close\r\n\r\n".encode()) + response = b"" + while chunk := conn.recv(4096): + response += chunk +print(response.decode(errors="replace")) +"#; + +/// A sandbox from the Python test image resolves `host.openshell.internal` +/// and gets a response from a server the test runs on the host. The image's +/// virtual environment runs the client, so its workspace content reached the +/// sandbox, and the policy lets only that Python reach the server's port. +#[testkit::test] +async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result { + let marker = format!("host-gateway-{}", cx.run_id()); + let server = HostServer::start(marker.clone()).await?; + let policy = PolicyFile::new(&policy(server.port))?; + let port = server.port.to_string(); + let image = testkit::python_image(); + + let name = cx.name("hg"); + cx.track(&name); + let create = cx + .cli([ + "sandbox", + "create", + "--name", + &name, + "--from", + &image, + "--policy", + policy.path()?, + "--no-tty", + "--", + VENV_PYTHON, + "-c", + CLIENT, + &port, + ]) + .await? + .success() + .context(format!("sandbox from {image} runs the client"))?; + testkit::ensure!( + create.stdout.contains("prefix=/sandbox/.venv"), + "the client did not run in the image's virtual environment:\n{}", + create.stdout + ); + testkit::ensure!( + create.stdout.contains(&marker), + "the client did not get the host server's response:\n{}{}", + create.stdout, + create.stderr + ); + Ok(()) +} + +/// A sandbox policy for the Python test image that allows its uv-managed +/// Python, which [`VENV_PYTHON`] links to, to connect to +/// `host.openshell.internal` on `port`. The endpoint names only the host and +/// port, as the policy docs describe, so the driver's address for the name +/// is what makes it reachable, not an `allowed_ips` range. +fn policy(port: u16) -> String { + format!( + r#"version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /lib64, /proc, /dev/urandom, /etc] + read_write: [/sandbox, /tmp, /dev/null] +landlock: + compatibility: best_effort + +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + host_server: + name: host_server + endpoints: + - host: host.openshell.internal + port: {port} + binaries: + - path: /sandbox/.uv/python/**/python* +"# + ) +} + +/// An HTTP server on the host that answers every request with `body`. It +/// stops when dropped. +struct HostServer { + port: u16, + task: JoinHandle<()>, +} + +impl HostServer { + /// Listen on every host address, since drivers map + /// `host.openshell.internal` to different ones. + async fn start(body: String) -> Result { + let listener = TcpListener::bind(("0.0.0.0", 0)) + .await + .map_err(|error| Error::from_error(&error)) + .context("bind the host server")?; + let port = listener + .local_addr() + .map_err(|error| Error::from_error(&error)) + .context("read the host server's address")? + .port(); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + tokio::spawn(respond(stream, body.clone())); + } + }); + Ok(Self { port, task }) + } +} + +impl Drop for HostServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +/// Read one request's headers from `stream` and answer with `body`. +async fn respond(mut stream: TcpStream, body: String) { + let _ = stream.set_nodelay(true); + let mut request = Vec::new(); + let mut buf = [0_u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + match stream.read(&mut buf).await { + Ok(read) if read > 0 && request.len() < 64 * 1024 => { + request.extend_from_slice(&buf[..read]); + } + _ => return, + } + } + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = stream.write_all(response.as_bytes()).await; + let _ = stream.shutdown().await; +} + +/// A sandbox policy written to a temporary file for `sandbox create --policy`. +struct PolicyFile(tempfile::NamedTempFile); + +impl PolicyFile { + fn new(contents: &str) -> Result { + let file = tempfile::NamedTempFile::new() + .map_err(|error| Error::from_error(&error)) + .context("create policy file")?; + std::fs::write(file.path(), contents) + .map_err(|error| Error::from_error(&error)) + .context(format!("write {}", file.path().display()))?; + Ok(Self(file)) + } + + fn path(&self) -> Result<&str> { + let path = self.0.path(); + path.to_str() + .ok_or_else(|| Error::msg(format!("{} is not UTF-8", path.display()))) + } +} From 2183ead151ad66585524efa83ab5e2e3169e88f0 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 4 Nov 2026 13:33:29 -0800 Subject: [PATCH 5/9] test(tmachine): use the k3s node's IPv4 address for the host gateway The node can list an IPv6 InternalIP after the IPv4 one, which made the address discovery retry until it failed. Signed-off-by: Matthew Grossman --- tests/tmachine/ansible/playbooks/openshell-k3s.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/tmachine/ansible/playbooks/openshell-k3s.yaml b/tests/tmachine/ansible/playbooks/openshell-k3s.yaml index abebd8fc39..70472f584d 100644 --- a/tests/tmachine/ansible/playbooks/openshell-k3s.yaml +++ b/tests/tmachine/ansible/playbooks/openshell-k3s.yaml @@ -63,6 +63,7 @@ # Tests run in the guest, so host.openshell.internal must resolve to the # node itself. QEMU user networking gives every boot the same guest # address (10.0.2.15), so the value stays valid when the disk is reused. + # The node can also list an IPv6 address after it, so take the IPv4 one. - name: Discover the K3s node address ansible.builtin.command: argv: @@ -74,10 +75,16 @@ - --output=jsonpath={.items[0].status.addresses[?(@.type=="InternalIP")].address} register: k3s_node_address changed_when: false - until: k3s_node_address.rc == 0 and k3s_node_address.stdout is match('^[0-9.]+$') + until: >- + k3s_node_address.rc == 0 + and k3s_node_address.stdout.split() | select('match', '^[0-9.]+$') | list | length > 0 retries: 30 delay: 2 + - name: Select the K3s node IPv4 address + ansible.builtin.set_fact: + k3s_node_ipv4: "{{ k3s_node_address.stdout.split() | select('match', '^[0-9.]+$') | first }}" + - name: Write OpenShell Helm values ansible.builtin.copy: dest: /var/lib/openshell/artifacts/values.yaml @@ -107,7 +114,7 @@ auth: allowUnauthenticatedUsers: true disableTls: true - hostGatewayIP: "{{ k3s_node_address.stdout }}" + hostGatewayIP: "{{ k3s_node_ipv4 }}" telemetryEnabled: false - name: Write OpenShell Helm values overlay From 64c4a0d74899797ecd55b0dc35fd9ba4c38fb4c7 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 4 Nov 2026 13:33:30 -0800 Subject: [PATCH 6/9] test(features): run the host-gateway client with sandbox exec On k3s-ha, attaching to a main process that already finished can reach a gateway replica without the sandbox's session and fail with "sandbox is not ready". Run the client in a Ready sandbox instead. Signed-off-by: Matthew Grossman --- .../host-gateway/tests/host_gateway.rs | 52 +++++++++++++------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/tests/suites/features/host-gateway/tests/host_gateway.rs b/tests/suites/features/host-gateway/tests/host_gateway.rs index 66af363071..5ee1ba11d0 100644 --- a/tests/suites/features/host-gateway/tests/host_gateway.rs +++ b/tests/suites/features/host-gateway/tests/host_gateway.rs @@ -5,9 +5,9 @@ //! compute driver. //! //! The test starts an HTTP server where the tests run, which is the host the -//! gateway runs its sandboxes on, and creates a sandbox from the Python test -//! image ([`testkit::python_image`]) whose main process requests a page from -//! that server through the name. +//! gateway runs its sandboxes on, and runs a client in a sandbox from the +//! Python test image ([`testkit::python_image`]) that requests a page from that +//! server through the name. use testkit::{Context, Error, Result, ResultExt as _}; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; @@ -57,18 +57,40 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result let port = server.port.to_string(); let image = testkit::python_image(); + // The client runs through `sandbox exec` in a Ready sandbox, not as the + // main process. A gateway with several replicas (k3s-ha) can refuse to + // attach to a main process that already finished with "sandbox is not + // ready" when the request reaches a replica that doesn't hold the + // sandbox's session. let name = cx.name("hg"); cx.track(&name); - let create = cx + cx.cli([ + "sandbox", + "create", + "--name", + &name, + "--from", + &image, + "--policy", + policy.path()?, + "--detach", + "--no-tty", + "--", + "sh", + "-c", + "exec sleep infinity", + ]) + .await? + .success() + .context(format!("create a sandbox from {image}"))?; + cx.adopt(&name).await?.wait_ready().await?; + + let client = cx .cli([ "sandbox", - "create", + "exec", "--name", &name, - "--from", - &image, - "--policy", - policy.path()?, "--no-tty", "--", VENV_PYTHON, @@ -78,17 +100,17 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result ]) .await? .success() - .context(format!("sandbox from {image} runs the client"))?; + .context("run the client in the sandbox")?; testkit::ensure!( - create.stdout.contains("prefix=/sandbox/.venv"), + client.stdout.contains("prefix=/sandbox/.venv"), "the client did not run in the image's virtual environment:\n{}", - create.stdout + client.stdout ); testkit::ensure!( - create.stdout.contains(&marker), + client.stdout.contains(&marker), "the client did not get the host server's response:\n{}{}", - create.stdout, - create.stderr + client.stdout, + client.stderr ); Ok(()) } From a5fef8b01336e89e55f4e9077a2d31df1b033d76 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 4 Nov 2026 22:27:32 -0800 Subject: [PATCH 7/9] Revert "test(runner): honor family nextest config and add --test-threads" This reverts commit 8441388e50a6dbac153b5c9b4a8b18c804bfc9d0. Signed-off-by: Matthew Grossman --- TESTING.md | 18 ------- tests/runner/src/bundle.rs | 59 +++------------------- tests/runner/src/execute.rs | 98 ++++++++++++------------------------- tests/runner/src/main.rs | 7 --- 4 files changed, 36 insertions(+), 146 deletions(-) diff --git a/TESTING.md b/TESTING.md index a697ffd0fb..84f3f6a1d8 100644 --- a/TESTING.md +++ b/TESTING.md @@ -247,24 +247,6 @@ need different `TMACHINE_SSH_PORT` values (default 2222). The installer inputs, such as the Debian package and the images, come from `artifacts/`, built by `nix run .#build-artifacts` or downloaded by CI. -The runner's nextest profile, `openshell`, runs tests one at a time and -fails a test that runs longer than two minutes (`slow-timeout` 30 s × 4). A -family that needs other settings puts them in -`tests/suites//.config/nextest.toml`, which takes priority over the -runner's profile: - -- `[[profile.default.overrides]]` for some tests, such as a longer - `slow-timeout` or a `test-group` from `[test-groups]` that keeps tests from - running at the same time. These also apply when you run the suite with - `cargo nextest run`. -- `[profile.openshell]` for every test of the family in the runner, such as - its `slow-timeout`. - -`--test-threads ` runs N tests at once. Tests share the gateway and the -guest, so it defaults to 1; raise it only for tests that tolerate that, and -put the ones that don't in a test group. A serial run streams test output into -`nextest.log`; a parallel run shows a test's output only when it fails. - A flaky or broken test is marked `#[ignore = ""]`, which takes it out of every environment until it's fixed. diff --git a/tests/runner/src/bundle.rs b/tests/runner/src/bundle.rs index d5eba208ab..08e66738c6 100644 --- a/tests/runner/src/bundle.rs +++ b/tests/runner/src/bundle.rs @@ -4,16 +4,10 @@ //! Building a family's test bundle for the guest, on the host. //! //! A bundle is a directory holding a nextest archive built for the guest target, -//! the family workspace's `Cargo.toml` files (for `--workspace-remap`), the -//! runner's nextest profile, and `testkit-ready`, which waits for the gateway -//! before the tests run. The guest needs only the `cargo-nextest` binary to run -//! it. -//! -//! The runner's profile is a nextest tool config, below the repository config -//! in priority. A family that needs other settings, such as longer timeouts for -//! some tests or a test group that runs serially, puts them in -//! `tests/suites//.config/nextest.toml`; the bundle carries it to the -//! same place in the remapped workspace, where nextest reads it. +//! the family workspace's `Cargo.toml` files (for `--workspace-remap`), a +//! `nextest` config with the runner's `JUnit` profile, and `testkit-ready`, +//! which waits for the gateway before the tests run. The guest needs only the +//! `cargo-nextest` binary to run it. use std::path::{Path, PathBuf}; use std::process::Command; @@ -28,20 +22,9 @@ pub const PROFILE: &str = "openshell"; /// The readiness check every bundle carries, at the bundle root. pub const READY_BINARY: &str = "testkit-ready"; -/// The runner's nextest config, at the bundle root. It is passed as the tool -/// config of [`TOOL_NAME`]. -pub const TOOL_CONFIG: &str = "runner-nextest.toml"; - -/// The tool name nextest gives the runner's config. -pub const TOOL_NAME: &str = "openshell-test-runner"; - -/// A family's nextest config, relative to its workspace and the bundle root. -const FAMILY_CONFIG: &str = ".config/nextest.toml"; - /// Nextest flags a test as slow after the period and terminates it after /// `terminate-after` periods, so a hung test fails after two minutes instead -/// of holding the lane until the job times out. A family's own config takes -/// priority over this. +/// of holding the lane until the job times out. const NEXTEST_CONFIG: &str = r#"# Written by the OpenShell test runner. [profile.openshell] fail-fast = false @@ -151,26 +134,12 @@ pub fn build(workspace: &Path, packages: &[String], target: &str, ready: &Path) run(&mut command).with_context(|| format!("build the {} test archive", workspace.display()))?; copy_manifests(workspace, stage)?; - copy_family_config(workspace, stage)?; - std::fs::write(stage.join(TOOL_CONFIG), NEXTEST_CONFIG)?; + std::fs::write(stage.join("nextest.toml"), NEXTEST_CONFIG)?; std::fs::copy(ready, stage.join(READY_BINARY)) .with_context(|| format!("copy {}", ready.display()))?; Ok(Bundle(dir)) } -/// Copy the family's nextest config, if it has one, to the same place in -/// `stage`. -fn copy_family_config(workspace: &Path, stage: &Path) -> Result<()> { - let config = workspace.join(FAMILY_CONFIG); - if !config.is_file() { - return Ok(()); - } - let dest = stage.join(FAMILY_CONFIG); - std::fs::create_dir_all(dest.parent().unwrap_or(stage))?; - std::fs::copy(&config, &dest).with_context(|| format!("copy {}", config.display()))?; - Ok(()) -} - /// Copy every `Cargo.toml` under `workspace` into `stage`, keeping relative /// paths and skipping build output. fn copy_manifests(workspace: &Path, stage: &Path) -> Result<()> { @@ -219,20 +188,4 @@ mod tests { ); assert_eq!(built_executable(messages, "other"), None); } - - #[test] - fn family_config_lands_where_nextest_reads_it() { - let workspace = tempfile::tempdir().unwrap(); - let stage = tempfile::tempdir().unwrap(); - copy_family_config(workspace.path(), stage.path()).unwrap(); - assert!(!stage.path().join(".config").exists()); - - std::fs::create_dir(workspace.path().join(".config")).unwrap(); - std::fs::write(workspace.path().join(FAMILY_CONFIG), "[test-groups]\n").unwrap(); - copy_family_config(workspace.path(), stage.path()).unwrap(); - assert_eq!( - std::fs::read_to_string(stage.path().join(".config/nextest.toml")).unwrap(), - "[test-groups]\n" - ); - } } diff --git a/tests/runner/src/execute.rs b/tests/runner/src/execute.rs index 10353eb177..8c46dbe103 100644 --- a/tests/runner/src/execute.rs +++ b/tests/runner/src/execute.rs @@ -12,7 +12,6 @@ //! [`crate::test_env`]). use std::fmt; -use std::num::NonZeroUsize; use std::path::Path; use std::process::{ExitCode, ExitStatus, Output}; use std::time::Duration; @@ -21,7 +20,7 @@ use anyhow::{Context, Result, ensure}; use tmachine::Session; use tmachine::config::Config; -use crate::bundle::{Bundle, PROFILE, READY_BINARY, TOOL_CONFIG, TOOL_NAME}; +use crate::bundle::{Bundle, PROFILE, READY_BINARY}; use crate::test_env; const GUEST_DIR: &str = "/var/tmp/openshell-tests"; @@ -115,8 +114,6 @@ pub struct Run<'a> { pub bundle: &'a Bundle, /// A nextest filterset (`-E`) narrowing the tests in the bundle. pub filter: Option<&'a str>, - /// How many tests run at once. - pub test_threads: NonZeroUsize, /// Where `junit.xml`, `nextest.log` and, after a failure, the guest's /// `diagnostics/` go. pub results: &'a Path, @@ -285,10 +282,37 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { )?; } + let archive = format!("{guest}/tests.tar.zst"); + let extract = format!("{guest}/extract"); + let config = format!("{guest}/nextest.toml"); let mut nextest = vec!["env"]; nextest.extend(env.iter().map(String::as_str)); - let args = nextest_args(guest, run.test_threads, run.filter); - nextest.extend(args.iter().map(String::as_str)); + nextest.extend([ + "cargo-nextest", + "nextest", + "run", + "--archive-file", + &archive, + "--workspace-remap", + guest, + "--extract-to", + &extract, + "--extract-overwrite", + "--config-file", + &config, + "--profile", + PROFILE, + "--no-capture", + // Tests share one gateway and the guest has 4 GiB; run them serially + // whatever --no-capture implies. + "--test-threads", + "1", + "--no-tests", + "fail", + ]); + if let Some(filter) = run.filter { + nextest.extend(["-E", filter]); + } // nextest writes its output to a file in the guest, so a run that times // out or loses SSH still leaves a log for `collect_results`. The guest // shell exports the published variables itself, so their values, which @@ -307,47 +331,6 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { )) } -/// The `cargo-nextest` command line that runs the bundle unpacked at `guest`. -/// -/// The runner's profile is a tool config, so the family's -/// `.config/nextest.toml` in the remapped workspace overrides it. Tests share -/// one gateway and the guest has 4 GiB, so they run one at a time unless the -/// run asks for more threads. A serial run uses `--no-capture`, which runs -/// tests one at a time and streams their output into the log. A parallel run -/// can't, so nextest captures each test's output and shows it when the test -/// fails. -fn nextest_args(guest: &str, test_threads: NonZeroUsize, filter: Option<&str>) -> Vec { - let mut args: Vec = [ - "cargo-nextest", - "nextest", - "run", - "--archive-file", - &format!("{guest}/tests.tar.zst"), - "--workspace-remap", - guest, - "--extract-to", - &format!("{guest}/extract"), - "--extract-overwrite", - "--tool-config-file", - &format!("{TOOL_NAME}:{guest}/{TOOL_CONFIG}"), - "--profile", - PROFILE, - "--no-tests", - "fail", - ] - .map(String::from) - .into(); - if test_threads.get() == 1 { - args.push("--no-capture".to_string()); - } else { - args.extend(["--test-threads".to_string(), test_threads.to_string()]); - } - if let Some(filter) = filter { - args.extend(["-E".to_string(), filter.to_string()]); - } - args -} - /// Copy the nextest log and the JUnit report out of the guest, whether or not /// the run finished. Either may be missing, for example when the gateway never /// became ready. @@ -542,27 +525,6 @@ mod tests { } } - #[test] - fn nextest_runs_serially_with_output_unless_asked_for_threads() { - let one = NonZeroUsize::MIN; - let serial = nextest_args("/g", one, None); - assert!(serial.contains(&"--no-capture".to_string())); - // nextest ignores --test-threads with --no-capture, and warns. - assert!(!serial.contains(&"--test-threads".to_string())); - assert!(serial.windows(2).any(|w| w - == [ - "--tool-config-file", - "openshell-test-runner:/g/runner-nextest.toml" - ])); - assert!(!serial.contains(&"--config-file".to_string())); - assert!(!serial.contains(&"-E".to_string())); - - let parallel = nextest_args("/g", NonZeroUsize::new(4).unwrap(), Some("test(a)")); - assert!(!parallel.contains(&"--no-capture".to_string())); - assert!(parallel.windows(2).any(|w| w == ["--test-threads", "4"])); - assert!(parallel.ends_with(&["-E".to_string(), "test(a)".to_string()])); - } - #[test] fn exit_code_separates_test_failures_from_errors() { assert_eq!(Status::Passed.exit_code(), ExitCode::SUCCESS); diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index b55c487c70..1503a4e0d7 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -7,7 +7,6 @@ //! provides the guest cross-compilation toolchain, QEMU, Ansible, and the //! tmachine configuration. -use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; use std::process::ExitCode; @@ -42,11 +41,6 @@ struct Cli { /// `binary(smoke)`. #[arg(long, value_name = "FILTERSET")] filter: Option, - /// How many tests run at once. Tests share the gateway and the guest, so - /// raise it only for tests that tolerate that; nextest then captures each - /// test's output and shows it when the test fails. - #[arg(long, default_value = "1", value_name = "N")] - test_threads: NonZeroUsize, /// The gateway name the installer registers with the CLI. The packaged /// installers register `openshell`. #[arg(long, default_value = "openshell")] @@ -108,7 +102,6 @@ async fn run(cli: Cli) -> Result { gateway: &cli.gateway, bundle: &bundle, filter: cli.filter.as_deref(), - test_threads: cli.test_threads, results: &results, keep: cli.keep, }, From ce41fc0e0a2327a2c78e18c5b7592c8b577a16fd Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 5 Nov 2026 10:15:24 -0800 Subject: [PATCH 8/9] test(features): use the system Python for the host-gateway client and trim exports The client needs only the standard library, so run /usr/bin/python3 instead of the image's uv virtual environment and let the policy name its real path. Keep the new image constants private to testkit, and fail the k3s installer if the cluster ever has more than one node, since the host gateway address is the node the tests run on. Refs #5275 Signed-off-by: Matthew Grossman --- .../host-gateway/tests/host_gateway.rs | 50 +++++++------------ tests/testkit/testkit/src/image.rs | 4 +- tests/testkit/testkit/src/lib.rs | 5 +- .../ansible/playbooks/openshell-k3s.yaml | 14 +++++- 4 files changed, 32 insertions(+), 41 deletions(-) diff --git a/tests/suites/features/host-gateway/tests/host_gateway.rs b/tests/suites/features/host-gateway/tests/host_gateway.rs index 5ee1ba11d0..024b1cd84b 100644 --- a/tests/suites/features/host-gateway/tests/host_gateway.rs +++ b/tests/suites/features/host-gateway/tests/host_gateway.rs @@ -14,18 +14,17 @@ use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; use tokio::net::{TcpListener, TcpStream}; use tokio::task::JoinHandle; -/// The Python test image's virtual environment, which links to a uv-managed -/// Python under `/sandbox/.uv/python`. -const VENV_PYTHON: &str = "/sandbox/.venv/bin/python3"; - -/// Prints the interpreter's prefix, then requests `/` from -/// `host.openshell.internal` on the port in `argv[1]` and prints the -/// response. A failed connection is retried for 20 seconds, so a slow -/// start doesn't fail the test, and reported on stderr. +/// The image's system Python, which needs nothing beyond its standard library. +/// The policy names its real path, `/usr/bin/python3.*`, because a rule matches +/// the executable behind a symlink. +const PYTHON: &str = "/usr/bin/python3"; + +/// Requests `/` from `host.openshell.internal` on the port in `argv[1]` and +/// prints the response. A failed connection is retried for 20 seconds, so a +/// slow start doesn't fail the test, and reported on stderr. const CLIENT: &str = r#" import socket, sys, time -print(f"prefix={sys.prefix}", flush=True) host, port = "host.openshell.internal", int(sys.argv[1]) deadline = time.monotonic() + 20 while True: @@ -46,9 +45,9 @@ print(response.decode(errors="replace")) "#; /// A sandbox from the Python test image resolves `host.openshell.internal` -/// and gets a response from a server the test runs on the host. The image's -/// virtual environment runs the client, so its workspace content reached the -/// sandbox, and the policy lets only that Python reach the server's port. +/// and gets a response from a server the test runs on the host. The default +/// policy denies all network access, so the test passes one that lets only the +/// image's Python reach the server's port. #[testkit::test] async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result { let marker = format!("host-gateway-{}", cx.run_id()); @@ -87,25 +86,11 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result let client = cx .cli([ - "sandbox", - "exec", - "--name", - &name, - "--no-tty", - "--", - VENV_PYTHON, - "-c", - CLIENT, - &port, + "sandbox", "exec", "--name", &name, "--no-tty", "--", PYTHON, "-c", CLIENT, &port, ]) .await? .success() .context("run the client in the sandbox")?; - testkit::ensure!( - client.stdout.contains("prefix=/sandbox/.venv"), - "the client did not run in the image's virtual environment:\n{}", - client.stdout - ); testkit::ensure!( client.stdout.contains(&marker), "the client did not get the host server's response:\n{}{}", @@ -115,11 +100,10 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result Ok(()) } -/// A sandbox policy for the Python test image that allows its uv-managed -/// Python, which [`VENV_PYTHON`] links to, to connect to -/// `host.openshell.internal` on `port`. The endpoint names only the host and -/// port, as the policy docs describe, so the driver's address for the name -/// is what makes it reachable, not an `allowed_ips` range. +/// A sandbox policy that allows the image's system Python ([`PYTHON`]) to +/// connect to `host.openshell.internal` on `port`. The endpoint names only the +/// host and port, as the policy docs describe, so the driver's address for the +/// name is what makes it reachable, not an `allowed_ips` range. fn policy(port: u16) -> String { format!( r#"version: 1 @@ -142,7 +126,7 @@ network_policies: - host: host.openshell.internal port: {port} binaries: - - path: /sandbox/.uv/python/**/python* + - path: /usr/bin/python3.* "# ) } diff --git a/tests/testkit/testkit/src/image.rs b/tests/testkit/testkit/src/image.rs index 06952a82cf..c52035c5c9 100644 --- a/tests/testkit/testkit/src/image.rs +++ b/tests/testkit/testkit/src/image.rs @@ -10,12 +10,12 @@ pub const BASE_IMAGE_ENV: &str = "OPENSHELL_TEST_BASE_IMAGE"; pub const DEFAULT_BASE_IMAGE: &str = "nvcr.io/nvidia/base/ubuntu:24.04"; /// Overrides [`DEFAULT_PYTHON_IMAGE`]. -pub const PYTHON_IMAGE_ENV: &str = "OPENSHELL_TEST_PYTHON_IMAGE"; +const PYTHON_IMAGE_ENV: &str = "OPENSHELL_TEST_PYTHON_IMAGE"; /// The workload image for tests that run `python3`, curl, git or openssl in a /// sandbox, built from `e2e/python/Dockerfile.workload`. It isn't published: /// the tmachine installers load it into every environment, and /// `mise run e2e:workload:build` builds it locally. -pub const DEFAULT_PYTHON_IMAGE: &str = "openshell/e2e-python:dev"; +const DEFAULT_PYTHON_IMAGE: &str = "openshell/e2e-python:dev"; /// The image named by [`BASE_IMAGE_ENV`], or [`DEFAULT_BASE_IMAGE`]. pub fn base_image() -> String { diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 7b5896349e..afcb0158e2 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -39,10 +39,7 @@ pub use context::{ }; pub use error::{Error, Result, ResultExt}; pub use gateway::Gateway; -pub use image::{ - BASE_IMAGE_ENV, DEFAULT_BASE_IMAGE, DEFAULT_PYTHON_IMAGE, PYTHON_IMAGE_ENV, base_image, - python_image, -}; +pub use image::{BASE_IMAGE_ENV, DEFAULT_BASE_IMAGE, base_image, python_image}; pub use openshell_sdk as sdk; pub use poll::{Poll, poll_until}; pub use run::{TestInfo, run}; diff --git a/tests/tmachine/ansible/playbooks/openshell-k3s.yaml b/tests/tmachine/ansible/playbooks/openshell-k3s.yaml index 70472f584d..5b68bc0067 100644 --- a/tests/tmachine/ansible/playbooks/openshell-k3s.yaml +++ b/tests/tmachine/ansible/playbooks/openshell-k3s.yaml @@ -61,8 +61,11 @@ mode: "0600" # Tests run in the guest, so host.openshell.internal must resolve to the - # node itself. QEMU user networking gives every boot the same guest - # address (10.0.2.15), so the value stays valid when the disk is reused. + # node itself. A tmachine environment is one VM, so the cluster has one + # node; the check below fails if that stops being true, because then the + # address must be the one of the node the tests run on. QEMU user + # networking gives every boot the same guest address (10.0.2.15), so the + # value stays valid when the disk is reused. # The node can also list an IPv6 address after it, so take the IPv4 one. - name: Discover the K3s node address ansible.builtin.command: @@ -81,6 +84,13 @@ retries: 30 delay: 2 + - name: Check the K3s cluster has a single node + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --request-timeout=10s, get, nodes, --output=name] + register: k3s_nodes + changed_when: false + failed_when: k3s_nodes.rc != 0 or k3s_nodes.stdout_lines | length != 1 + - name: Select the K3s node IPv4 address ansible.builtin.set_fact: k3s_node_ipv4: "{{ k3s_node_address.stdout.split() | select('match', '^[0-9.]+$') | first }}" From b2dadcdd20807f5459d080c79f31167fd09d2fd4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 5 Nov 2026 10:40:34 -0800 Subject: [PATCH 9/9] refactor(testkit): stop exporting environment variable names Tests read the image, CLI binary and gateway settings through functions such as base_image() and python_image(). Nothing outside testkit used the variable-name constants, so keep them private. Refs #5275 Signed-off-by: Matthew Grossman --- tests/testkit/testkit/src/cli.rs | 2 +- tests/testkit/testkit/src/gateway.rs | 6 +++--- tests/testkit/testkit/src/image.rs | 4 ++-- tests/testkit/testkit/src/lib.rs | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/testkit/testkit/src/cli.rs b/tests/testkit/testkit/src/cli.rs index abba6065c3..5ac54e4108 100644 --- a/tests/testkit/testkit/src/cli.rs +++ b/tests/testkit/testkit/src/cli.rs @@ -21,7 +21,7 @@ use crate::gateway::{ENDPOINT_ENV, GATEWAY_ENV, Gateway, WORKSPACE_ENV}; /// How long one CLI invocation may take. pub const CLI_TIMEOUT: Duration = Duration::from_mins(5); /// The CLI binary to run. Defaults to `openshell` on PATH. -pub const BIN_ENV: &str = "OPENSHELL_BIN"; +pub(crate) const BIN_ENV: &str = "OPENSHELL_BIN"; /// Runs the CLI against the gateway and workspace the tests use. #[derive(Clone, Debug)] diff --git a/tests/testkit/testkit/src/gateway.rs b/tests/testkit/testkit/src/gateway.rs index a3c27341a2..cf2dc1a3f9 100644 --- a/tests/testkit/testkit/src/gateway.rs +++ b/tests/testkit/testkit/src/gateway.rs @@ -21,9 +21,9 @@ use openshell_sdk::{ClientConfig, ClientIdentity, Health, OpenShellClient, Servi use crate::error::{Error, Result, ResultExt}; -pub const ENDPOINT_ENV: &str = "OPENSHELL_GATEWAY_ENDPOINT"; -pub const GATEWAY_ENV: &str = "OPENSHELL_GATEWAY"; -pub const WORKSPACE_ENV: &str = "OPENSHELL_WORKSPACE"; +pub(crate) const ENDPOINT_ENV: &str = "OPENSHELL_GATEWAY_ENDPOINT"; +pub(crate) const GATEWAY_ENV: &str = "OPENSHELL_GATEWAY"; +pub(crate) const WORKSPACE_ENV: &str = "OPENSHELL_WORKSPACE"; const DEFAULT_WORKSPACE: &str = "default"; // Mirrors the SDK's endpoint settings for channels testkit builds itself. diff --git a/tests/testkit/testkit/src/image.rs b/tests/testkit/testkit/src/image.rs index c52035c5c9..26bb2278fe 100644 --- a/tests/testkit/testkit/src/image.rs +++ b/tests/testkit/testkit/src/image.rs @@ -4,10 +4,10 @@ //! The images tests create sandboxes from or build their images on. /// Overrides [`DEFAULT_BASE_IMAGE`], e.g. with a mirror. -pub const BASE_IMAGE_ENV: &str = "OPENSHELL_TEST_BASE_IMAGE"; +const BASE_IMAGE_ENV: &str = "OPENSHELL_TEST_BASE_IMAGE"; /// OpenShell's built-in default sandbox image. Pulling it needs access to /// nvcr.io. -pub const DEFAULT_BASE_IMAGE: &str = "nvcr.io/nvidia/base/ubuntu:24.04"; +const DEFAULT_BASE_IMAGE: &str = "nvcr.io/nvidia/base/ubuntu:24.04"; /// Overrides [`DEFAULT_PYTHON_IMAGE`]. const PYTHON_IMAGE_ENV: &str = "OPENSHELL_TEST_PYTHON_IMAGE"; diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index afcb0158e2..ae09cf1543 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -32,14 +32,14 @@ mod poll; mod run; pub use cleanup::CLEANUP_TIMEOUT; -pub use cli::{BIN_ENV, CLI_TIMEOUT, Cli, CliOutput}; +pub use cli::{CLI_TIMEOUT, Cli, CliOutput}; pub use context::{ Context, DELETE_TIMEOUT, EXEC_TIMEOUT, ExecOutput, READY_TIMEOUT, STOP_TIMEOUT, Sandbox, SandboxBuilder, }; pub use error::{Error, Result, ResultExt}; pub use gateway::Gateway; -pub use image::{BASE_IMAGE_ENV, DEFAULT_BASE_IMAGE, base_image, python_image}; +pub use image::{base_image, python_image}; pub use openshell_sdk as sdk; pub use poll::{Poll, poll_until}; pub use run::{TestInfo, run};