From 65fdb0b6090670ef55c371d23b05b615e9047376 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 2 Nov 2026 09:17:12 -0800 Subject: [PATCH 1/5] feat(testkit): register tests and add a standalone test program #[testkit::test] now also registers each test in a link-time registry, alongside the existing #[tokio::test] wrapper, so suites keep running under cargo test and nextest unchanged. testkit::main() runs the registered tests a binary links. Without a subcommand it speaks libtest's interface, so a harness = false target works under cargo test and nextest. Its subcommands are: - list: print each test's name and description - ready: wait for the gateway, sharing its check with testkit-ready, and exit 2 if it is not ready in time - run (Unix only): run tests one at a time, each in its own process group with a timeout (SIGTERM, grace, SIGKILL), write a JUnit report, and exit 1 when a test failed or 2 when tests could not run A stopped test does not clean up its sandboxes yet; the README notes this. Signed-off-by: Matthew Grossman --- tests/suites/conformance/Cargo.lock | 208 ++++++++ tests/suites/features/Cargo.lock | 208 ++++++++ tests/support/Cargo.lock | 208 ++++++++ tests/testkit/Cargo.lock | 209 ++++++++ tests/testkit/README.md | 54 +- tests/testkit/testkit-macros/src/lib.rs | 212 +++++++- tests/testkit/testkit/Cargo.toml | 16 + .../testkit/testkit/src/bin/testkit-ready.rs | 29 +- tests/testkit/testkit/src/gateway.rs | 17 + tests/testkit/testkit/src/lib.rs | 12 + tests/testkit/testkit/src/program/capture.rs | 109 ++++ tests/testkit/testkit/src/program/junit.rs | 128 +++++ tests/testkit/testkit/src/program/mod.rs | 374 +++++++++++++ tests/testkit/testkit/src/program/process.rs | 216 ++++++++ .../testkit/testkit/src/program/supervisor.rs | 334 ++++++++++++ tests/testkit/testkit/src/ready.rs | 36 ++ tests/testkit/testkit/src/registry.rs | 208 ++++++++ tests/testkit/testkit/tests/attribute.rs | 116 ++++ tests/testkit/testkit/tests/program.rs | 504 ++++++++++++++++++ 19 files changed, 3147 insertions(+), 51 deletions(-) create mode 100644 tests/testkit/testkit/src/program/capture.rs create mode 100644 tests/testkit/testkit/src/program/junit.rs create mode 100644 tests/testkit/testkit/src/program/mod.rs create mode 100644 tests/testkit/testkit/src/program/process.rs create mode 100644 tests/testkit/testkit/src/program/supervisor.rs create mode 100644 tests/testkit/testkit/src/ready.rs create mode 100644 tests/testkit/testkit/src/registry.rs create mode 100644 tests/testkit/testkit/tests/attribute.rs create mode 100644 tests/testkit/testkit/tests/program.rs diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index f45f1866ee..ea04cd150d 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -54,6 +54,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -382,6 +432,46 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmake" version = "0.1.58" @@ -397,6 +487,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -679,6 +775,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + [[package]] name = "etcetera" version = "0.11.0" @@ -1395,6 +1497,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.14.0" @@ -1641,6 +1749,38 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1758,6 +1898,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "newtype-uuid" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa56da58097c7ae893c4097bb6d91bb5f7fa330053e07e033349db4d5f83ba8" +dependencies = [ + "uuid", +] + [[package]] name = "nix" version = "0.29.0" @@ -1873,6 +2022,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openshell-bootstrap" version = "0.0.0" @@ -1995,11 +2150,16 @@ dependencies = [ name = "openshell-testkit" version = "0.0.0" dependencies = [ + "clap", "futures", + "libc", + "libtest-mimic", + "linkme", "openshell-bootstrap", "openshell-core", "openshell-sdk", "openshell-testkit-macros", + "quick-junit", "rand 0.9.4", "serde", "serde_json", @@ -2357,6 +2517,30 @@ dependencies = [ "pulldown-cmark", ] +[[package]] +name = "quick-junit" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae23b8e1a59a90f7bf98caa2f4c8c2eedf6df011c6c88ed633e85a87c84526e" +dependencies = [ + "chrono", + "indexmap", + "newtype-uuid", + "quick-xml", + "strip-ansi-escapes", + "thiserror 2.0.20", + "uuid", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.45" @@ -3134,6 +3318,15 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3720,6 +3913,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.23.1" @@ -3743,6 +3942,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "want" version = "0.3.1" diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 1206d952e4..12945d3f1f 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -54,6 +54,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -425,6 +475,46 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmake" version = "0.1.58" @@ -440,6 +530,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -721,6 +817,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + [[package]] name = "etcetera" version = "0.11.0" @@ -1477,6 +1579,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.14.0" @@ -1741,6 +1849,38 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1878,6 +2018,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "newtype-uuid" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa56da58097c7ae893c4097bb6d91bb5f7fa330053e07e033349db4d5f83ba8" +dependencies = [ + "uuid", +] + [[package]] name = "nix" version = "0.29.0" @@ -2002,6 +2151,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openshell-bootstrap" version = "0.0.0" @@ -2270,11 +2425,16 @@ dependencies = [ name = "openshell-testkit" version = "0.0.0" dependencies = [ + "clap", "futures", + "libc", + "libtest-mimic", + "linkme", "openshell-bootstrap", "openshell-core", "openshell-sdk", "openshell-testkit-macros", + "quick-junit", "rand 0.9.4", "serde", "serde_json", @@ -2715,6 +2875,30 @@ dependencies = [ "pulldown-cmark", ] +[[package]] +name = "quick-junit" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae23b8e1a59a90f7bf98caa2f4c8c2eedf6df011c6c88ed633e85a87c84526e" +dependencies = [ + "chrono", + "indexmap", + "newtype-uuid", + "quick-xml", + "strip-ansi-escapes", + "thiserror 2.0.20", + "uuid", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.45" @@ -3518,6 +3702,15 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -4173,6 +4366,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.23.1" @@ -4202,6 +4401,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "want" version = "0.3.1" diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index a4ba9ea16c..f85db132ef 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -54,6 +54,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -382,6 +432,46 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmake" version = "0.1.58" @@ -397,6 +487,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -669,6 +765,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + [[package]] name = "etcetera" version = "0.11.0" @@ -1385,6 +1487,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.14.0" @@ -1640,6 +1748,38 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1757,6 +1897,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "newtype-uuid" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa56da58097c7ae893c4097bb6d91bb5f7fa330053e07e033349db4d5f83ba8" +dependencies = [ + "uuid", +] + [[package]] name = "nix" version = "0.29.0" @@ -1872,6 +2021,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openshell-bootstrap" version = "0.0.0" @@ -1994,11 +2149,16 @@ dependencies = [ name = "openshell-testkit" version = "0.0.0" dependencies = [ + "clap", "futures", + "libc", + "libtest-mimic", + "linkme", "openshell-bootstrap", "openshell-core", "openshell-sdk", "openshell-testkit-macros", + "quick-junit", "rand 0.9.4", "serde", "serde_json", @@ -2356,6 +2516,30 @@ dependencies = [ "pulldown-cmark", ] +[[package]] +name = "quick-junit" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae23b8e1a59a90f7bf98caa2f4c8c2eedf6df011c6c88ed633e85a87c84526e" +dependencies = [ + "chrono", + "indexmap", + "newtype-uuid", + "quick-xml", + "strip-ansi-escapes", + "thiserror 2.0.20", + "uuid", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.45" @@ -3133,6 +3317,15 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3719,6 +3912,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.23.1" @@ -3742,6 +3941,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "want" version = "0.3.1" diff --git a/tests/testkit/Cargo.lock b/tests/testkit/Cargo.lock index 6889775edc..689ecea41c 100644 --- a/tests/testkit/Cargo.lock +++ b/tests/testkit/Cargo.lock @@ -54,6 +54,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -382,6 +432,46 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmake" version = "0.1.58" @@ -397,6 +487,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -669,6 +765,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + [[package]] name = "etcetera" version = "0.11.0" @@ -1385,6 +1487,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.14.0" @@ -1631,6 +1739,38 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1748,6 +1888,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "newtype-uuid" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa56da58097c7ae893c4097bb6d91bb5f7fa330053e07e033349db4d5f83ba8" +dependencies = [ + "uuid", +] + [[package]] name = "nix" version = "0.29.0" @@ -1863,6 +2012,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openshell-bootstrap" version = "0.0.0" @@ -1985,14 +2140,20 @@ dependencies = [ name = "openshell-testkit" version = "0.0.0" dependencies = [ + "clap", "futures", + "libc", + "libtest-mimic", + "linkme", "openshell-bootstrap", "openshell-core", "openshell-sdk", "openshell-testkit-macros", + "quick-junit", "rand 0.9.4", "serde", "serde_json", + "tempfile", "tokio", ] @@ -2347,6 +2508,30 @@ dependencies = [ "pulldown-cmark", ] +[[package]] +name = "quick-junit" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae23b8e1a59a90f7bf98caa2f4c8c2eedf6df011c6c88ed633e85a87c84526e" +dependencies = [ + "chrono", + "indexmap", + "newtype-uuid", + "quick-xml", + "strip-ansi-escapes", + "thiserror 2.0.20", + "uuid", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.45" @@ -3124,6 +3309,15 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3710,6 +3904,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.23.1" @@ -3733,6 +3933,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "want" version = "0.3.1" diff --git a/tests/testkit/README.md b/tests/testkit/README.md index ae71d0f81b..b9bd671fd6 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -7,7 +7,7 @@ This is a separate Cargo workspace with its own lockfile: | Crate | Purpose | |---|---| -| `testkit` | Finds and connects to the gateway, creates and cleans up sandboxes, runs the CLI. Also builds `testkit-ready`. | +| `testkit` | Finds and connects to the gateway, creates and cleans up sandboxes, runs the CLI, and runs test programs. Also builds `testkit-ready`. | | `testkit-macros` | The `#[testkit::test]` attribute. | ## Writing a test @@ -99,6 +99,58 @@ In a tmachine VM, the test runner (`tests/runner/`) does the same: it sets `OPENSHELL_BIN` to the installed CLI, then runs the suites with nextest. See [TESTING.md](../../TESTING.md). +## Test programs + +`#[testkit::test]` also registers the test with testkit. A binary whose +`main` calls `testkit::main()` runs every registered test it links, without +cargo or nextest: + +```rust +fn main() -> std::process::ExitCode { + testkit::main() +} +``` + +| Command | What it does | +|---|---| +| `list [FILTER]...` | Prints each test's name and description. Ignored tests are included and marked. | +| `ready [--timeout SECS]` | Waits for the gateway, like `testkit-ready` (default 60 seconds). Exits 2 if it is not ready in time. | +| `run [FILTER]...` | Runs the tests one at a time, each in its own process. Unix only. | + +A filter selects tests whose name contains it. `run` also takes: + +- `--include-ignored`: runs ignored tests too, instead of reporting them as + skipped. +- `--timeout SECS` (default 900): a test that runs longer gets SIGTERM, then + SIGKILL after `--grace SECS` (default 10). +- `--junit PATH`: writes a JUnit report with one test suite per family. + +Each test runs in its own process group. When the test's process exits, `run` +kills anything left in the group, such as CLI processes the test started. +Interrupting `run` stops the current test the same way and runs no more +tests; a second interrupt skips the grace period. If `run` itself is killed +with SIGKILL, the current test's process group keeps running. In a container +where `run` is PID 1, start it under an init, for example with +`docker run --init`. + +`run` exits 0 when every test passed, 1 when a test failed, timed out, or +crashed, and 2 when tests could not run: no tests matched, a test process +could not start or run its test, or the run was interrupted. + +A test that is stopped does not clean up: sandboxes it created stay on the +gateway. Run against a gateway you can throw away, or delete them by hand. + +Without a subcommand, the program takes libtest's arguments instead. A +`[[test]]` target with `harness = false` therefore also runs under `cargo test` +and nextest. In that mode tests run in one process, one at a time unless +`--test-threads` says otherwise. + +A test's name is its module path without the crate name, then its function +name, as in nextest. Its family, which groups tests in the JUnit report, is +the first directory under the last `suites` directory in the crate's path: +`conformance` for `tests/suites/conformance`, `features` for +`tests/suites/features/`. + ## Checks ```shell diff --git a/tests/testkit/testkit-macros/src/lib.rs b/tests/testkit/testkit-macros/src/lib.rs index fc3dbbdb68..b8a614f3c6 100644 --- a/tests/testkit/testkit-macros/src/lib.rs +++ b/tests/testkit/testkit-macros/src/lib.rs @@ -13,12 +13,28 @@ //! async fn expose_service(cx: &mut Context) -> testkit::Result { /* ... */ } //! ``` //! -//! The attribute expands to a `#[tokio::test]` wrapper with the same name, so -//! the nextest test name is the function name. The original function is nested -//! in the wrapper and passed to `testkit::run`. Other attributes, such as -//! `#[ignore]` or `#[cfg(...)]`, move to the wrapper. `#[should_panic]` is -//! rejected: a testkit test panics on any failure, so it would pass for the -//! wrong reason. Check the expected failure in the body instead. +//! The attribute generates three items: +//! +//! - a hidden `async fn` that runs the original function through +//! `testkit::run`, with the original function nested in it; +//! - under `#[cfg(test)]`, a `#[tokio::test]` wrapper with the same name that +//! calls it, so `cargo test` and nextest run the test like any other and the +//! test name is the function name; +//! - an entry in `testkit`'s test registry, with the test's family, name, +//! description, and `#[ignore]`, which `testkit::main()` runs. +//! +//! Other attributes, such as `#[ignore]` or `#[cfg(...)]`, move to the +//! wrapper. `#[cfg(...)]` also applies to the other two items, and lint +//! attributes (`allow`, `warn`, `deny`, `forbid`) also to the hidden +//! function. `#[should_panic]` is rejected: a testkit test panics on any +//! failure, so it would pass for the wrong reason. Check the expected failure +//! in the body instead. `#[cfg_attr]` and `#[expect]` are rejected too. +//! +//! A test's family is the first directory under `tests/suites/` in its +//! crate's path: `conformance` for `tests/suites/conformance`, `features` for +//! `tests/suites/features/`. + +use std::path::Path; use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; @@ -27,12 +43,17 @@ use syn::{Attribute, Expr, ExprLit, ItemFn, Lit, LitStr, Meta}; #[proc_macro_attribute] pub fn test(attr: TokenStream, item: TokenStream) -> TokenStream { - expand(attr.into(), item.into()) + let family = std::env::var_os("CARGO_MANIFEST_DIR").and_then(|dir| family_of(Path::new(&dir))); + expand(attr.into(), item.into(), family.as_deref()) .unwrap_or_else(syn::Error::into_compile_error) .into() } -fn expand(attr: TokenStream2, item: TokenStream2) -> syn::Result { +fn expand( + attr: TokenStream2, + item: TokenStream2, + family: Option<&str>, +) -> syn::Result { let mut body: ItemFn = syn::parse2(item)?; if !attr.is_empty() { return Err(syn::Error::new_spanned( @@ -58,17 +79,24 @@ fn expand(attr: TokenStream2, item: TokenStream2) -> syn::Result { "`#[testkit::test]` already makes this a test; remove this attribute", )); } - if let Some(attr) = body - .attrs - .iter() - .find(|attr| attr.path().is_ident("should_panic")) - { - return Err(syn::Error::new_spanned( - attr, + for attr in &body.attrs { + let message = if attr.path().is_ident("should_panic") { "testkit tests panic on any failure, so `#[should_panic]` would pass for the \ wrong reason; check the expected failure in the body, for example with \ - `CliOutput::failure` and `testkit::ensure!`", - )); + `CliOutput::failure` and `testkit::ensure!`" + } else if attr.path().is_ident("cfg_attr") { + // The registration reads `#[ignore]` and `#[cfg]` directly, so it + // would not see them inside `cfg_attr`. + "`#[cfg_attr]` is not supported on testkit tests; use `#[cfg]` or `#[ignore]` \ + directly" + } else if attr.path().is_ident("expect") { + // The lint attributes are copied to two functions, and only one + // of them would raise the lint. + "`#[expect]` is not supported on testkit tests; use `#[allow]`" + } else { + continue; + }; + return Err(syn::Error::new_spanned(attr, message)); } let description = doc_comment(&body.attrs); if description.is_empty() { @@ -77,20 +105,46 @@ fn expand(attr: TokenStream2, item: TokenStream2) -> syn::Result { "testkit tests need a doc comment describing the behavior they verify", )); } + let ignore = ignore(&body.attrs)?; let name = body.sig.ident.clone(); let name_str = LitStr::new(&name.to_string(), name.span()); let description = LitStr::new(&description, name.span()); + let family = match family { + Some(family) => quote!(::core::option::Option::Some(#family)), + None => quote!(::core::option::Option::None), + }; + let (ignored, ignore_reason) = match ignore { + None => (false, quote!(::core::option::Option::None)), + Some(None) => (true, quote!(::core::option::Option::None)), + Some(Some(reason)) => (true, quote!(::core::option::Option::Some(#reason))), + }; let vis = body.vis.clone(); let attrs = std::mem::take(&mut body.attrs); + let cfgs: Vec<&Attribute> = attrs + .iter() + .filter(|attr| attr.path().is_ident("cfg")) + .collect(); + // `#[cfg]` and lint attributes such as `#[allow(...)]` must reach the + // body, which is nested in the hidden function. Anything else is for the + // test itself, so goes on the wrapper only. + let body_attrs: Vec<&Attribute> = attrs + .iter() + .filter(|attr| { + ["cfg", "allow", "warn", "deny", "forbid"] + .iter() + .any(|name| attr.path().is_ident(name)) + }) + .collect(); body.vis = syn::Visibility::Inherited; body.sig.ident = format_ident!("body"); + let entry = format_ident!("__testkit_{}", name); Ok(quote! { - #(#attrs)* - #[::testkit::__private::tokio::test(crate = "::testkit::__private::tokio")] - #vis async fn #name() { + #(#body_attrs)* + #[doc(hidden)] + async fn #entry() { #body ::testkit::run( ::testkit::TestInfo { @@ -101,6 +155,30 @@ fn expand(attr: TokenStream2, item: TokenStream2) -> syn::Result { ) .await; } + + #(#attrs)* + #[cfg(test)] + #[::testkit::__private::tokio::test(crate = "::testkit::__private::tokio")] + #vis async fn #name() { + #entry().await; + } + + #(#cfgs)* + const _: () = { + #[::testkit::__private::linkme::distributed_slice(::testkit::__private::TESTS)] + #[linkme(crate = ::testkit::__private::linkme)] + static TEST: ::testkit::Test = ::testkit::Test { + info: ::testkit::TestInfo { + name: #name_str, + description: #description, + }, + family: #family, + module_path: ::core::module_path!(), + ignored: #ignored, + ignore_reason: #ignore_reason, + run: || ::std::boxed::Box::pin(#entry()), + }; + }; }) } @@ -112,6 +190,31 @@ fn is_test_attr(attr: &Attribute) -> bool { .is_some_and(|segment| segment.ident == "test") } +/// `None` if the test is not ignored, else the `#[ignore = "..."]` reason if +/// it has one. +fn ignore(attrs: &[Attribute]) -> syn::Result>> { + let Some(attr) = attrs.iter().find(|attr| attr.path().is_ident("ignore")) else { + return Ok(None); + }; + match &attr.meta { + Meta::Path(_) => Ok(Some(None)), + Meta::NameValue(meta) => match &meta.value { + Expr::Lit(ExprLit { + lit: Lit::Str(reason), + .. + }) => Ok(Some(Some(reason.clone()))), + value => Err(syn::Error::new_spanned( + value, + "the `#[ignore]` reason must be a string literal", + )), + }, + Meta::List(_) => Err(syn::Error::new_spanned( + attr, + "use `#[ignore]` or `#[ignore = \"reason\"]`", + )), + } +} + /// The doc comment, joined into one line. Empty if there is none. fn doc_comment(attrs: &[Attribute]) -> String { attrs @@ -131,21 +234,40 @@ fn doc_comment(attrs: &[Attribute]) -> String { .join(" ") } +/// The family of a suite crate in `manifest_dir`: the first directory under +/// the nearest enclosing `suites/`. +fn family_of(manifest_dir: &Path) -> Option { + let components: Vec<_> = manifest_dir + .components() + .filter_map(|component| component.as_os_str().to_str()) + .collect(); + let suites = components + .iter() + .rposition(|component| *component == "suites")?; + components + .get(suites + 1) + .map(|family| (*family).to_string()) +} + #[cfg(test)] mod tests { - use super::expand; + use std::path::Path; + + use super::{expand, family_of}; + + const OK_FN: &str = "async fn t(cx: &mut Context) -> testkit::Result { Ok(()) }"; // A suite only compiles correct tests, so it never reaches these errors. + // `testkit/tests/macro.rs` checks what correct tests expand to. #[test] fn rejects_misuse() { - let ok_fn = "async fn t(cx: &mut Context) -> testkit::Result { Ok(()) }"; for (attr, item, error) in [ ( "id = \"t-1\"", - format!("/// Doc.\n{ok_fn}"), + format!("/// Doc.\n{OK_FN}"), "takes no arguments", ), - ("", ok_fn.to_string(), "doc comment"), + ("", OK_FN.to_string(), "doc comment"), ( "", "/// Doc.\nfn t(cx: &mut Context) -> testkit::Result { Ok(()) }".to_string(), @@ -158,17 +280,53 @@ mod tests { ), ( "", - format!("/// Doc.\n#[tokio::test]\n{ok_fn}"), + format!("/// Doc.\n#[tokio::test]\n{OK_FN}"), "already makes this a test", ), ( "", - format!("/// Doc.\n#[should_panic]\n{ok_fn}"), + format!("/// Doc.\n#[should_panic]\n{OK_FN}"), "`#[should_panic]`", ), + ( + "", + format!("/// Doc.\n#[ignore(later)]\n{OK_FN}"), + "`#[ignore]` or", + ), + ( + "", + format!("/// Doc.\n#[ignore = 1]\n{OK_FN}"), + "string literal", + ), + ( + "", + format!("/// Doc.\n#[cfg_attr(unix, ignore)]\n{OK_FN}"), + "`#[cfg_attr]` is not supported", + ), + ( + "", + format!("/// Doc.\n#[expect(unused)]\n{OK_FN}"), + "`#[expect]` is not supported", + ), ] { - let err = expand(attr.parse().unwrap(), item.parse().unwrap()).unwrap_err(); + let err = expand(attr.parse().unwrap(), item.parse().unwrap(), None).unwrap_err(); assert!(err.to_string().contains(error), "{item}: {err}"); } } + + #[test] + fn family_is_the_directory_under_suites() { + for (dir, family) in [ + ("/repo/tests/suites/conformance", Some("conformance")), + ( + "/repo/tests/suites/features/provider-refresh/keycloak", + Some("features"), + ), + ("/a/suites/x/tests/suites/drivers/podman", Some("drivers")), + ("/repo/tests/testkit/testkit", None), + ("/repo/tests/suites", None), + ] { + assert_eq!(family_of(Path::new(dir)).as_deref(), family, "{dir}"); + } + } } diff --git a/tests/testkit/testkit/Cargo.toml b/tests/testkit/testkit/Cargo.toml index cfcf0391ee..f8f489c828 100644 --- a/tests/testkit/testkit/Cargo.toml +++ b/tests/testkit/testkit/Cargo.toml @@ -19,8 +19,24 @@ openshell-bootstrap = { path = "../../../crates/openshell-bootstrap" } openshell-core = { path = "../../../crates/openshell-core", default-features = false } openshell-sdk = { path = "../../../crates/openshell-sdk" } openshell-testkit-macros = { path = "../testkit-macros" } +clap = { version = "4", features = ["derive"] } futures = "0.3" +libtest-mimic = "0.8" +linkme = "0.3" +quick-junit = "0.8" rand = "0.9" serde = "1" serde_json = "1" tokio = { version = "1.43", features = ["macros", "process", "rt", "time"] } + +[target.'cfg(unix)'.dependencies] +libc = "0.2" + +[dev-dependencies] +tempfile = "3" + +# A test program, run by `cargo test` and nextest through `testkit::main`'s +# libtest interface, that checks the program's own subcommands. +[[test]] +name = "program" +harness = false diff --git a/tests/testkit/testkit/src/bin/testkit-ready.rs b/tests/testkit/testkit/src/bin/testkit-ready.rs index 4f41b3bed4..03db9d3547 100644 --- a/tests/testkit/testkit/src/bin/testkit-ready.rs +++ b/tests/testkit/testkit/src/bin/testkit-ready.rs @@ -6,13 +6,12 @@ //! Usage: `testkit-ready [TIMEOUT_SECS]` (default 60), with the gateway //! selected as for tests. Run it once per environment before the test //! binaries; exits non-zero with the last error if the gateway is not healthy -//! in time, or if the CLI does not reach it. +//! in time, or if the CLI does not reach it. A test program's `ready` +//! subcommand does the same. use std::process::ExitCode; use std::time::Duration; -use testkit::{Cli, Gateway}; - const DEFAULT_TIMEOUT_SECS: u64 = 60; const USAGE: &str = "usage: testkit-ready [TIMEOUT_SECS] (default 60)"; @@ -32,25 +31,9 @@ async fn main() -> ExitCode { } }, }; - let result = async { - let gateway = Gateway::from_env()?; - eprintln!("testkit-ready: {}", gateway.describe()); - let health = gateway.wait_ready(Duration::from_secs(timeout)).await?; - Cli::from_env(&gateway).check_connected().await?; - testkit::Result::Ok(health) - } - .await; - match result { - Ok(health) => { - eprintln!( - "testkit-ready: gateway {} is healthy and the CLI reaches it", - health.version - ); - ExitCode::SUCCESS - } - Err(error) => { - eprintln!("testkit-ready: {error}"); - ExitCode::FAILURE - } + if testkit::check_ready("testkit-ready", Duration::from_secs(timeout)).await { + ExitCode::SUCCESS + } else { + ExitCode::FAILURE } } diff --git a/tests/testkit/testkit/src/gateway.rs b/tests/testkit/testkit/src/gateway.rs index ef40cb1bf9..cba28b88a4 100644 --- a/tests/testkit/testkit/src/gateway.rs +++ b/tests/testkit/testkit/src/gateway.rs @@ -243,6 +243,23 @@ mod tests { use super::Gateway; + #[test] + fn unsupported_auth_modes_are_named() { + let error = Gateway::registered( + "edge".to_string(), + "https://gateway.example".to_string(), + Some("oidc"), + "/nonexistent".into(), + "default".to_string(), + false, + ) + .expect_err("OIDC is not supported"); + assert!( + error.to_string().contains("gateway edge uses oidc auth"), + "{error}" + ); + } + /// A peer that accepts connections and never answers must not hang the /// readiness wait: the timeout covers the connect and health calls too. #[tokio::test] diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 168fdbe894..e2c49e3caf 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -21,12 +21,19 @@ //! //! Tests find the gateway the same way the `openshell` CLI does (see //! [`gateway`]). +//! +//! `#[testkit::test]` also registers each test, so a binary that calls +//! [`main`] is a test program that lists, runs, and reports its tests without +//! Cargo (see [`program`]). mod cli; mod context; mod error; pub mod gateway; mod poll; +pub mod program; +mod ready; +mod registry; mod run; pub use cli::{BIN_ENV, CLI_TIMEOUT, Cli, CliOutput}; @@ -38,10 +45,15 @@ pub use error::{Error, Result, ResultExt}; pub use gateway::Gateway; pub use openshell_sdk as sdk; pub use poll::{Poll, poll_until}; +pub use program::main; +pub use ready::check as check_ready; +pub use registry::Test; pub use run::{TestInfo, run}; pub use testkit_macros::test; #[doc(hidden)] pub mod __private { + pub use crate::registry::TESTS; + pub use linkme; pub use tokio; } diff --git a/tests/testkit/testkit/src/program/capture.rs b/tests/testkit/testkit/src/program/capture.rs new file mode 100644 index 0000000000..329c22ac68 --- /dev/null +++ b/tests/testkit/testkit/src/program/capture.rs @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Copying a test's output to the console while keeping it for the report. + +use std::collections::VecDeque; +use std::io::{self, Read, Write}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +/// How much of each stream is kept for the report. Output beyond this is +/// still shown; the report keeps the end, where failures are. +const KEPT_BYTES: usize = 4 << 20; + +/// How long to wait for a stream to close after the test's process group is +/// gone. A process that left the group can hold it open. +const CLOSE_TIMEOUT: Duration = Duration::from_secs(5); + +/// One stream being copied. +pub struct Capture { + kept: Arc>, + thread: JoinHandle<()>, +} + +#[derive(Default)] +struct Kept { + bytes: VecDeque, + dropped: usize, +} + +impl Capture { + /// Copy `stream` to `console` on a thread, keeping a copy. + pub fn start( + mut stream: impl Read + Send + 'static, + mut console: impl Write + Send + 'static, + ) -> Self { + let kept = Arc::new(Mutex::new(Kept::default())); + let thread = std::thread::spawn({ + let kept = Arc::clone(&kept); + move || { + let mut buffer = [0; 8192]; + loop { + let read = match stream.read(&mut buffer) { + Ok(0) => return, + Ok(read) => read, + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(_) => return, + }; + let chunk = &buffer[..read]; + let _ = console.write_all(chunk).and_then(|()| console.flush()); + kept.lock() + .unwrap_or_else(PoisonError::into_inner) + .push(chunk); + } + } + }); + Self { kept, thread } + } + + /// The output kept, once the stream closes or after a short wait. + pub fn finish(self) -> Vec { + let deadline = Instant::now() + CLOSE_TIMEOUT; + while !self.thread.is_finished() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + if self.thread.is_finished() { + let _ = self.thread.join(); + } + let kept = self.kept.lock().unwrap_or_else(PoisonError::into_inner); + let mut bytes = Vec::with_capacity(kept.bytes.len() + 64); + if kept.dropped > 0 { + bytes.extend(format!("[{} earlier bytes not kept]\n", kept.dropped).as_bytes()); + } + bytes.extend(&kept.bytes); + bytes + } +} + +impl Kept { + fn push(&mut self, chunk: &[u8]) { + self.bytes.extend(chunk); + let excess = self.bytes.len().saturating_sub(KEPT_BYTES); + self.bytes.drain(..excess); + self.dropped += excess; + } +} + +#[cfg(test)] +mod tests { + use super::{Capture, KEPT_BYTES, Kept}; + + #[test] + fn copies_and_keeps_the_stream() { + let capture = Capture::start(&b"one\ntwo\n"[..], std::io::sink()); + assert_eq!(capture.finish(), b"one\ntwo\n"); + } + + #[test] + fn keeps_the_end_of_long_output() { + let mut kept = Kept::default(); + kept.push(&vec![b'a'; KEPT_BYTES]); + kept.push(b"end"); + assert_eq!(kept.bytes.len(), KEPT_BYTES); + let tail: Vec = kept.bytes.iter().rev().take(4).rev().copied().collect(); + assert_eq!(tail, b"aend"); + assert_eq!(kept.dropped, 3); + } +} diff --git a/tests/testkit/testkit/src/program/junit.rs b/tests/testkit/testkit/src/program/junit.rs new file mode 100644 index 0000000000..0a1e66d648 --- /dev/null +++ b/tests/testkit/testkit/src/program/junit.rs @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The `JUnit` report of a run. +//! +//! One `` per family. Each `` is named by the test name, +//! with the module path as its class name and the description and family as +//! properties. + +use std::collections::BTreeMap; +use std::path::Path; + +use quick_junit::{NonSuccessKind, Property, Report, TestCase, TestCaseStatus, TestSuite}; + +use super::supervisor::{Outcome, TestRun}; + +pub fn report(program: &str, runs: &[TestRun<'_>]) -> Report { + let mut suites: BTreeMap<&str, TestSuite> = BTreeMap::new(); + for run in runs { + let family = run.test.family(); + suites + .entry(family) + .or_insert_with(|| TestSuite::new(family)) + .add_test_case(test_case(run)); + } + let mut report = Report::new(program); + report.add_test_suites(suites.into_values()); + report +} + +fn test_case(run: &TestRun<'_>) -> TestCase { + let status = match &run.outcome { + Outcome::Passed => TestCaseStatus::success(), + Outcome::Skipped => { + let mut status = TestCaseStatus::skipped(); + status.set_message(match run.test.ignore_reason { + Some(reason) => format!("ignored: {reason}"), + None => "ignored".to_string(), + }); + status + } + Outcome::Failed(message) => non_success(NonSuccessKind::Failure, "failure", message), + Outcome::TimedOut(message) => non_success(NonSuccessKind::Failure, "timeout", message), + Outcome::Error(message) => non_success(NonSuccessKind::Error, "error", message), + Outcome::Interrupted => non_success(NonSuccessKind::Error, "interrupted", "interrupted"), + }; + let mut case = TestCase::new(run.test.name(), status); + case.set_classname(run.test.module_path) + .add_property(Property::new("family", run.test.family())) + .add_property(Property::new("description", run.test.info.description)); + if let Some(duration) = run.duration { + case.set_time(duration); + } + if !run.stdout.is_empty() { + case.set_system_out_lossy(&run.stdout); + } + if !run.stderr.is_empty() { + case.set_system_err_lossy(&run.stderr); + } + case +} + +fn non_success(kind: NonSuccessKind, ty: &str, message: &str) -> TestCaseStatus { + let mut status = TestCaseStatus::non_success(kind); + status.set_type(ty).set_message(message); + status +} + +pub fn write(path: &Path, report: &Report) -> std::io::Result<()> { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + std::fs::create_dir_all(parent)?; + } + let xml = report.to_string().map_err(std::io::Error::other)?; + std::fs::write(path, xml) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::report; + use crate::program::supervisor::{Outcome, TestRun}; + use crate::registry::Test; + use crate::registry::tests::test; + + const PASSED: Test = test("sandbox::smoke", "create_exec_delete", false); + const FAILED: Test = test("sandbox::lifecycle", "stop_start", false); + const SKIPPED: Test = Test { + ignore_reason: Some("https://github.com/NVIDIA/OpenShell/issues/1"), + ..test("sandbox::lifecycle", "later", true) + }; + + #[test] + fn reports_every_outcome() { + let run = |test, outcome, duration| TestRun { + test, + outcome, + duration, + stdout: b"out\n".to_vec(), + stderr: Vec::new(), + }; + let runs = [ + run(&PASSED, Outcome::Passed, Some(Duration::from_millis(1500))), + run( + &FAILED, + Outcome::TimedOut("timed out after 1s".into()), + Some(Duration::from_secs(1)), + ), + run(&SKIPPED, Outcome::Skipped, None), + ]; + let xml = report("openshell-conformance", &runs).to_string().unwrap(); + for expected in [ + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + "out\n", + ] { + assert!(xml.contains(expected), "missing {expected} in\n{xml}"); + } + } +} diff --git a/tests/testkit/testkit/src/program/mod.rs b/tests/testkit/testkit/src/program/mod.rs new file mode 100644 index 0000000000..eb8591833d --- /dev/null +++ b/tests/testkit/testkit/src/program/mod.rs @@ -0,0 +1,374 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `testkit::main()`: a test program anyone can run against their gateway. +//! +//! A program is a binary whose `main` calls [`main`], linked with the crates +//! holding its `#[testkit::test]`s: +//! +//! ```ignore +//! fn main() -> std::process::ExitCode { +//! testkit::main() +//! } +//! ``` +//! +//! It has three subcommands: +//! +//! - `list [FILTER]...` prints each test's name and description. +//! - `ready [--timeout SECS]` waits for the gateway, as `testkit-ready` does. +//! - `run [FILTER]...` runs tests one at a time, each in its own process with +//! a timeout, and can write a `JUnit` report. Unix only. +//! +//! Any other arguments are libtest's, so the program also works as a +//! `harness = false` test target under `cargo test` and nextest: it answers +//! `--list --format terse`, `--ignored`, and ` --exact --nocapture`. +//! In that mode tests run in the program's own process, one at a time unless +//! `--test-threads` says otherwise. + +#[cfg(unix)] +mod capture; +#[cfg(unix)] +mod junit; +#[cfg(unix)] +mod process; +#[cfg(unix)] +mod supervisor; + +use std::ffi::OsString; +use std::path::PathBuf; +use std::process::ExitCode; +use std::time::Duration; + +use clap::{Parser, Subcommand}; + +use crate::registry::{RESERVED_NAMES, Selection, TESTS, Test, validate}; + +/// Exit code when a test failed. +const FAILED_EXIT: u8 = 1; +/// Exit code when tests could not run, including usage errors. +const ERROR_EXIT: u8 = 2; +/// Set by `run` for the test processes it starts. +const SUPERVISED_ENV: &str = "TESTKIT_SUPERVISED"; + +/// Run the program: the subcommands, or libtest's interface. +pub fn main() -> ExitCode { + main_with(&TESTS) +} + +/// [`main`] over `tests` instead of every registered test. +#[doc(hidden)] +pub fn main_with(tests: &'static [Test]) -> ExitCode { + let args: Vec = std::env::args_os().collect(); + let program = program_name(&args); + if let Err(error) = validate(tests) { + eprintln!("{program}: {error}"); + return ExitCode::from(ERROR_EXIT); + } + let first = args.get(1).and_then(|arg| arg.to_str()); + if first.is_some_and(|arg| RESERVED_NAMES.contains(&arg) || arg == "-h" || arg == "--help") { + let cli = match Cli::try_parse_from(&args) { + Ok(cli) => cli, + Err(error) => { + let _ = error.print(); + return ExitCode::from(if error.use_stderr() { ERROR_EXIT } else { 0 }); + } + }; + command(&program, tests, cli.command) + } else { + libtest(tests, args) + } +} + +#[derive(Parser)] +#[command( + about = "Run OpenShell tests against a gateway", + after_help = "The gateway is the one the openshell CLI selects: OPENSHELL_GATEWAY_ENDPOINT, \ + OPENSHELL_GATEWAY, or the active gateway. OPENSHELL_WORKSPACE selects the \ + workspace and OPENSHELL_BIN the CLI. Only plaintext and mTLS gateways are \ + supported.\n\nWithout a subcommand the program takes libtest's arguments, \ + as under cargo test and nextest." +)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Print each test's name and description, including ignored tests. + List { + /// Show tests whose name contains any of these (all if none). + filters: Vec, + }, + /// Wait until the gateway is healthy and the CLI reaches it. Exits 2 if + /// it is not ready in time. + Ready { + /// How long to wait, in seconds. + #[arg(long, default_value = "60", value_name = "SECS", value_parser = seconds)] + timeout: Duration, + }, + /// Run tests one at a time, each in its own process. Exits 1 if a test + /// failed and 2 if tests could not run. + Run { + /// Run tests whose name contains any of these (all if none). + filters: Vec, + /// Run ignored tests too. + #[arg(long)] + include_ignored: bool, + /// Stop a test that runs longer than this, in seconds. Longer than + /// testkit's own step timeouts, so those report first. + #[arg(long, default_value = "900", value_name = "SECS", value_parser = seconds)] + timeout: Duration, + /// How long a stopped test has between SIGTERM and SIGKILL, in seconds. + #[arg(long, default_value = "10", value_name = "SECS", value_parser = seconds)] + grace: Duration, + /// Write a JUnit report here. + #[arg(long, value_name = "PATH")] + junit: Option, + }, +} + +/// A positive, finite number of seconds. +fn seconds(value: &str) -> Result { + let seconds: f64 = value + .parse() + .map_err(|_| format!("{value:?} is not a number"))?; + Duration::try_from_secs_f64(seconds) + .ok() + .filter(|duration| !duration.is_zero()) + .ok_or_else(|| format!("{value:?} is not a positive number of seconds")) +} + +fn command(program: &str, tests: &'static [Test], command: Command) -> ExitCode { + match command { + Command::List { filters } => { + list(tests, filters); + ExitCode::SUCCESS + } + Command::Ready { timeout } => ready(program, timeout), + Command::Run { + filters, + include_ignored, + timeout, + grace, + junit, + } => run( + program, + tests, + Selection { + filters, + include_ignored, + }, + timeout, + grace, + junit, + ), + } +} + +#[cfg(unix)] +fn run( + program: &str, + tests: &'static [Test], + selection: Selection, + timeout: Duration, + grace: Duration, + junit: Option, +) -> ExitCode { + let exe = match std::env::current_exe() { + Ok(exe) => exe, + Err(error) => { + eprintln!("{program}: find this program's path: {error}"); + return ExitCode::from(ERROR_EXIT); + } + }; + let options = supervisor::Options { + selection, + timeout, + grace, + junit, + }; + supervisor::run(program, &exe, tests, &options) +} + +#[cfg(not(unix))] +fn run( + program: &str, + _tests: &'static [Test], + _selection: Selection, + _timeout: Duration, + _grace: Duration, + _junit: Option, +) -> ExitCode { + eprintln!( + "{program}: `run` is only supported on Unix; run the tests with cargo test or nextest" + ); + ExitCode::from(ERROR_EXIT) +} + +/// One line per test: name and description, in aligned columns. +/// An ignored test's description ends with `[ignored]` or its reason. +fn list(tests: &[Test], filters: Vec) { + let selection = Selection { + filters, + include_ignored: true, + }; + let rows: Vec<_> = selection + .apply(tests) + .into_iter() + .map(|(test, _)| (test.name(), describe(test))) + .collect(); + let name_width = rows.iter().map(|row| row.0.len()).max().unwrap_or(0); + for (name, description) in rows { + println!("{name:name_width$} {description}"); + } +} + +fn describe(test: &Test) -> String { + let description = test.info.description; + match (test.ignored, test.ignore_reason) { + (false, _) => description.to_string(), + (true, None) => format!("{description} [ignored]"), + (true, Some(reason)) => format!("{description} [ignored: {reason}]"), + } +} + +fn ready(program: &str, timeout: Duration) -> ExitCode { + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + eprintln!("{program}: start the async runtime: {error}"); + return ExitCode::from(ERROR_EXIT); + } + }; + // A gateway that is not ready means the tests cannot run. + if runtime.block_on(crate::ready::check(program, timeout)) { + ExitCode::SUCCESS + } else { + ExitCode::from(ERROR_EXIT) + } +} + +/// libtest's interface, for `cargo test` and nextest, and for `run`'s test +/// processes. +fn libtest(tests: &'static [Test], args: Vec) -> ExitCode { + let mut args = libtest_mimic::Arguments::from_iter(args); + // Tests share one gateway; run them one at a time unless asked otherwise. + if args.test_threads.is_none() { + args.test_threads = Some(1); + } + let mut tests: Vec<&'static Test> = tests.iter().collect(); + tests.sort_by_cached_key(|test| test.name()); + let trials = tests + .into_iter() + .map(|test| { + libtest_mimic::Trial::test(test.name(), move || { + run_in_process(test); + Ok(()) + }) + .with_ignored_flag(test.ignored) + }) + .collect(); + let conclusion = libtest_mimic::run(&args, trials); + // `run` starts this process for exactly one test. Running none would + // look like a pass. + if std::env::var_os(SUPERVISED_ENV).is_some() + && !args.list + && conclusion.num_passed + conclusion.num_failed != 1 + { + eprintln!( + "testkit: expected to run exactly one test, ran {}", + conclusion.num_passed + conclusion.num_failed + ); + return ExitCode::from(ERROR_EXIT); + } + conclusion.exit_code() +} + +/// Run one test on a fresh current-thread runtime, as `#[tokio::test]` does. +/// Panics to fail it. +fn run_in_process(test: &Test) { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("start the async runtime") + .block_on((test.run)()); +} + +fn program_name(args: &[OsString]) -> String { + args.first() + .map(PathBuf::from) + .and_then(|path| { + path.file_name() + .map(|name| name.to_string_lossy().into_owned()) + }) + .unwrap_or_else(|| "testkit".to_string()) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use clap::Parser; + + use super::{Cli, Command}; + + #[test] + fn cli_is_consistent() { + ::command().debug_assert(); + } + + #[test] + fn run_parses_selection_and_limits() { + let cli = Cli::try_parse_from([ + "p", + "run", + "smoke", + "life", + "--include-ignored", + "--timeout", + "1.5", + "--junit", + "out/junit.xml", + ]) + .unwrap(); + let Command::Run { + filters, + include_ignored, + timeout, + grace, + junit, + } = cli.command + else { + panic!("not run"); + }; + assert_eq!(filters, ["smoke", "life"]); + assert!(include_ignored); + assert_eq!(timeout, Duration::from_millis(1500)); + assert_eq!(grace, Duration::from_secs(10)); + assert_eq!(junit.unwrap().to_str(), Some("out/junit.xml")); + + let Command::Ready { timeout } = Cli::try_parse_from(["p", "ready"]).unwrap().command + else { + panic!("not ready"); + }; + assert_eq!(timeout, Duration::from_secs(60)); + } + + #[test] + fn rejects_bad_limits() { + for bad in ["0", "-1", "soon", "inf", "NaN"] { + assert!( + Cli::try_parse_from(["p", "run", "--timeout", bad]).is_err(), + "{bad}" + ); + assert!( + Cli::try_parse_from(["p", "ready", "--timeout", bad]).is_err(), + "{bad}" + ); + } + } +} diff --git a/tests/testkit/testkit/src/program/process.rs b/tests/testkit/testkit/src/program/process.rs new file mode 100644 index 0000000000..1959452953 --- /dev/null +++ b/tests/testkit/testkit/src/program/process.rs @@ -0,0 +1,216 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! One test process: started in its own process group, waited for with a +//! timeout, and stopped with `SIGTERM`, a grace period, then `SIGKILL`. +//! +//! The process group is killed whenever the test process ends, so a test's +//! helpers, such as CLI processes, cannot outlive it. + +use std::io; +use std::os::unix::process::CommandExt as _; +use std::process::{Child, Command, ExitStatus}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +const POLL_INTERVAL: Duration = Duration::from_millis(20); + +/// How the test process ended. +#[derive(Debug)] +pub enum Ended { + Exited(ExitStatus), + /// Stopped after the timeout. + TimedOut, + /// Stopped because the supervisor was asked to stop. + Interrupted, +} + +/// Start `command` in its own process group. +pub fn spawn(command: &mut Command) -> io::Result { + command.process_group(0).spawn() +} + +/// Wait for `child`, stopping it after `timeout` or once the supervisor has +/// been asked to stop (see [`install_stop_handlers`]). The child is reaped +/// and its group killed on every path, including errors. +pub fn wait(child: &mut Child, timeout: Duration, grace: Duration) -> io::Result { + let result = wait_or_stop(child, timeout, grace); + if result.is_err() { + let _ = reap(child); + } + result +} + +fn wait_or_stop(child: &mut Child, timeout: Duration, grace: Duration) -> io::Result { + let deadline = Instant::now() + timeout; + loop { + if has_exited(child)? { + return Ok(Ended::Exited(reap(child)?)); + } + if stop_requested() { + stop(child, grace)?; + return Ok(Ended::Interrupted); + } + if Instant::now() >= deadline { + stop(child, grace)?; + return Ok(Ended::TimedOut); + } + std::thread::sleep(POLL_INTERVAL); + } +} + +/// `SIGTERM` the group, wait up to `grace`, then `SIGKILL` it. A stop signal +/// during the grace period, or a second one in all, cuts it short. +fn stop(child: &mut Child, grace: Duration) -> io::Result { + signal_group(child, libc::SIGTERM); + let deadline = Instant::now() + grace; + let before = stop_requests(); + let cut_short = || { + let now = stop_requests(); + now > before || now >= 2 + }; + while Instant::now() < deadline && !cut_short() && !has_exited(child)? { + std::thread::sleep(POLL_INTERVAL); + } + reap(child) +} + +/// Kill whatever is left of the group, then collect the exit status. +fn reap(child: &mut Child) -> io::Result { + signal_group(child, libc::SIGKILL); + child.wait() +} + +static STOP_REQUESTS: AtomicUsize = AtomicUsize::new(0); + +fn stop_requests() -> usize { + STOP_REQUESTS.load(Ordering::SeqCst) +} + +/// Whether the supervisor has been asked to stop since it started. +pub fn stop_requested() -> bool { + stop_requests() > 0 +} + +fn pid(child: &Child) -> libc::pid_t { + libc::pid_t::try_from(child.id()).expect("pid fits in pid_t") +} + +/// Whether the child has exited, without reaping it. While it is unreaped +/// its PID, and so its process group ID, cannot be reused, so signalling the +/// group cannot reach an unrelated process. +#[cfg(target_os = "linux")] +fn has_exited(child: &mut Child) -> io::Result { + // SAFETY: `info` is a valid out-pointer, and `WNOWAIT` leaves the child + // for `Child::wait` to reap. + unsafe { + let mut info: libc::siginfo_t = std::mem::zeroed(); + let result = libc::waitid( + libc::P_PID, + child.id(), + &raw mut info, + libc::WEXITED | libc::WNOHANG | libc::WNOWAIT, + ); + if result == -1 { + return Err(io::Error::last_os_error()); + } + Ok(info.si_pid() != 0) + } +} + +/// Whether the child has exited, reaping it. `waitid` with `WNOWAIT` is not +/// reliable outside Linux (see golang/go#19314), so the group is killed just +/// after the leader is reaped. Its ID could in principle be reused in that +/// window, though a new process would have to get the same PID and lead a +/// group. +#[cfg(not(target_os = "linux"))] +fn has_exited(child: &mut Child) -> io::Result { + Ok(child.try_wait()?.is_some()) +} + +fn signal_group(child: &Child, signal: libc::c_int) { + // SAFETY: plain syscall to the child's own group (see `spawn`). ESRCH, an + // empty group, is fine. + unsafe { + libc::kill(-pid(child), signal); + } +} + +extern "C" fn on_stop(_signal: libc::c_int) { + STOP_REQUESTS.fetch_add(1, Ordering::SeqCst); +} + +/// Count `SIGINT`, `SIGTERM`, and `SIGHUP` as stop requests instead of dying. +/// Tests run in their own process group, so a Ctrl-C at the terminal reaches +/// only the supervisor, which must stop the test. +pub fn install_stop_handlers() -> io::Result<()> { + for signal in [libc::SIGINT, libc::SIGTERM, libc::SIGHUP] { + // SAFETY: the handler only touches an atomic, which is + // async-signal-safe. + unsafe { + let mut action: libc::sigaction = std::mem::zeroed(); + action.sa_sigaction = on_stop as *const () as libc::sighandler_t; + action.sa_flags = libc::SA_RESTART; + libc::sigemptyset(&raw mut action.sa_mask); + if libc::sigaction(signal, &raw const action, std::ptr::null_mut()) == -1 { + return Err(io::Error::last_os_error()); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::process::{Command, Stdio}; + use std::time::{Duration, Instant}; + + use super::{Ended, spawn, wait}; + + fn sh(script: &str) -> Command { + let mut command = Command::new("sh"); + command.args(["-c", script]).stdin(Stdio::null()); + command + } + + #[test] + fn exit_status_is_kept() { + let mut child = spawn(&mut sh("exit 3")).unwrap(); + let ended = wait(&mut child, Duration::from_secs(10), Duration::from_secs(1)).unwrap(); + assert!( + matches!(ended, Ended::Exited(status) if status.code() == Some(3)), + "{ended:?}" + ); + } + + #[test] + fn timeout_terminates_the_test() { + let started = Instant::now(); + let mut child = spawn(&mut sh("sleep 30")).unwrap(); + let ended = wait( + &mut child, + Duration::from_millis(200), + Duration::from_secs(10), + ) + .unwrap(); + assert!(matches!(ended, Ended::TimedOut), "{ended:?}"); + // `sleep` dies on SIGTERM, well inside the grace period. + assert!(started.elapsed() < Duration::from_secs(5)); + } + + #[test] + fn a_test_ignoring_sigterm_is_killed_after_the_grace_period() { + let started = Instant::now(); + let mut child = spawn(&mut sh("trap '' TERM; sleep 30")).unwrap(); + let ended = wait( + &mut child, + Duration::from_millis(200), + Duration::from_millis(500), + ) + .unwrap(); + assert!(matches!(ended, Ended::TimedOut), "{ended:?}"); + let elapsed = started.elapsed(); + assert!(elapsed >= Duration::from_millis(700), "{elapsed:?}"); + assert!(elapsed < Duration::from_secs(5), "{elapsed:?}"); + } +} diff --git a/tests/testkit/testkit/src/program/supervisor.rs b/tests/testkit/testkit/src/program/supervisor.rs new file mode 100644 index 0000000000..6dced12ae1 --- /dev/null +++ b/tests/testkit/testkit/src/program/supervisor.rs @@ -0,0 +1,334 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `run`: run the selected tests one at a time, each in a new process. +//! +//! The program starts itself once per test as ` --exact --nocapture`, +//! the same interface nextest uses, in a new process group. A test that runs +//! past its timeout gets `SIGTERM`, a grace period, then `SIGKILL`. Output is +//! shown as it comes and kept for the `JUnit` report. +//! +//! A test stopped this way does not run its cleanup: testkit tests do not +//! handle `SIGTERM` yet, so sandboxes named `test--*` can be left +//! behind. + +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; +use std::process::{Command, ExitCode, ExitStatus, Stdio}; +use std::time::{Duration, Instant}; + +use super::capture::Capture; +use super::process::{self, Ended}; +use super::{ERROR_EXIT, FAILED_EXIT, SUPERVISED_ENV, junit}; +use crate::registry::{Disposition, Selection, Test}; + +/// libtest's exit code when a test failed. +const LIBTEST_FAILED: i32 = 101; + +pub struct Options { + pub selection: Selection, + pub timeout: Duration, + pub grace: Duration, + pub junit: Option, +} + +/// One test's result. +pub struct TestRun<'a> { + pub test: &'a Test, + pub outcome: Outcome, + /// `None` for a test that did not run. + pub duration: Option, + pub stdout: Vec, + pub stderr: Vec, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Outcome { + Passed, + /// Ignored and not asked for. + Skipped, + Failed(String), + TimedOut(String), + /// The test could not be run. + Error(String), + /// Stopped because the run was interrupted. + Interrupted, +} + +impl Outcome { + fn label(&self) -> &'static str { + match self { + Self::Passed => "PASS", + Self::Skipped => "SKIP", + Self::Failed(_) => "FAIL", + Self::TimedOut(_) => "TIMEOUT", + Self::Error(_) | Self::Interrupted => "ERROR", + } + } + + fn message(&self) -> Option<&str> { + match self { + Self::Passed | Self::Skipped => None, + Self::Failed(message) | Self::TimedOut(message) | Self::Error(message) => Some(message), + Self::Interrupted => Some("interrupted"), + } + } +} + +pub fn run(program: &str, exe: &Path, tests: &[Test], options: &Options) -> ExitCode { + let selected = options.selection.apply(tests); + let to_run = selected + .iter() + .filter(|(_, disposition)| *disposition == Disposition::Run) + .count(); + if to_run == 0 { + eprintln!("{program}: no tests to run; `{program} list` shows the tests"); + return ExitCode::from(ERROR_EXIT); + } + if let Err(error) = process::install_stop_handlers() { + eprintln!("{program}: install signal handlers: {error}"); + return ExitCode::from(ERROR_EXIT); + } + let skipped = selected.len() - to_run; + eprintln!( + "{program}: running {to_run} test{} one at a time{}", + plural(to_run), + if skipped > 0 { + format!(", skipping {skipped} ignored") + } else { + String::new() + } + ); + + let started = Instant::now(); + let selected_count = selected.len(); + let mut runs = Vec::with_capacity(selected_count); + let mut position = 0; + for (test, disposition) in selected { + // A stop signal that arrives between tests still ends the run. + if process::stop_requested() { + break; + } + if disposition == Disposition::Skip { + runs.push(TestRun { + test, + outcome: Outcome::Skipped, + duration: None, + stdout: Vec::new(), + stderr: Vec::new(), + }); + continue; + } + position += 1; + let name = test.name(); + eprintln!("==> [{position}/{to_run}] {name}"); + let run = run_one(exe, test, options); + let elapsed = run.duration.unwrap_or_default(); + match run.outcome.message() { + None => eprintln!("<== {} {name} ({elapsed:.1?})", run.outcome.label()), + Some(message) => eprintln!( + "<== {} {name} ({elapsed:.1?}): {message}", + run.outcome.label() + ), + } + runs.push(run); + } + + let not_run = selected_count - runs.len(); + let mut status = summarize(program, &runs, not_run, started.elapsed()); + if let Some(path) = &options.junit { + match junit::write(path, &junit::report(program, &runs)) { + Ok(()) => eprintln!("{program}: wrote {}", path.display()), + Err(error) => { + eprintln!("{program}: write {}: {error}", path.display()); + status = ERROR_EXIT; + } + } + } + ExitCode::from(status) +} + +fn run_one<'a>(exe: &Path, test: &'a Test, options: &Options) -> TestRun<'a> { + let name = test.name(); + let mut command = Command::new(exe); + command + .args([name.as_str(), "--exact", "--nocapture"]) + .env(SUPERVISED_ENV, "1") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if test.ignored { + command.arg("--ignored"); + } + + let started = Instant::now(); + let mut child = match process::spawn(&mut command) { + Ok(child) => child, + Err(error) => { + return TestRun { + test, + outcome: Outcome::Error(format!("start {}: {error}", exe.display())), + duration: Some(started.elapsed()), + stdout: Vec::new(), + stderr: Vec::new(), + }; + } + }; + let stdout = child + .stdout + .take() + .map(|stream| Capture::start(stream, std::io::stdout())); + let stderr = child + .stderr + .take() + .map(|stream| Capture::start(stream, std::io::stderr())); + let ended = process::wait(&mut child, options.timeout, options.grace); + let duration = started.elapsed(); + let outcome = match ended { + Ok(ended) => outcome(ended, options.timeout), + Err(error) => Outcome::Error(format!("wait for the test process: {error}")), + }; + TestRun { + test, + outcome, + duration: Some(duration), + stdout: stdout.map(Capture::finish).unwrap_or_default(), + stderr: stderr.map(Capture::finish).unwrap_or_default(), + } +} + +fn outcome(ended: Ended, timeout: Duration) -> Outcome { + match ended { + Ended::Exited(status) => exited(status), + Ended::TimedOut => Outcome::TimedOut(format!( + "timed out after {}; stopped without cleanup", + seconds(timeout) + )), + Ended::Interrupted => Outcome::Interrupted, + } +} + +fn exited(status: ExitStatus) -> Outcome { + match status.code() { + Some(0) => Outcome::Passed, + Some(LIBTEST_FAILED) => Outcome::Failed("test failed".to_string()), + Some(code) if code == i32::from(ERROR_EXIT) => { + Outcome::Error("the test process could not run the test; see its output".to_string()) + } + Some(code) => Outcome::Failed(format!("test process exited with code {code}")), + None => Outcome::Failed(format!("test process ended by {status}")), + } +} + +/// Print the totals and the tests that did not pass; return the exit code: +/// 1 if a test failed, 2 if one could not run or the run was interrupted +/// before `not_run` tests. +fn summarize(program: &str, runs: &[TestRun<'_>], not_run: usize, elapsed: Duration) -> u8 { + let count = + |predicate: fn(&Outcome) -> bool| runs.iter().filter(|run| predicate(&run.outcome)).count(); + let passed = count(|outcome| matches!(outcome, Outcome::Passed)); + let failed = count(|outcome| matches!(outcome, Outcome::Failed(_) | Outcome::TimedOut(_))); + let errors = count(|outcome| matches!(outcome, Outcome::Error(_) | Outcome::Interrupted)); + let skipped = count(|outcome| matches!(outcome, Outcome::Skipped)); + let mut summary = format!("{program}: {passed} passed, {failed} failed"); + if errors > 0 { + let _ = write!(summary, ", {errors} could not run"); + } + if skipped > 0 { + let _ = write!(summary, ", {skipped} skipped"); + } + let _ = write!(summary, " in {elapsed:.1?}"); + eprintln!("{summary}"); + for run in runs { + if let Some(message) = run.outcome.message() { + eprintln!(" {} {}: {message}", run.outcome.label(), run.test.name()); + } + } + if not_run > 0 { + eprintln!( + "{program}: interrupted; {not_run} later test{} did not run", + plural(not_run) + ); + } + if errors > 0 || not_run > 0 { + ERROR_EXIT + } else if failed > 0 { + FAILED_EXIT + } else { + 0 + } +} + +fn seconds(duration: Duration) -> String { + format!("{}s", duration.as_secs_f64()) +} + +fn plural(count: usize) -> &'static str { + if count == 1 { "" } else { "s" } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::{Outcome, TestRun, summarize}; + use crate::registry::Test; + use crate::registry::tests::test; + + const TEST: Test = test("suite::m", "t", false); + + fn runs(outcomes: Vec) -> Vec> { + outcomes + .into_iter() + .map(|outcome| TestRun { + test: &TEST, + outcome, + duration: None, + stdout: Vec::new(), + stderr: Vec::new(), + }) + .collect() + } + + #[test] + fn exit_code_separates_failures_from_runs_that_could_not_complete() { + let code = |outcomes| summarize("p", &runs(outcomes), 0, Duration::ZERO); + assert_eq!(code(vec![Outcome::Passed, Outcome::Skipped]), 0); + assert_eq!(code(vec![Outcome::Passed, Outcome::Failed("f".into())]), 1); + assert_eq!(code(vec![Outcome::TimedOut("t".into())]), 1); + assert_eq!( + code(vec![ + Outcome::Failed("f".into()), + Outcome::Error("e".into()) + ]), + 2 + ); + assert_eq!(code(vec![Outcome::Interrupted]), 2); + // Interrupted between tests: every test that ran passed. + assert_eq!( + summarize("p", &runs(vec![Outcome::Passed]), 1, Duration::ZERO), + 2 + ); + } + + #[test] + fn exit_statuses_map_to_outcomes() { + use std::os::unix::process::ExitStatusExt; + + use super::exited; + + let status = |code: i32| std::process::ExitStatus::from_raw(code << 8); + assert_eq!(exited(status(0)), Outcome::Passed); + assert_eq!(exited(status(101)), Outcome::Failed("test failed".into())); + assert!(matches!(exited(status(2)), Outcome::Error(_))); + assert_eq!( + exited(status(7)), + Outcome::Failed("test process exited with code 7".into()) + ); + let killed = exited(std::process::ExitStatus::from_raw(libc::SIGSEGV)); + assert!( + matches!(&killed, Outcome::Failed(message) if message.contains("signal")), + "{killed:?}" + ); + } +} diff --git a/tests/testkit/testkit/src/ready.rs b/tests/testkit/testkit/src/ready.rs new file mode 100644 index 0000000000..db36f67659 --- /dev/null +++ b/tests/testkit/testkit/src/ready.rs @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Waiting until the gateway is ready for tests. + +use std::time::Duration; + +use crate::cli::Cli; +use crate::gateway::Gateway; + +/// Wait until the gateway selected as for tests is healthy and the CLI +/// reports it connected, logging to stderr as `program`. Returns whether it +/// is ready; if not, the last error has been logged. +pub async fn check(program: &str, timeout: Duration) -> bool { + let result = async { + let gateway = Gateway::from_env()?; + eprintln!("{program}: {}", gateway.describe()); + let health = gateway.wait_ready(timeout).await?; + Cli::from_env(&gateway).check_connected().await?; + crate::Result::Ok(health) + } + .await; + match result { + Ok(health) => { + eprintln!( + "{program}: gateway {} is healthy and the CLI reaches it", + health.version + ); + true + } + Err(error) => { + eprintln!("{program}: {error}"); + false + } + } +} diff --git a/tests/testkit/testkit/src/registry.rs b/tests/testkit/testkit/src/registry.rs new file mode 100644 index 0000000000..aceaed673f --- /dev/null +++ b/tests/testkit/testkit/src/registry.rs @@ -0,0 +1,208 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The tests linked into a program, registered by `#[testkit::test]`. + +use std::collections::BTreeSet; +use std::future::Future; +use std::pin::Pin; + +use crate::run::TestInfo; + +/// Every `#[testkit::test]` linked into the program. +#[linkme::distributed_slice] +pub static TESTS: [Test]; + +/// A registered test. +#[derive(Debug)] +pub struct Test { + pub info: TestInfo, + /// The directory under `tests/suites/` holding the test's crate, if any. + pub family: Option<&'static str>, + /// `module_path!()` where the test is defined. + pub module_path: &'static str, + pub ignored: bool, + /// The reason given in `#[ignore = "..."]`. + pub ignore_reason: Option<&'static str>, + /// Runs the test through [`crate::run`]; panics to fail it. + pub run: fn() -> Pin>>, +} + +impl Test { + /// The test's name, as libtest and nextest name it: the module path + /// without the crate, then the function name. + pub fn name(&self) -> String { + match self.module_path.split_once("::") { + Some((_crate, module)) => format!("{module}::{}", self.info.name), + None => self.info.name.to_string(), + } + } + + pub fn family(&self) -> &'static str { + self.family.unwrap_or("-") + } +} + +/// Names a test cannot have at the top level: the program would read them as +/// its own subcommands, so nextest could not run the test by name. +pub const RESERVED_NAMES: &[&str] = &["help", "list", "ready", "run"]; + +/// Check the tests can be told apart by name. +pub fn validate(tests: &[Test]) -> Result<(), String> { + let mut names = BTreeSet::new(); + for test in tests { + let name = test.name(); + if RESERVED_NAMES.contains(&name.as_str()) { + return Err(format!( + "test {name} has the name of a subcommand; rename it or move it into a module" + )); + } + if !names.insert(name.clone()) { + return Err(format!( + "two tests are named {name}; test names must be unique in a program" + )); + } + } + Ok(()) +} + +/// Which tests to run. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Selection { + /// Run tests whose name contains any of these; all tests if empty. + pub filters: Vec, + /// Run `#[ignore]`d tests too, instead of reporting them as skipped. + pub include_ignored: bool, +} + +/// What a selection does with one test. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Disposition { + Run, + /// Selected, but ignored and not asked for. + Skip, +} + +impl Selection { + /// The tests this selects, in name order, with what to do with each. + pub fn apply<'a>(&self, tests: &'a [Test]) -> Vec<(&'a Test, Disposition)> { + let mut selected: Vec<_> = tests + .iter() + .filter_map(|test| { + self.disposition(test) + .map(|disposition| (test, disposition)) + }) + .collect(); + selected.sort_by_cached_key(|(test, _)| test.name()); + selected + } + + fn disposition(&self, test: &Test) -> Option { + let name = test.name(); + if !self.filters.is_empty() + && !self + .filters + .iter() + .any(|filter| name.contains(filter.as_str())) + { + return None; + } + if test.ignored && !self.include_ignored { + Some(Disposition::Skip) + } else { + Some(Disposition::Run) + } + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::{Disposition, Selection, Test, validate}; + use crate::run::TestInfo; + + pub(crate) const fn test(module_path: &'static str, name: &'static str, ignored: bool) -> Test { + Test { + info: TestInfo { + name, + description: "A test.", + }, + family: Some("conformance"), + module_path, + ignored, + ignore_reason: None, + run: || Box::pin(async {}), + } + } + + const TESTS: &[Test] = &[ + test("sandbox::smoke", "create_exec_delete", false), + test("sandbox::lifecycle", "stop_start", false), + test("sandbox::lifecycle", "restart_later", true), + test("smoke", "top_level", false), + ]; + + fn names(selection: &Selection) -> Vec<(String, Disposition)> { + selection + .apply(TESTS) + .into_iter() + .map(|(test, disposition)| (test.name(), disposition)) + .collect() + } + + #[test] + fn names_follow_libtest() { + assert_eq!(TESTS[0].name(), "smoke::create_exec_delete"); + assert_eq!(TESTS[3].name(), "top_level"); + } + + #[test] + fn selection_follows_libtest() { + use Disposition::{Run, Skip}; + let all = Selection::default(); + assert_eq!( + names(&all), + [ + ("lifecycle::restart_later".into(), Skip), + ("lifecycle::stop_start".into(), Run), + ("smoke::create_exec_delete".into(), Run), + ("top_level".into(), Run), + ] + ); + + let filtered = Selection { + filters: vec!["stop".into(), "top".into()], + ..Selection::default() + }; + assert_eq!( + names(&filtered), + [ + ("lifecycle::stop_start".into(), Run), + ("top_level".into(), Run) + ] + ); + + let include = Selection { + filters: vec!["lifecycle".into()], + include_ignored: true, + }; + assert_eq!( + names(&include), + [ + ("lifecycle::restart_later".into(), Run), + ("lifecycle::stop_start".into(), Run) + ] + ); + } + + #[test] + fn validate_rejects_ambiguous_names() { + assert!(validate(TESTS).is_ok()); + let duplicate = [test("a::m", "t", false), test("b::m", "t", false)]; + let error = validate(&duplicate).unwrap_err(); + assert!(error.contains("two tests are named m::t"), "{error}"); + let reserved = [test("suite", "run", false)]; + let error = validate(&reserved).unwrap_err(); + assert!(error.contains("name of a subcommand"), "{error}"); + assert!(validate(&[test("suite::m", "run", false)]).is_ok()); + } +} diff --git a/tests/testkit/testkit/tests/attribute.rs b/tests/testkit/testkit/tests/attribute.rs new file mode 100644 index 0000000000..1f49adea40 --- /dev/null +++ b/tests/testkit/testkit/tests/attribute.rs @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! What `#[testkit::test]` expands to, checked by compiling tests with it in +//! an ordinary libtest target. The tests need a gateway, so they are ignored; +//! the check compares the registry with what libtest lists. + +use std::collections::BTreeSet; +use std::process::Command; + +/// Ignored with a reason. +#[testkit::test] +#[ignore = "needs a gateway"] +async fn ignored_with_reason(_cx: &mut testkit::Context) -> testkit::Result { + Ok(()) +} + +/// Lint attributes reach the body: `cx` is unused. +#[testkit::test] +#[ignore] +#[allow(unused_variables)] +async fn lint_allowed(cx: &mut testkit::Context) -> testkit::Result { + Ok(()) +} + +/// Compiled out, so neither a test nor registered. +#[testkit::test] +#[cfg(any())] +async fn compiled_out(_cx: &mut testkit::Context) -> testkit::Result { + Ok(()) +} + +mod nested { + /// Named with its module. + #[testkit::test] + #[ignore] + async fn in_a_module(_cx: &mut testkit::Context) -> testkit::Result { + Ok(()) + } +} + +/// The names libtest lists for this binary, with `extra` arguments. +fn libtest_list(extra: &[&str]) -> BTreeSet { + let output = Command::new(std::env::current_exe().expect("current exe")) + .args(["--list", "--format", "terse"]) + .args(extra) + .output() + .expect("list the tests"); + assert!(output.status.success(), "{output:?}"); + String::from_utf8(output.stdout) + .expect("utf-8") + .lines() + .filter_map(|line| line.strip_suffix(": test")) + .filter(|name| *name != "registry_matches_libtest") + .map(str::to_string) + .collect() +} + +#[test] +fn registry_matches_libtest() { + let registered: Vec<_> = testkit::__private::TESTS + .iter() + .map(|test| { + ( + test.name(), + test.module_path, + test.family, + test.ignored, + test.ignore_reason, + test.info.description, + ) + }) + .collect::>() + .into_iter() + .collect(); + assert_eq!( + registered, + [ + ( + "ignored_with_reason".to_string(), + "attribute", + None, + true, + Some("needs a gateway"), + "Ignored with a reason." + ), + ( + "lint_allowed".to_string(), + "attribute", + None, + true, + None, + "Lint attributes reach the body: `cx` is unused." + ), + ( + "nested::in_a_module".to_string(), + "attribute::nested", + None, + true, + None, + "Named with its module." + ), + ] + ); + + // libtest runs and ignores the same tests, under the same names. + let names = |ignored_only: bool| -> BTreeSet { + testkit::__private::TESTS + .iter() + .filter(|test| test.ignored || !ignored_only) + .map(testkit::Test::name) + .collect() + }; + assert_eq!(libtest_list(&[]), names(false)); + assert_eq!(libtest_list(&["--ignored"]), names(true)); +} diff --git a/tests/testkit/testkit/tests/program.rs b/tests/testkit/testkit/tests/program.rs new file mode 100644 index 0000000000..fed3a7ff55 --- /dev/null +++ b/tests/testkit/testkit/tests/program.rs @@ -0,0 +1,504 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Checks of `testkit::main()`, as a test program. +//! +//! This binary is three programs, chosen by `TESTKIT_FIXTURE`: +//! +//! - unset: the checks below, run by `cargo test` and nextest through the +//! program's libtest interface; +//! - `fixtures`: tests that pass, fail, hang, ignore `SIGTERM`, or leave +//! processes behind, which the checks run through the program's +//! subcommands; +//! - `registered`: `testkit::main()` over the `#[testkit::test]`s below. + +#[cfg(unix)] +fn main() -> std::process::ExitCode { + match std::env::var("TESTKIT_FIXTURE").as_deref() { + Ok("fixtures") => testkit::program::main_with(&unix::FIXTURES), + Ok("registered") => testkit::main(), + _ => testkit::program::main_with(&unix::CHECKS), + } +} + +#[cfg(not(unix))] +fn main() {} + +/// Needs a gateway. +#[testkit::test] +async fn registered_needs_a_gateway(cx: &mut testkit::Context) -> testkit::Result { + cx.cli(["status"]).await?.success()?; + Ok(()) +} + +/// Ignored until later. +#[testkit::test] +#[ignore = "https://github.com/NVIDIA/OpenShell/issues/1"] +async fn registered_ignored(_cx: &mut testkit::Context) -> testkit::Result { + Ok(()) +} + +#[cfg(unix)] +mod unix { + use std::future::Future; + use std::os::unix::process::CommandExt as _; + use std::path::{Path, PathBuf}; + use std::pin::Pin; + use std::process::{Child, Command, Output, Stdio}; + use std::time::{Duration, Instant}; + + use testkit::{Test, TestInfo}; + + const PID_FILE_ENV: &str = "FIXTURE_PID_FILE"; + + type Run = fn() -> Pin>>; + + const fn test(module_path: &'static str, name: &'static str, run: Run) -> Test { + Test { + info: TestInfo { + name, + description: "A fixture.", + }, + family: Some("fixtures"), + module_path, + ignored: false, + ignore_reason: None, + run, + } + } + + const fn fixture(name: &'static str, run: Run) -> Test { + test("program::fixture", name, run) + } + + const fn check(name: &'static str, run: Run) -> Test { + test("program::check", name, run) + } + + fn spawn_helper_and_record_its_pid() { + // A helper in the test's process group, like a CLI process. It is + // never waited on: the supervisor must kill it. + #[allow(clippy::zombie_processes)] + let helper = Command::new("sleep") + .arg("300") + .spawn() + .expect("spawn sleep"); + let pid_file = std::env::var(PID_FILE_ENV).expect("pid file"); + std::fs::write(pid_file, helper.id().to_string()).expect("write pid"); + } + + pub static FIXTURES: [Test; 7] = [ + fixture("passes", || { + Box::pin(async { + println!("passes: stdout"); + eprintln!("passes: stderr"); + }) + }), + fixture("fails", || Box::pin(async { panic!("fixture failure") })), + fixture("hangs", || { + Box::pin(async { + spawn_helper_and_record_its_pid(); + std::thread::sleep(Duration::from_secs(300)); + }) + }), + fixture("leaves_a_helper", || { + Box::pin(async { spawn_helper_and_record_its_pid() }) + }), + fixture("interrupts_after_exit", || { + Box::pin(async { + // Signal the supervisor once this test has exited, while it + // waits for the test's output to close. The helper leaves + // the test's process group so that it survives the test, and + // holds the output open until it has sent the signal. + let supervisor = std::os::unix::process::parent_id(); + #[allow(clippy::zombie_processes)] + let _signaller = Command::new("sh") + .args(["-c", &format!("sleep 1; kill -TERM {supervisor}")]) + .process_group(0) + .spawn() + .expect("spawn the signaller"); + }) + }), + fixture("ignores_sigterm", || { + Box::pin(async { + // SAFETY: plain syscall. + unsafe { + libc::signal(libc::SIGTERM, libc::SIG_IGN); + } + std::thread::sleep(Duration::from_secs(300)); + }) + }), + Test { + ignored: true, + ignore_reason: Some("not today"), + ..fixture("ignored", || Box::pin(async {})) + }, + ]; + + pub static CHECKS: [Test; 11] = [ + check("nextest_interface", || { + Box::pin(async { nextest_interface() }) + }), + check("list_shows_name_and_description", || { + Box::pin(async { list_shows_name_and_description() }) + }), + check("run_reports_every_outcome", || { + Box::pin(async { run_reports_every_outcome() }) + }), + check("run_exit_codes", || Box::pin(async { run_exit_codes() })), + check("timeout_kills_the_process_group", || { + Box::pin(async { timeout_kills_the_process_group() }) + }), + check("sigterm_is_followed_by_sigkill", || { + Box::pin(async { sigterm_is_followed_by_sigkill() }) + }), + check("interrupt_stops_the_test_and_the_run", || { + Box::pin(async { interrupt_stops_the_test_and_the_run() }) + }), + check("interrupt_between_tests_ends_the_run", || { + Box::pin(async { interrupt_between_tests_ends_the_run() }) + }), + check("a_passing_test_leaves_nothing_behind", || { + Box::pin(async { a_passing_test_leaves_nothing_behind() }) + }), + check("registered_tests_run_through_testkit", || { + Box::pin(async { registered_tests_run_through_testkit() }) + }), + check("ready_fails_without_a_gateway", || { + Box::pin(async { ready_fails_without_a_gateway() }) + }), + ]; + + fn program(fixture: &str, args: &[&str]) -> Command { + let mut command = Command::new(std::env::current_exe().expect("current exe")); + command + .args(args) + .env("TESTKIT_FIXTURE", fixture) + .env_remove("TESTKIT_SUPERVISED") + .stdin(Stdio::null()); + command + } + + fn output(mut command: Command) -> Output { + let output = command.output().expect("run the program"); + eprintln!( + "{command:?}: {}\n--- stdout\n{}--- stderr\n{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + output + } + + fn stdout(output: &Output) -> &str { + std::str::from_utf8(&output.stdout).expect("utf-8") + } + + fn stderr(output: &Output) -> &str { + std::str::from_utf8(&output.stderr).expect("utf-8") + } + + fn junit(dir: &Path) -> (PathBuf, String) { + let path = dir.join("results").join("junit.xml"); + (path.clone(), path.to_str().expect("utf-8 path").to_string()) + } + + /// nextest lists with `--list --format terse [--ignored]` and runs each + /// test with ` --exact --nocapture`. + fn nextest_interface() { + let list = output(program("fixtures", &["--list", "--format", "terse"])); + assert!(list.status.success()); + assert_eq!( + stdout(&list), + "fixture::fails: test\nfixture::hangs: test\nfixture::ignored: test\n\ + fixture::ignores_sigterm: test\nfixture::interrupts_after_exit: test\n\ + fixture::leaves_a_helper: test\nfixture::passes: test\n" + ); + let ignored = output(program( + "fixtures", + &["--list", "--format", "terse", "--ignored"], + )); + assert_eq!(stdout(&ignored), "fixture::ignored: test\n"); + + let pass = output(program( + "fixtures", + &["fixture::passes", "--exact", "--nocapture"], + )); + assert!(pass.status.success()); + assert!(stdout(&pass).contains("passes: stdout")); + assert!(stdout(&pass).contains("1 passed")); + + let fail = output(program( + "fixtures", + &["fixture::fails", "--exact", "--nocapture"], + )); + assert_eq!(fail.status.code(), Some(101)); + assert!(stderr(&fail).contains("fixture failure")); + + // An ignored test runs only when asked for. + let skip = output(program("fixtures", &["fixture::ignored", "--exact"])); + assert!(stdout(&skip).contains("1 ignored")); + let run = output(program( + "fixtures", + &["fixture::ignored", "--exact", "--ignored"], + )); + assert!(stdout(&run).contains("1 passed")); + } + + fn list_shows_name_and_description() { + let list = output(program("registered", &["list"])); + assert!(list.status.success()); + let lines: Vec<_> = stdout(&list).lines().collect(); + assert_eq!(lines.len(), 2, "{lines:?}"); + assert_eq!( + lines[0], + "registered_ignored Ignored until later. \ + [ignored: https://github.com/NVIDIA/OpenShell/issues/1]" + ); + assert_eq!(lines[1], "registered_needs_a_gateway Needs a gateway."); + + let filtered = output(program("fixtures", &["list", "pass"])); + assert_eq!(stdout(&filtered), "fixture::passes A fixture.\n"); + } + + fn run_reports_every_outcome() { + let dir = tempfile::tempdir().expect("temp dir"); + let (path, junit_arg) = junit(dir.path()); + let run = output(program( + "fixtures", + &["run", "passes", "fails", "ignored", "--junit", &junit_arg], + )); + assert_eq!(run.status.code(), Some(1)); + // Test output is passed through. + assert!(stdout(&run).contains("passes: stdout")); + assert!(stderr(&run).contains("passes: stderr")); + assert!(stderr(&run).contains("1 passed, 1 failed, 1 skipped")); + assert!(stderr(&run).contains("FAIL fixture::fails: test failed")); + + let xml = std::fs::read_to_string(path).expect("junit report"); + for expected in [ + r#""#, + r#""#, + r#""#, + "", + "passes: stdout", + "fixture failure", + ] { + assert!(xml.contains(expected), "missing {expected} in\n{xml}"); + } + } + + fn run_exit_codes() { + let pass = output(program("fixtures", &["run", "passes"])); + assert_eq!(pass.status.code(), Some(0)); + let ignored = output(program( + "fixtures", + &["run", "ignored", "--include-ignored"], + )); + assert_eq!(ignored.status.code(), Some(0)); + assert!(stderr(&ignored).contains("1 passed, 0 failed")); + // Selecting nothing to run is not a pass. + for args in [&["run", "nothing-matches"][..], &["run", "ignored"]] { + let none = output(program("fixtures", args)); + assert_eq!(none.status.code(), Some(2), "{args:?}"); + assert!(stderr(&none).contains("no tests to run")); + } + let usage = output(program("fixtures", &["run", "--timeout", "0"])); + assert_eq!(usage.status.code(), Some(2)); + } + + fn is_alive(pid: libc::pid_t) -> bool { + // SAFETY: signal 0 only checks the process exists. + unsafe { libc::kill(pid, 0) == 0 } + } + + /// Wait for a killed helper to go: once its parent is gone, init reaps it. + fn assert_gone(pid: libc::pid_t) { + let deadline = Instant::now() + Duration::from_secs(10); + while is_alive(pid) { + assert!(Instant::now() < deadline, "helper {pid} outlived its test"); + std::thread::sleep(Duration::from_millis(50)); + } + } + + fn read_pid(path: &Path) -> libc::pid_t { + let deadline = Instant::now() + Duration::from_secs(30); + loop { + if let Ok(pid) = std::fs::read_to_string(path) + && let Ok(pid) = pid.trim().parse() + { + return pid; + } + assert!(Instant::now() < deadline, "the hanging test never started"); + std::thread::sleep(Duration::from_millis(20)); + } + } + + fn timeout_kills_the_process_group() { + let dir = tempfile::tempdir().expect("temp dir"); + let pid_file = dir.path().join("pid"); + let (path, junit_arg) = junit(dir.path()); + let mut command = program( + "fixtures", + &[ + "run", + "hangs", + "--timeout", + "1", + "--grace", + "5", + "--junit", + &junit_arg, + ], + ); + command.env(PID_FILE_ENV, &pid_file); + let started = Instant::now(); + let run = output(command); + assert_eq!(run.status.code(), Some(1)); + // `SIGTERM` stops the test well inside the grace period. + assert!( + started.elapsed() < Duration::from_secs(5), + "{:?}", + started.elapsed() + ); + assert!(stderr(&run).contains("TIMEOUT fixture::hangs: timed out after 1s")); + assert_gone(read_pid(&pid_file)); + let xml = std::fs::read_to_string(path).expect("junit report"); + assert!(xml.contains(r#"type="timeout""#), "{xml}"); + } + + fn sigterm_is_followed_by_sigkill() { + let started = Instant::now(); + let run = output(program( + "fixtures", + &[ + "run", + "ignores_sigterm", + "--timeout", + "0.2", + "--grace", + "0.5", + ], + )); + assert_eq!(run.status.code(), Some(1)); + let elapsed = started.elapsed(); + assert!(elapsed >= Duration::from_millis(700), "{elapsed:?}"); + assert!(elapsed < Duration::from_secs(10), "{elapsed:?}"); + assert!(stderr(&run).contains("TIMEOUT fixture::ignores_sigterm")); + } + + fn interrupt_stops_the_test_and_the_run() { + let dir = tempfile::tempdir().expect("temp dir"); + let pid_file = dir.path().join("pid"); + let (path, junit_arg) = junit(dir.path()); + let mut command = program( + "fixtures", + &[ + "run", "hangs", "passes", "--grace", "5", "--junit", &junit_arg, + ], + ); + command + .env(PID_FILE_ENV, &pid_file) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let child: Child = command.spawn().expect("start the run"); + let helper = read_pid(&pid_file); + // SAFETY: plain syscall to our own child, not yet reaped. + unsafe { + libc::kill( + libc::pid_t::try_from(child.id()).expect("pid"), + libc::SIGTERM, + ); + } + let run = child.wait_with_output().expect("wait for the run"); + eprintln!("{}", String::from_utf8_lossy(&run.stderr)); + assert_eq!(run.status.code(), Some(2)); + assert!(stderr(&run).contains("ERROR fixture::hangs: interrupted")); + assert_gone(helper); + let xml = std::fs::read_to_string(path).expect("junit report"); + assert!(xml.contains(r#"type="interrupted""#), "{xml}"); + assert!(!xml.contains("fixture::passes"), "{xml}"); + } + + fn interrupt_between_tests_ends_the_run() { + let dir = tempfile::tempdir().expect("temp dir"); + let (path, junit_arg) = junit(dir.path()); + let run = output(program( + "fixtures", + &[ + "run", + "interrupts_after_exit", + "passes", + "--junit", + &junit_arg, + ], + )); + assert_eq!(run.status.code(), Some(2)); + assert!(stderr(&run).contains("<== PASS fixture::interrupts_after_exit")); + assert!(stderr(&run).contains("interrupted; 1 later test did not run")); + assert!(!stdout(&run).contains("passes: stdout")); + let xml = std::fs::read_to_string(path).expect("junit report"); + assert!(!xml.contains("fixture::passes"), "{xml}"); + } + + fn a_passing_test_leaves_nothing_behind() { + let dir = tempfile::tempdir().expect("temp dir"); + let pid_file = dir.path().join("pid"); + let mut command = program("fixtures", &["run", "leaves_a_helper"]); + command.env(PID_FILE_ENV, &pid_file); + let run = output(command); + assert_eq!(run.status.code(), Some(0)); + assert_gone(read_pid(&pid_file)); + } + + /// Point the gateway lookup at a closed port, away from any registered + /// gateways. + fn without_gateway(mut command: Command, config: &Path) -> Command { + let closed = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); + let endpoint = format!("http://{}", closed.local_addr().expect("address")); + drop(closed); + command + .env("XDG_CONFIG_HOME", config) + .env("OPENSHELL_GATEWAY_ENDPOINT", endpoint) + .env_remove("OPENSHELL_GATEWAY") + .env("OPENSHELL_BIN", config.join("no-openshell")); + command + } + + fn registered_tests_run_through_testkit() { + let dir = tempfile::tempdir().expect("temp dir"); + let (path, junit_arg) = junit(dir.path()); + let run = output(without_gateway( + program("registered", &["run", "--junit", &junit_arg]), + dir.path(), + )); + assert_eq!(run.status.code(), Some(1)); + // `testkit::run` logs the description and the gateway. + assert!(stderr(&run).contains("[registered_needs_a_gateway] Needs a gateway.")); + assert!(stderr(&run).contains("1 failed, 1 skipped")); + let xml = std::fs::read_to_string(path).expect("junit report"); + assert!( + xml.contains(r#""#), + "{xml}" + ); + } + + fn ready_fails_without_a_gateway() { + let dir = tempfile::tempdir().expect("temp dir"); + let started = Instant::now(); + let ready = output(without_gateway( + program("registered", &["ready", "--timeout", "1"]), + dir.path(), + )); + assert_eq!(ready.status.code(), Some(2)); + assert!(stderr(&ready).contains("not ready after 1s")); + assert!(started.elapsed() < Duration::from_secs(10)); + } +} From e6ac23be247b127a49eed9c482bd675ee578e8cd Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 2 Nov 2026 13:04:28 -0800 Subject: [PATCH 2/5] refactor(conformance): move the conformance suites into a library Cargo builds each file in tests/ as its own test executable, which no other program can link. Move the conformance suites into src/ as modules of a library, so a test program can link them. Test code is unchanged; cargo test and nextest run the same 15 tests as the library's unit tests. Nextest now names them as binary `conformance`, test `::`, so filters that selected a suite by binary select it by test name instead. Signed-off-by: Matthew Grossman --- CI.md | 2 +- e2e/policy-advisor/README.md | 2 +- tests/runner/src/main.rs | 2 +- tests/suites/conformance/Cargo.toml | 2 +- tests/suites/conformance/README.md | 9 +++++---- tests/suites/conformance/{tests => src}/cli.rs | 0 .../conformance/{tests => src}/file_transfer.rs | 0 tests/suites/conformance/src/lib.rs | 16 ++++++++++++++++ .../conformance/{tests => src}/lifecycle.rs | 0 .../{tests => src}/persistent_workspace.rs | 0 .../conformance/{tests => src}/policy_advisor.rs | 0 tests/suites/conformance/{tests => src}/smoke.rs | 0 12 files changed, 25 insertions(+), 8 deletions(-) rename tests/suites/conformance/{tests => src}/cli.rs (100%) rename tests/suites/conformance/{tests => src}/file_transfer.rs (100%) create mode 100644 tests/suites/conformance/src/lib.rs rename tests/suites/conformance/{tests => src}/lifecycle.rs (100%) rename tests/suites/conformance/{tests => src}/persistent_workspace.rs (100%) rename tests/suites/conformance/{tests => src}/policy_advisor.rs (100%) rename tests/suites/conformance/{tests => src}/smoke.rs (100%) diff --git a/CI.md b/CI.md index 6fe38b9a8b..db4a8e7995 100644 --- a/CI.md +++ b/CI.md @@ -214,7 +214,7 @@ nextest filterset: nix run .#openshell-test-runner -- \ --environment ubuntu-docker-rootful --installer deb \ --family conformance --package conformance \ - --filter 'binary_id(=conformance::policy_advisor)' + --filter 'test(/^policy_advisor::/)' ``` A local run does not replace the required PR E2E gate. diff --git a/e2e/policy-advisor/README.md b/e2e/policy-advisor/README.md index c5c4e79ca0..adb700306f 100644 --- a/e2e/policy-advisor/README.md +++ b/e2e/policy-advisor/README.md @@ -69,7 +69,7 @@ test runner: nix run .#openshell-test-runner -- \ --environment ubuntu-docker-rootful --installer deb \ --family conformance --package conformance \ - --filter 'binary_id(=conformance::policy_advisor)' + --filter 'test(/^policy_advisor::/)' ``` The GitHub write test above and the regressions below still exercise distinct diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index dfe8958ada..59a8dd7a83 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -37,7 +37,7 @@ struct Cli { #[arg(long = "package", short = 'p', value_name = "PACKAGE")] packages: Vec, /// A nextest filterset narrowing the tests that run, e.g. - /// `binary(smoke)`. + /// `test(/^smoke::/)`. #[arg(long, value_name = "FILTERSET")] filter: Option, /// The gateway name the installer registers with the CLI. The packaged diff --git a/tests/suites/conformance/Cargo.toml b/tests/suites/conformance/Cargo.toml index 40541d88c4..f40930d409 100644 --- a/tests/suites/conformance/Cargo.toml +++ b/tests/suites/conformance/Cargo.toml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# The conformance family is one crate; each file in `tests/` is a suite. +# The conformance family is one crate; each module in `src/` is a suite. [package] name = "conformance" description = "Conformance suite: sandbox behavior on every compute driver" diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index ee0502d072..c645fe7fa8 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -2,7 +2,8 @@ The `conformance` crate holds the testkit suites for sandbox behavior that every compute driver must provide. The family is this one crate, its own workspace. -Each file in `tests/` is one suite. +It is a library: each module in `src/` is one suite, and `cargo test` and +nextest run the tests as the library's unit tests. | Suite | Covers | |---|---| @@ -28,9 +29,9 @@ gateway and in a tmachine VM, as CI does. ## Adding a test -Add a `#[testkit::test]` function to a suite, or a new file in `tests/` for a new -area. No playbook, test archive or workflow entry is needed: the runner builds -and runs every test in the crate. +Add a `#[testkit::test]` function to a suite, or for a new area, a new file in +`src/` and a `mod` line for it in `src/lib.rs`. No playbook, test archive or +workflow entry is needed: the runner builds and runs every test in the crate. ## What a test may assert diff --git a/tests/suites/conformance/tests/cli.rs b/tests/suites/conformance/src/cli.rs similarity index 100% rename from tests/suites/conformance/tests/cli.rs rename to tests/suites/conformance/src/cli.rs diff --git a/tests/suites/conformance/tests/file_transfer.rs b/tests/suites/conformance/src/file_transfer.rs similarity index 100% rename from tests/suites/conformance/tests/file_transfer.rs rename to tests/suites/conformance/src/file_transfer.rs diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs new file mode 100644 index 0000000000..728c138b9e --- /dev/null +++ b/tests/suites/conformance/src/lib.rs @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Conformance suite: sandbox behavior that every compute driver must +//! provide. Each module is one suite. +//! +//! `cargo test` and nextest run the tests as this library's unit tests. +//! `#[testkit::test]` also registers them, so a test program runs them by +//! linking this crate. + +mod cli; +mod file_transfer; +mod lifecycle; +mod persistent_workspace; +mod policy_advisor; +mod smoke; diff --git a/tests/suites/conformance/tests/lifecycle.rs b/tests/suites/conformance/src/lifecycle.rs similarity index 100% rename from tests/suites/conformance/tests/lifecycle.rs rename to tests/suites/conformance/src/lifecycle.rs diff --git a/tests/suites/conformance/tests/persistent_workspace.rs b/tests/suites/conformance/src/persistent_workspace.rs similarity index 100% rename from tests/suites/conformance/tests/persistent_workspace.rs rename to tests/suites/conformance/src/persistent_workspace.rs diff --git a/tests/suites/conformance/tests/policy_advisor.rs b/tests/suites/conformance/src/policy_advisor.rs similarity index 100% rename from tests/suites/conformance/tests/policy_advisor.rs rename to tests/suites/conformance/src/policy_advisor.rs diff --git a/tests/suites/conformance/tests/smoke.rs b/tests/suites/conformance/src/smoke.rs similarity index 100% rename from tests/suites/conformance/tests/smoke.rs rename to tests/suites/conformance/src/smoke.rs From 1d3ed2d62eeb33c8cac2868aeaf451a5592c2d91 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 2 Nov 2026 10:31:09 -0800 Subject: [PATCH 3/5] feat(conformance): add the openshell-conformance program The conformance crate now also builds openshell-conformance, a binary whose main calls testkit::main() and links the crate's library. It lists, waits for the gateway, and runs the conformance tests without cargo or nextest. It runs only the conformance suite and promises only the conformance contract; feature and driver suites are not part of it. The binary has test = false, so cargo test and nextest still see one conformance test binary, the library's unit tests, and the e2e lanes and the runner's filters are unchanged. The library is linked without cfg(test), so the program gets the registrations and none of the tokio test wrappers. tasks/scripts/check-conformance-program.sh checks that openshell-conformance list names the same tests as nextest; Branch Checks and mise test:rust run it. A dependency that is never named is not linked, so a suite the program forgets to name would silently go missing. testkit::main() now fails with exit 2 when a program links no tests, under every command. Signed-off-by: Matthew Grossman --- .github/workflows/branch-checks.yml | 1 + TESTING.md | 6 +++++ tasks/scripts/check-conformance-program.sh | 30 ++++++++++++++++++++++ tasks/test.toml | 1 + tests/suites/conformance/Cargo.toml | 8 ++++++ tests/suites/conformance/README.md | 25 ++++++++++++++++-- tests/suites/conformance/src/lib.rs | 4 +-- tests/suites/conformance/src/main.rs | 18 +++++++++++++ tests/testkit/README.md | 9 +++++++ tests/testkit/testkit/src/program/mod.rs | 9 +++++++ tests/testkit/testkit/tests/program.rs | 27 +++++++++++++++++-- 11 files changed, 132 insertions(+), 6 deletions(-) create mode 100755 tasks/scripts/check-conformance-program.sh create mode 100644 tests/suites/conformance/src/main.rs diff --git a/.github/workflows/branch-checks.yml b/.github/workflows/branch-checks.yml index a62d442bc9..7af659acd2 100644 --- a/.github/workflows/branch-checks.yml +++ b/.github/workflows/branch-checks.yml @@ -389,6 +389,7 @@ jobs: cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/testkit/Cargo.toml cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/tmachine/Cargo.toml cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/runner/Cargo.toml + tasks/scripts/check-conformance-program.sh rust-postgres-test: name: Rust PostgreSQL tests diff --git a/TESTING.md b/TESTING.md index f914f254a3..7894fa78a2 100644 --- a/TESTING.md +++ b/TESTING.md @@ -181,6 +181,12 @@ by `OPENSHELL_BIN`. See [tests/testkit/README.md](tests/testkit/README.md) for how to write and run tests. +The conformance crate also builds `openshell-conformance`, a program that +runs the conformance suite against a gateway without cargo or nextest +(`openshell-conformance list`, `ready`, `run`). It covers only the +conformance suite. See +[tests/suites/conformance/README.md](tests/suites/conformance/README.md). + Suites live under `tests/suites//`. Each family is a Cargo workspace with its own lockfile. Run the `conformance` suites against a local Docker gateway with: diff --git a/tasks/scripts/check-conformance-program.sh b/tasks/scripts/check-conformance-program.sh new file mode 100755 index 0000000000..3284f20418 --- /dev/null +++ b/tasks/scripts/check-conformance-program.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Check that `openshell-conformance list` names the same tests that nextest +# lists for the conformance crate, so the program runs the whole suite. +# Neither command needs a gateway. + +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +manifest="${root}/tests/suites/conformance/Cargo.toml" +work="$(mktemp -d)" +trap 'rm -rf "${work}"' EXIT + +cargo run --quiet --locked --manifest-path "${manifest}" --bin openshell-conformance -- list \ + | awk '{print $1}' | sort > "${work}/program" +cargo nextest list --locked --color never --manifest-path "${manifest}" --message-format oneline \ + | awk '{print $2}' | sort > "${work}/nextest" + +if ! diff -u "${work}/nextest" "${work}/program"; then + echo "error: openshell-conformance list differs from nextest's list (-nextest +program)" >&2 + exit 1 +fi +if [[ ! -s "${work}/program" ]]; then + echo "error: openshell-conformance lists no tests" >&2 + exit 1 +fi +echo "openshell-conformance lists the same $(wc -l < "${work}/program" | tr -d ' ') tests as nextest" diff --git a/tasks/test.toml b/tasks/test.toml index 558d7a2401..81b0585ace 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -127,6 +127,7 @@ run = [ "cargo nextest run --config-file .config/nextest.toml --manifest-path tests/testkit/Cargo.toml", "cargo nextest run --config-file .config/nextest.toml --manifest-path tests/tmachine/Cargo.toml", "cargo nextest run --config-file .config/nextest.toml --manifest-path tests/runner/Cargo.toml", + "tasks/scripts/check-conformance-program.sh", ] run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 test-precommit native" hide = true diff --git a/tests/suites/conformance/Cargo.toml b/tests/suites/conformance/Cargo.toml index f40930d409..99274d8dfa 100644 --- a/tests/suites/conformance/Cargo.toml +++ b/tests/suites/conformance/Cargo.toml @@ -12,6 +12,14 @@ license = "Apache-2.0" repository = "https://github.com/NVIDIA/OpenShell" publish = false +# `openshell-conformance`: the suite as one program to run against a gateway. +# No test target of its own: cargo test and nextest run the tests through the +# library. +[[bin]] +name = "openshell-conformance" +path = "src/main.rs" +test = false + [dependencies] openshell-testkit = { path = "../../testkit/testkit" } serde_json = "1" diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index c645fe7fa8..7592e776e5 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -3,7 +3,8 @@ The `conformance` crate holds the testkit suites for sandbox behavior that every compute driver must provide. The family is this one crate, its own workspace. It is a library: each module in `src/` is one suite, and `cargo test` and -nextest run the tests as the library's unit tests. +nextest run the tests as the library's unit tests. The crate also builds the +`openshell-conformance` program, which runs the same tests. | Suite | Covers | |---|---| @@ -27,11 +28,31 @@ environments). [TESTING.md](../../../TESTING.md) covers running it against a local Docker gateway and in a tmachine VM, as CI does. +### The `openshell-conformance` program + +`openshell-conformance` runs the suite against a gateway without cargo or +nextest, so a driver author or operator can check their own gateway. It finds +the gateway and the CLI as the tests do. Build it and run it with: + +```shell +cargo build --locked --manifest-path tests/suites/conformance/Cargo.toml --bin openshell-conformance +tests/suites/conformance/target/debug/openshell-conformance list +tests/suites/conformance/target/debug/openshell-conformance ready +tests/suites/conformance/target/debug/openshell-conformance run --junit conformance.xml +``` + +The program runs only this suite: passing it means a gateway meets the +conformance contract below. Feature and driver suites are not part of it. The +[testkit README](../../testkit/README.md#test-programs) describes the commands +and exit codes. `tasks/scripts/check-conformance-program.sh` checks that it +lists the same tests as nextest. + ## Adding a test Add a `#[testkit::test]` function to a suite, or for a new area, a new file in `src/` and a `mod` line for it in `src/lib.rs`. No playbook, test archive or -workflow entry is needed: the runner builds and runs every test in the crate. +workflow entry is needed: the runner builds and runs every test in the +crate, and `openshell-conformance` includes it. ## What a test may assert diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index 728c138b9e..00a4d45027 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -5,8 +5,8 @@ //! provide. Each module is one suite. //! //! `cargo test` and nextest run the tests as this library's unit tests. -//! `#[testkit::test]` also registers them, so a test program runs them by -//! linking this crate. +//! `#[testkit::test]` also registers them, so the `openshell-conformance` +//! program in `src/main.rs` runs them by linking this library. mod cli; mod file_transfer; diff --git a/tests/suites/conformance/src/main.rs b/tests/suites/conformance/src/main.rs new file mode 100644 index 0000000000..7b4e899c60 --- /dev/null +++ b/tests/suites/conformance/src/main.rs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `openshell-conformance`: the conformance suite as one program, to run +//! against a gateway without cargo or nextest. +//! +//! The tests are registered by `#[testkit::test]` in this crate's library; +//! see `testkit::program` for the commands. + +use std::process::ExitCode; + +// A dependency that is never named is not linked, and its tests would be +// missing. +use conformance as _; + +fn main() -> ExitCode { + testkit::main() +} diff --git a/tests/testkit/README.md b/tests/testkit/README.md index b9bd671fd6..4bba57931a 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -106,11 +106,20 @@ In a tmachine VM, the test runner (`tests/runner/`) does the same: it sets cargo or nextest: ```rust +use conformance as _; + fn main() -> std::process::ExitCode { testkit::main() } ``` +The program must name each suite crate it runs, as `use conformance as _;` does +above: Rust doesn't link a dependency that is never named, and its tests would +be missing. A program that links no tests exits 2 with an error under every +command. `openshell-conformance`, a binary in the conformance crate, is the +program for the conformance suite; see +[its README](../suites/conformance/README.md). + | Command | What it does | |---|---| | `list [FILTER]...` | Prints each test's name and description. Ignored tests are included and marked. | diff --git a/tests/testkit/testkit/src/program/mod.rs b/tests/testkit/testkit/src/program/mod.rs index eb8591833d..c7ca98f3ba 100644 --- a/tests/testkit/testkit/src/program/mod.rs +++ b/tests/testkit/testkit/src/program/mod.rs @@ -60,6 +60,15 @@ pub fn main() -> ExitCode { pub fn main_with(tests: &'static [Test]) -> ExitCode { let args: Vec = std::env::args_os().collect(); let program = program_name(&args); + // A suite crate that the program never names is not linked, and its + // tests are silently missing. Fail instead of passing with nothing. + if tests.is_empty() { + eprintln!( + "{program}: no tests are linked into this program; it must name each suite crate \ + it runs, for example with `use as _;`" + ); + return ExitCode::from(ERROR_EXIT); + } if let Err(error) = validate(tests) { eprintln!("{program}: {error}"); return ExitCode::from(ERROR_EXIT); diff --git a/tests/testkit/testkit/tests/program.rs b/tests/testkit/testkit/tests/program.rs index fed3a7ff55..c414e8dd27 100644 --- a/tests/testkit/testkit/tests/program.rs +++ b/tests/testkit/testkit/tests/program.rs @@ -10,13 +10,15 @@ //! - `fixtures`: tests that pass, fail, hang, ignore `SIGTERM`, or leave //! processes behind, which the checks run through the program's //! subcommands; -//! - `registered`: `testkit::main()` over the `#[testkit::test]`s below. +//! - `registered`: `testkit::main()` over the `#[testkit::test]`s below; +//! - `empty`: a program with no tests. #[cfg(unix)] fn main() -> std::process::ExitCode { match std::env::var("TESTKIT_FIXTURE").as_deref() { Ok("fixtures") => testkit::program::main_with(&unix::FIXTURES), Ok("registered") => testkit::main(), + Ok("empty") => testkit::program::main_with(&[]), _ => testkit::program::main_with(&unix::CHECKS), } } @@ -135,7 +137,7 @@ mod unix { }, ]; - pub static CHECKS: [Test; 11] = [ + pub static CHECKS: [Test; 12] = [ check("nextest_interface", || { Box::pin(async { nextest_interface() }) }), @@ -146,6 +148,9 @@ mod unix { Box::pin(async { run_reports_every_outcome() }) }), check("run_exit_codes", || Box::pin(async { run_exit_codes() })), + check("a_program_without_tests_fails", || { + Box::pin(async { a_program_without_tests_fails() }) + }), check("timeout_kills_the_process_group", || { Box::pin(async { timeout_kills_the_process_group() }) }), @@ -310,6 +315,24 @@ mod unix { assert_eq!(usage.status.code(), Some(2)); } + /// A program whose suites were not linked fails rather than passing with + /// no tests, under every interface. + fn a_program_without_tests_fails() { + for args in [ + &["list"][..], + &["run"], + &["--list", "--format", "terse"], + &[], + ] { + let empty = output(program("empty", args)); + assert_eq!(empty.status.code(), Some(2), "{args:?}"); + assert!( + stderr(&empty).contains("no tests are linked into this program"), + "{args:?}" + ); + } + } + fn is_alive(pid: libc::pid_t) -> bool { // SAFETY: signal 0 only checks the process exists. unsafe { libc::kill(pid, 0) == 0 } From 62e1f572e1d6c4f64be2dbb4d16d7f8423389c00 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 3 Nov 2026 13:11:32 -0800 Subject: [PATCH 4/5] test(e2e): run the conformance host lanes with openshell-conformance The Docker, Podman, VM and Kubernetes host lanes ran the conformance suite with cargo nextest. They now run it with openshell-conformance: ready, then run --junit, so CI exercises the program that gateway owners use. The script uses OPENSHELL_CONFORMANCE_BIN when set and otherwise builds the program from the checkout. The JUnit report goes to OPENSHELL_CONFORMANCE_JUNIT or results/conformance-.xml; the Kubernetes runner names it after its report so credential-driver runs keep both. Signed-off-by: Matthew Grossman --- TESTING.md | 4 +++- e2e/rust/e2e-kubernetes.sh | 3 ++- e2e/support/conformance.sh | 45 +++++++++++++++++++++++++++++--------- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/TESTING.md b/TESTING.md index 7894fa78a2..86ce03bc9a 100644 --- a/TESTING.md +++ b/TESTING.md @@ -420,7 +420,9 @@ Suites: - Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests. - Conformance (`tests/suites/conformance`) - the testkit `conformance` crate: lifecycle, the CLI, persistent workspaces, file transfer, and the policy - advisor. Driver E2E runs it with nextest against the lane's gateway. The + advisor. Driver E2E runs it with `openshell-conformance` against the lane's + gateway and writes a JUnit report under `results/`; set + `OPENSHELL_CONFORMANCE_BIN` to use a prebuilt program. The installed-artifact conformance lanes run the same crate through `openshell-test-runner` (see above). - Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or diff --git a/e2e/rust/e2e-kubernetes.sh b/e2e/rust/e2e-kubernetes.sh index 9bdcf4e6a2..fc90a827ee 100755 --- a/e2e/rust/e2e-kubernetes.sh +++ b/e2e/rust/e2e-kubernetes.sh @@ -79,7 +79,8 @@ run_conformance() { return 0 fi - e2e_run_openshell_conformance "Kubernetes" + OPENSHELL_CONFORMANCE_JUNIT="${OPENSHELL_CONFORMANCE_JUNIT:-${ROOT}/results/${OPENSHELL_E2E_REPORT_NAME:-e2e-kubernetes}-conformance.xml}" \ + e2e_run_openshell_conformance "Kubernetes" } # `OPENSHELL_E2E_REPORT_NAME` (default `e2e-kubernetes`) names the report diff --git a/e2e/support/conformance.sh b/e2e/support/conformance.sh index 26cb3ed9f6..82594439c0 100644 --- a/e2e/support/conformance.sh +++ b/e2e/support/conformance.sh @@ -16,30 +16,55 @@ e2e_require_openshell_bin() { return 2 fi - # Cargo runs each test from its package directory, so resolve the candidate - # relative to the caller's working directory before passing it to the tests. + # Tests may run from another directory, so resolve the candidate relative + # to the caller's working directory before passing it to them. local binary_dir binary_dir="$(cd "$(dirname "${OPENSHELL_BIN}")" && pwd -P)" || return export OPENSHELL_BIN="${binary_dir}/$(basename "${OPENSHELL_BIN}")" } +# Print the path of the openshell-conformance program: OPENSHELL_CONFORMANCE_BIN +# if set, else a debug build from this checkout. +e2e_openshell_conformance_bin() { + local root=$1 + + if [ -n "${OPENSHELL_CONFORMANCE_BIN:-}" ]; then + if [ ! -x "${OPENSHELL_CONFORMANCE_BIN}" ]; then + echo "ERROR: openshell-conformance is not executable: ${OPENSHELL_CONFORMANCE_BIN}" >&2 + return 2 + fi + echo "${OPENSHELL_CONFORMANCE_BIN}" + return + fi + + cargo build \ + --locked \ + --manifest-path "${root}/tests/suites/conformance/Cargo.toml" \ + --bin openshell-conformance >&2 || return + # Cargo resolves a relative CARGO_TARGET_DIR against the working directory. + local target_dir + target_dir="$(cd "${CARGO_TARGET_DIR:-${root}/tests/suites/conformance/target}" && pwd)" || return + echo "${target_dir}/debug/openshell-conformance" +} + +# Run the conformance suite with openshell-conformance. The JUnit report goes +# to OPENSHELL_CONFORMANCE_JUNIT, else results/conformance-.xml. e2e_run_openshell_conformance() { local gateway_label=${1:-OpenShell} - local root + local root program junit root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" e2e_require_openshell_bin || return + program="$(e2e_openshell_conformance_bin "${root}")" || return + junit="${OPENSHELL_CONFORMANCE_JUNIT:-${root}/results/conformance-$(echo "${gateway_label}" | tr '[:upper:]' '[:lower:]').xml}" + mkdir -p "$(dirname "${junit}")" || return - # The tests find the gateway as the CLI does: OPENSHELL_GATEWAY_ENDPOINT, + # The program finds the gateway as the CLI does: OPENSHELL_GATEWAY_ENDPOINT, # else OPENSHELL_GATEWAY, else the active gateway. The gateway wrappers set # these for the gateway they started. echo "==> Running conformance suites against the ${gateway_label} gateway" - cargo nextest run \ - --locked \ - --manifest-path "${root}/tests/suites/conformance/Cargo.toml" \ - --test-threads 1 \ - --no-fail-fast \ - --no-tests fail + "${program}" ready || return + "${program}" run --junit "${junit}" } if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then From b8a1ef3e3e8633dc0d8adb5625e9496ca712651d Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 3 Nov 2026 13:16:51 -0800 Subject: [PATCH 5/5] ci(e2e): build openshell-conformance once per architecture in Branch E2E Each Docker, Podman, VM and Kubernetes host lane built the conformance suite itself. Branch E2E now builds openshell-conformance once per architecture as a static musl binary, checks that it is static and lists its tests, and uploads it. setup-e2e-cli downloads the one that matches the runner when a lane sets prebuilt-conformance, and the lane runs it through OPENSHELL_CONFORMANCE_BIN. The job is separate from build-binaries.yml, which the release workflows also call: the program has its own workspace and is not released. Lanes called without prebuilt-conformance, such as the release qualification lanes, still build the program from the checkout. Signed-off-by: Matthew Grossman --- .github/actions/setup-e2e-cli/action.yml | 22 +++++ .github/workflows/branch-e2e.yml | 100 ++++++++++++++++++---- .github/workflows/e2e-docker-test.yml | 6 ++ .github/workflows/e2e-kubernetes-test.yml | 7 ++ .github/workflows/e2e-podman-test.yml | 6 ++ .github/workflows/e2e-vm-test.yml | 6 ++ TESTING.md | 3 +- 7 files changed, 134 insertions(+), 16 deletions(-) diff --git a/.github/actions/setup-e2e-cli/action.yml b/.github/actions/setup-e2e-cli/action.yml index 8b30970d8d..6bc2a78ec2 100644 --- a/.github/actions/setup-e2e-cli/action.yml +++ b/.github/actions/setup-e2e-cli/action.yml @@ -1,6 +1,12 @@ name: Setup E2E CLI description: Download architecture-matched prebuilt OpenShell host CLIs for E2E tests +inputs: + conformance: + description: Also download the prebuilt openshell-conformance program and set OPENSHELL_CONFORMANCE_BIN + required: false + default: "false" + runs: using: composite steps: @@ -17,3 +23,19 @@ runs: chmod +x "$cli" "$cli" --version echo "OPENSHELL_BIN=$cli" >> "$GITHUB_ENV" + + - name: Download prebuilt openshell-conformance + if: inputs.conformance == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ runner.arch == 'X64' && 'openshell-conformance-x86_64-unknown-linux-musl' || 'openshell-conformance-aarch64-unknown-linux-musl' }} + path: .e2e/prebuilt-conformance + + - name: Configure prebuilt openshell-conformance + if: inputs.conformance == 'true' + shell: bash + run: | # zizmor: ignore[github-env] persists only a fixed path under the trusted workspace + program="$GITHUB_WORKSPACE/.e2e/prebuilt-conformance/openshell-conformance" + chmod +x "$program" + "$program" list + echo "OPENSHELL_CONFORMANCE_BIN=$program" >> "$GITHUB_ENV" diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 3647f3ee02..18843fcaa5 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -144,6 +144,60 @@ jobs: cargo-version: ${{ needs.version.outputs.cargo }} extra-cargo-flags: --no-default-features + # The conformance program, built once per architecture for the host lanes. + # It lives in its own workspace and is not released, so it is not part of + # build-binaries.yml. + build-conformance: + name: openshell-conformance (${{ matrix.triple }}) + needs: [pr_metadata] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_any_e2e == 'true' + permissions: + contents: read + strategy: + matrix: + include: + - triple: x86_64-unknown-linux-musl + runner: linux-amd64-cpu8 + - triple: aarch64-unknown-linux-musl + runner: linux-arm64-cpu8 + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.sha }} + - uses: ./.github/actions/setup-nix + with: + prepare-shell: "true" + cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} + - uses: ./.github/actions/setup-rust + with: + cache-key: binaries-v2-openshell-conformance-${{ matrix.triple }} + save-cache: ${{ github.ref == 'refs/heads/main' }} + - name: Build openshell-conformance + shell: nix develop -c bash -euo pipefail {0} + env: + # Build into the root target directory, which setup-rust caches. + CARGO_TARGET_DIR: ${{ github.workspace }}/target + TRIPLE: ${{ matrix.triple }} + run: | + cargo build --release --locked --target "$TRIPLE" \ + --manifest-path tests/suites/conformance/Cargo.toml \ + --bin openshell-conformance + binary="target/$TRIPLE/release/openshell-conformance" + # The program must run on any lane host. + tasks/scripts/verify-static-binary.sh "$binary" + # Fails if the program has no tests linked. + "$binary" list + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: openshell-conformance-${{ matrix.triple }} + path: target/${{ matrix.triple }}/release/openshell-conformance + compression-level: 0 + retention-days: 5 + if-no-files-found: error + build-external-drivers: name: ${{ matrix.binary }} (${{ matrix.triple }}) needs: [pr_metadata, version] @@ -316,7 +370,7 @@ jobs: CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} docker-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -324,11 +378,12 @@ jobs: packages: read uses: ./.github/workflows/e2e-docker-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} runner: linux-arm64-cpu8 podman-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -336,12 +391,13 @@ jobs: packages: read uses: ./.github/workflows/e2e-podman-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} suite-matrix: >- [{"suite":"ci","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:ci"}] podman-additional-ca-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -349,21 +405,24 @@ jobs: packages: read uses: ./.github/workflows/e2e-podman-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} suite-matrix: >- [{"suite":"additional-ca","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"OPENSHELL_E2E_PODMAN_TEST=additional_ca mise run --no-deps --skip-deps e2e:podman"}] vm-e2e: - needs: [pr_metadata, build-binaries, build-vm-driver] + needs: [pr_metadata, build-binaries, build-conformance, build-vm-driver] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read contents: read packages: read uses: ./.github/workflows/e2e-vm-test.yml + with: + prebuilt-conformance: true vm-additional-ca-e2e: - needs: [pr_metadata, build-binaries, build-vm-driver] + needs: [pr_metadata, build-binaries, build-conformance, build-vm-driver] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -371,11 +430,12 @@ jobs: packages: read uses: ./.github/workflows/e2e-vm-test.yml with: + prebuilt-conformance: true suite-name: additional-ca test-name: additional_ca docker-external-driver-e2e: - needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-gateway-plain, build-external-drivers, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -383,6 +443,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-docker-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} runner: linux-arm64-cpu8 gateway-artifact: openshell-gateway-plain-aarch64-unknown-linux-gnu @@ -391,7 +452,7 @@ jobs: [{"suite":"external-driver","cmd":"mise run --no-deps --skip-deps e2e:docker:external-driver","apt_packages":"openssh-client","python_proto":false,"mcp":false}] podman-external-driver-e2e: - needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-gateway-plain, build-external-drivers, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -399,6 +460,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-podman-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu external-driver-binary: openshell-driver-podman @@ -406,7 +468,7 @@ jobs: [{"suite":"external-driver","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:external-driver"}] vm-external-driver-e2e: - needs: [pr_metadata, build-binaries, build-gateway-plain, build-vm-driver] + needs: [pr_metadata, build-binaries, build-conformance, build-gateway-plain, build-vm-driver] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -414,6 +476,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-vm-test.yml with: + prebuilt-conformance: true gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu suite-name: external-driver e2e-task: e2e:vm:external-driver @@ -430,7 +493,7 @@ jobs: image-tag: ${{ github.sha }} kubernetes-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' strategy: fail-fast: false @@ -454,13 +517,14 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: ${{ matrix.job_name }} agent-sandbox-version: ${{ matrix.agent_sandbox_version }} test-name: ${{ matrix.test_name }} kubernetes-app-listener-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' strategy: fail-fast: false @@ -488,6 +552,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: Kubernetes app listener (${{ matrix.name }}) e2e-task: e2e:kubernetes:app-listener @@ -497,7 +562,7 @@ jobs: listener-takeover: ${{ matrix.takeover }} kubernetes-workspace-managed-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -505,12 +570,13 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: Kubernetes E2E (workspace managed mode) e2e-task: e2e:kubernetes:workspace-managed kubernetes-external-driver-e2e: - needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-gateway-plain, build-external-drivers, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -518,6 +584,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: Kubernetes E2E (external compute driver) e2e-task: e2e:kubernetes:external-driver @@ -526,7 +593,7 @@ jobs: cluster-images: sandbox supervisor kubernetes-workspace-operator-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -534,12 +601,13 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: Kubernetes E2E (workspace operator mode) e2e-task: e2e:kubernetes:workspace-operator kubernetes-ha-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true' permissions: actions: read @@ -547,6 +615,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: Kubernetes HA E2E (Rust smoke) extra-helm-values: deploy/helm/openshell/ci/values-high-availability.yaml @@ -556,7 +625,7 @@ jobs: use-envoy-gateway: true kubernetes-credential-drivers-e2e: - needs: [pr_metadata, build-binaries, build-images] + needs: [pr_metadata, build-binaries, build-conformance, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true' permissions: actions: read @@ -564,6 +633,7 @@ jobs: packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: + prebuilt-conformance: true image-tag: ${{ github.sha }} job-name: Kubernetes Credential Drivers E2E e2e-task: e2e:kubernetes:credential-drivers diff --git a/.github/workflows/e2e-docker-test.yml b/.github/workflows/e2e-docker-test.yml index 501ded8d34..faa0121cd2 100644 --- a/.github/workflows/e2e-docker-test.yml +++ b/.github/workflows/e2e-docker-test.yml @@ -21,6 +21,10 @@ on: required: false type: string default: "" + prebuilt-conformance: + required: false + type: boolean + default: false external-driver-binary: required: false type: string @@ -67,6 +71,8 @@ jobs: persist-credentials: false - uses: ./.github/actions/setup-e2e-cli + with: + conformance: ${{ inputs.prebuilt-conformance }} - uses: ./.github/actions/setup-e2e-gateway with: diff --git a/.github/workflows/e2e-kubernetes-test.yml b/.github/workflows/e2e-kubernetes-test.yml index 867b8c1dd7..1ae616e36e 100644 --- a/.github/workflows/e2e-kubernetes-test.yml +++ b/.github/workflows/e2e-kubernetes-test.yml @@ -67,6 +67,11 @@ on: required: false type: string default: "" + prebuilt-conformance: + description: "Run conformance with the openshell-conformance artifact instead of building it" + required: false + type: boolean + default: false external-driver-binary: description: "Optional standalone compute driver binary used to compose a local test image" required: false @@ -124,6 +129,8 @@ jobs: - name: Use prebuilt OpenShell CLI uses: ./.github/actions/setup-e2e-cli + with: + conformance: ${{ inputs.prebuilt-conformance }} - name: Use prebuilt OpenShell gateway if: inputs.gateway-artifact != '' diff --git a/.github/workflows/e2e-podman-test.yml b/.github/workflows/e2e-podman-test.yml index d70ad6c60c..ac6d7d66ef 100644 --- a/.github/workflows/e2e-podman-test.yml +++ b/.github/workflows/e2e-podman-test.yml @@ -17,6 +17,10 @@ on: required: false type: string default: "" + prebuilt-conformance: + required: false + type: boolean + default: false external-driver-binary: required: false type: string @@ -52,6 +56,8 @@ jobs: persist-credentials: false - uses: ./.github/actions/setup-e2e-cli + with: + conformance: ${{ inputs.prebuilt-conformance }} - uses: ./.github/actions/setup-e2e-gateway with: diff --git a/.github/workflows/e2e-vm-test.yml b/.github/workflows/e2e-vm-test.yml index 28b4c81db5..82d86058b8 100644 --- a/.github/workflows/e2e-vm-test.yml +++ b/.github/workflows/e2e-vm-test.yml @@ -14,6 +14,10 @@ on: required: false type: string default: "" + prebuilt-conformance: + required: false + type: boolean + default: false suite-name: required: false type: string @@ -47,6 +51,8 @@ jobs: persist-credentials: false - uses: ./.github/actions/setup-e2e-cli + with: + conformance: ${{ inputs.prebuilt-conformance }} - uses: ./.github/actions/setup-e2e-gateway with: diff --git a/TESTING.md b/TESTING.md index 86ce03bc9a..287078a900 100644 --- a/TESTING.md +++ b/TESTING.md @@ -422,7 +422,8 @@ Suites: lifecycle, the CLI, persistent workspaces, file transfer, and the policy advisor. Driver E2E runs it with `openshell-conformance` against the lane's gateway and writes a JUnit report under `results/`; set - `OPENSHELL_CONFORMANCE_BIN` to use a prebuilt program. The + `OPENSHELL_CONFORMANCE_BIN` to use a prebuilt program, as Branch E2E does + with the static build it makes once per architecture. The installed-artifact conformance lanes run the same crate through `openshell-test-runner` (see above). - Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or