runtime-adapter: start and run from a template, and run the stage's scenarios as one suite

.github/actions/validate-nvx/action.yml

@@ -18,6 +18,7 @@ runs:
         python3 scripts/nvx.py test-openvmm-unit --help
         python3 scripts/nvx.py test-openvmm --help
         python3 scripts/nvx.py test-microvm --help
+        python3 scripts/nvx.py test-runtime-adapter --help
         python3 scripts/nvx.py test-adversarial --help
         python3 scripts/nvx.py benchmark --help
         python3 scripts/nvx.py doctor --help
@@ -45,6 +46,8 @@ runs:
         if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
         python scripts\nvx.py test-microvm --help
         if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+        python scripts\nvx.py test-runtime-adapter --help
+        if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
         python scripts\nvx.py test-adversarial --help
         if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
         python scripts\nvx.py benchmark --help

.github/workflows/run-nvx-microvm-tests.yml

@@ -1274,19 +1274,23 @@ jobs:
           Invoke-Nvx sandbox deprovision --state-dir $state
           Remove-Item -Recurse -Force $share, $scratch
 
-      # The runtime adapter's hypervisor scenarios on this backend: the ignored
-      # tests of its host-only configuration, which read the backend through
-      # `nvx::Hypervisor::from_env_or_default`. Its host program launches VMs
-      # of the artifacts above, with the runtime identity that their build
-      # provenance gives, whose initramfs runs the adapter's guest agent in
-      # place of NVX's agents. Later processes of the host program reattach to
-      # a VM, as after a restart of the orchestrator, and stop it. The layered
-      # scenarios boot the Ubuntu EROFS image over a copy of the scratch
-      # template, from the guest artifacts, as the sandbox steps above do, and
-      # the guest agent's worker assembles the root outside PID 1. The guest
-      # agent is a static musl executable: Linux builds it, and Windows takes
-      # the one that distro-artifacts builds on Linux, as it takes the Ubuntu
-      # and Azure Linux images.
+      # The runtime adapter's consolidated suite on this backend: each scenario
+      # that tests/runtime-adapter/scenarios.txt names, by name, in its
+      # host-only configuration, whose VM scenarios are its ignored tests and
+      # read the backend through `nvx::Hypervisor::from_env_or_default`. The
+      # command fails once a scenario names a test that its target does not
+      # have, a named target has a test that no scenario names, the library or
+      # another target has an ignored test, or a named test does not run and
+      # pass. Its host program launches VMs of the artifacts above, with the
+      # runtime identity that their build provenance gives, whose initramfs runs
+      # the adapter's guest agent in place of NVX's agents. Later processes of
+      # the host program reattach to a VM, as after a restart of the
+      # orchestrator, and stop it. The layered scenarios boot the Ubuntu EROFS
+      # image over a copy of the scratch template, from the guest artifacts, as
+      # the sandbox steps above do, and the guest agent's worker assembles the
+      # root outside PID 1. The guest agent is a static musl executable: Linux
+      # builds it, and Windows takes the one that distro-artifacts builds on
+      # Linux, as it takes the Ubuntu and Azure Linux images.
       - name: Build the runtime adapter's guest agent on Linux
         if: runner.os != 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel
         shell: bash
@@ -1298,12 +1302,10 @@ jobs:
       - name: Run runtime adapter scenarios on Linux
         if: runner.os != 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel
         shell: bash
-        working-directory: tests/runtime-adapter
-        env:
-          NVX_HYPERVISOR: ${{ inputs.backend }}
-          NVX_ARTIFACTS_DIR: ${{ github.workspace }}
-          RUNTIME_ADAPTER_GUEST: ${{ github.workspace }}/tests/runtime-adapter/target/x86_64-unknown-linux-musl/release/runtime-adapter-guest
-        run: cargo test --locked --features host -- --ignored
+        run: >-
+          python3 scripts/nvx.py test-runtime-adapter
+          --backend "${{ inputs.backend }}"
+          --guest-agent tests/runtime-adapter/target/x86_64-unknown-linux-musl/release/runtime-adapter-guest
 
       - name: Download the runtime adapter's guest agent on Windows
         if: runner.os == 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel
@@ -1315,12 +1317,10 @@ jobs:
       - name: Run runtime adapter scenarios on Windows
         if: runner.os == 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel
         shell: powershell
-        working-directory: tests/runtime-adapter
-        env:
-          NVX_HYPERVISOR: ${{ inputs.backend }}
-          NVX_ARTIFACTS_DIR: ${{ github.workspace }}
-          RUNTIME_ADAPTER_GUEST: ${{ github.workspace }}\build\runtime-adapter\runtime-adapter-guest
-        run: cargo test --locked --features host -- --ignored
+        run: >-
+          python scripts\nvx.py test-runtime-adapter
+          --backend "${{ inputs.backend }}"
+          --guest-agent build\runtime-adapter\runtime-adapter-guest
 
       - name: Upload NVX microVM failure logs
         if: failure()

doc/ci.md

@@ -1858,9 +1858,14 @@ anything, it checks that each platform's package ships `bin/openvmm`, which
 `nvx-rs download` installs, and `bin/nvx-rs`, or their `.exe` forms on Windows.
 
 Each `nvx-microvm-guests-{kvm,mshv,whp}` job builds the runtime adapter's
-host-only configuration on its backend and runs its ignored tests, its VM
-scenarios, with `NVX_HYPERVISOR` naming the backend. Through NVX's
-supervisor, the adapter's host program
+host-only configuration on its backend and runs its consolidated suite, the
+[stage scenarios](../tests/runtime-adapter/README.md#stage-scenarios), with
+`scripts/nvx.py test-runtime-adapter`, which names the backend in
+`NVX_HYPERVISOR`. The command runs each scenario that
+`tests/runtime-adapter/scenarios.txt` names, by name, its VM scenarios
+included, which are the adapter's ignored tests, and fails once a scenario is
+renamed, added, or gone, or does not run and pass. Through NVX's supervisor,
+the adapter's host program
 launches a VM of the job's artifacts, whose initramfs runs the adapter's guest
 agent in place of NVX's agents, with the runtime identity that the artifacts'
 build provenance gives. Later processes reattach to the VM, as after a restart
@@ -1872,7 +1877,9 @@ where the guest agent's worker, outside PID 1, assembles the root through
 beneath it, through the root and through `nvx::guest::mappings` with reserved
 targets of its own, and those that the mapper refuses, and that a failed
 assembly, an invalid root configuration, and a failed teardown each reach the
-host program with the stage that failed, and still end the VM cleanly. The
+host program with the stage that failed, and still end the VM cleanly. Its
+template and clone scenarios capture templates, and start and run their
+clones. The
 Linux jobs build the guest agent, a static musl executable, first, and the
 Windows job takes the one that `distro-artifacts` builds on Linux, as it takes
 the Ubuntu and Azure Linux images. The adapter's other tests, including those

doc/contribute.md

@@ -119,8 +119,9 @@ mechanics, such as the conformance vectors of the control protocol, belong in
 `nvx`. Tests of what an independent adapter observes through
 the public API of `nvx` alone, such as its own framing over the control
 session between the host and the guest, belong in the
-[runtime adapter](../tests/runtime-adapter/README.md), whose ignored tests the
-microVM jobs run on each backend. [Continuous integration](ci.md)
+[runtime adapter](../tests/runtime-adapter/README.md), whose consolidated
+suite the microVM jobs run on each backend with
+`scripts/nvx.py test-runtime-adapter --backend BACKEND`. [Continuous integration](ci.md)
 describes what the OpenVMM VMM tests and the NVX microVM suite cover.
 
 ## Test expectations

doc/usage.md

@@ -438,6 +438,7 @@ is authorized for the organization when it enforces single sign-on.
 | `test-microvm` | Run NVX Linux and device correctness tests through OpenVMM. |
 | `plan-microvm-shards` | Split the default microVM suite into shards of balanced scenario durations for CI. |
 | `doctor` | Qualify this host for the NVX time ABI. |
+| `test-runtime-adapter` | Run the runtime adapter's consolidated suite by name on a real hypervisor. |
 | `test-adversarial` | Run a brokered Copilot-driven adversarial campaign. |
 | `build` | Build the guest artifacts and OpenVMM. |
 | `benchmark` | Run the OpenVMM-native benchmark coordinator. |
@@ -1266,6 +1267,34 @@ check and the subset that CI runs.
 | `--summary PATH` | none | Append a Markdown summary, for example to `$GITHUB_STEP_SUMMARY`. |
 | `--timeout SECONDS` | `120` | Set the seconds allowed for each probe or guest. `H4` gets its sampling window on top. |
 
+### `test-runtime-adapter`
+
+```text
+python3 scripts/nvx.py test-runtime-adapter
+    --backend {kvm,mshv,whp}
+    [--guest-agent PATH]
+    [--check-only]
+    [--cargo CARGO]
+```
+
+Runs the consolidated suite of the [runtime adapter](../tests/runtime-adapter/README.md),
+the [stage scenarios](../tests/runtime-adapter/README.md#stage-scenarios) that
+`tests/runtime-adapter/scenarios.txt` names, against a real hypervisor. Each
+line of that file names one of the adapter's test targets, then one of its
+tests. The command first checks that the lines name exactly the tests of their
+targets, and that neither the adapter's library nor another target has an
+ignored test, as the adapter's VM scenarios are, so that a scenario that is
+renamed, added, or gone fails it rather than drop out of the suite. It then
+runs each target's scenarios by
+name, in the adapter's host-only configuration, the ignored ones with the
+others, with the backend in `NVX_HYPERVISOR`, NVX's artifacts from the
+directory that `NVX_ARTIFACTS_DIR` names, from the current directory if it is
+relative, or this checkout unless it names one, and the adapter's guest agent
+from `--guest-agent` (default
+`tests/runtime-adapter/target/x86_64-unknown-linux-musl/release/runtime-adapter-guest`),
+and fails unless each of them ran and passed. `--check-only` checks the names
+alone, and needs neither a hypervisor nor the guest agent.
+
 ### `test-adversarial`
 
 ```text

scripts/nvx.py

@@ -105,6 +105,9 @@
     package_release,
     verify_source_tree,
 )
+from nvx_tools.runtime_adapter_tests import (
+    configure_parser as configure_runtime_adapter_test_parser,
+)
 from nvx_tools.shard_plans import parse_shard
 from nvx_tools.time_abi import (
     benchmark_cpu_profile_refusal,
@@ -867,6 +870,12 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
     )
     configure_doctor_parser(doctor)
 
+    runtime_adapter_tests = subparsers.add_parser(
+        "test-runtime-adapter",
+        help="run the runtime adapter's consolidated suite by name on a real hypervisor",
+    )
+    configure_runtime_adapter_test_parser(runtime_adapter_tests)
+
     adversarial_tests = subparsers.add_parser(
         "test-adversarial",
         help="run a brokered Copilot-driven adversarial campaign",

scripts/nvx_tools/ci.py

@@ -262,8 +262,9 @@
     r"scripts/nvx_tools/(?!(adversarial|adversarial_broker|adversarial_executor|"
     r"crate_release|create_linux_source_archive|managed_exec_tests|"
     r"microvm_durations|microvm_shards|microvm_tests|nightly_ci|"
-    r"openvmm_process|openvmm_test_shards|sandbox_lifecycle_tests|"
-    r"sandbox_proxy_tests|sandbox_ram_upper_tests|shard_plans)\.py$|"
+    r"openvmm_process|openvmm_test_shards|runtime_adapter_tests|"
+    r"sandbox_lifecycle_tests|sandbox_proxy_tests|sandbox_ram_upper_tests|"
+    r"shard_plans)\.py$|"
     r"(microvm|openvmm_test)_shards\.json$|adversarial_cases/|"
     r"microvm_test_scripts/))"
 )