From 1d957e98b14ebd644c208c072130fd7a36fbfb91 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 4 Nov 2026 21:01:42 -0800 Subject: [PATCH 1/2] test(conformance): bound running-delete with delayed TERM Signed-off-by: Matthew Grossman --- tests/suites/conformance/README.md | 2 +- tests/suites/conformance/src/lifecycle.rs | 58 +++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index fe5c95b5f8..befd3e9273 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -9,7 +9,7 @@ nextest run the tests as the library's unit tests. The crate also builds the | Suite | Covers | |---|---| | `smoke` | Create a sandbox, run `echo` in it, delete it | -| `lifecycle` | SDK: long-running and run-to-completion sandboxes, the default command, restart of a stopped sandbox, deleting a stopped sandbox | +| `lifecycle` | SDK: long-running and run-to-completion sandboxes, running-delete within 10 seconds with a one-second TERM delay, the default command, restart of a stopped sandbox, deleting a stopped sandbox | | `cli` | The same lifecycle through the `openshell` CLI: status, create, get, list, exec, stop, start, delete | | `persistent-workspace` | Files in `/sandbox` survive stop and start | | `policy-advisor` | `policy.local` accepts a proposed rule; a denied address or hostname yields one draft rule | diff --git a/tests/suites/conformance/src/lifecycle.rs b/tests/suites/conformance/src/lifecycle.rs index 02e4f6f117..80656647a6 100644 --- a/tests/suites/conformance/src/lifecycle.rs +++ b/tests/suites/conformance/src/lifecycle.rs @@ -7,6 +7,8 @@ //! NVIDIA/OpenShell#4629 by Evan Lezar. Tests that need a driver with a //! persistent workspace live in `persistent_workspace.rs`. +use std::time::Duration; + use testkit::Context; use testkit::sdk::{SandboxPhase, SandboxRestartPolicy}; @@ -40,6 +42,62 @@ async fn long_running_exec_delete(cx: &mut Context) -> testkit::Result { cx.expect_absent(&sandbox).await } +/// Deleting a running workload that delays TERM handling by one second finishes +/// within ten seconds, including disappearance from get and list. +#[testkit::test] +async fn running_delete_with_delayed_term_completes_within_ten_seconds( + cx: &mut Context, +) -> testkit::Result { + let sandbox = cx + .sandbox("dt") + .command([ + "sh", + "-c", + "trap 'sleep 1; exit 0' TERM; \ + touch /tmp/delayed-term-ready; \ + while :; do sleep 3600 & wait $!; done", + ]) + .create() + .await?; + sandbox.wait_ready().await?; + + // Ready alone does not prove that the workload has installed its trap. + testkit::poll_until( + "workload TERM handler readiness", + Duration::from_secs(10), + Duration::from_millis(100), + async || { + let output = sandbox + .exec(["sh", "-c", "test -f /tmp/delayed-term-ready"]) + .await?; + Ok(if output.exit_code == 0 { + testkit::Poll::Ready(()) + } else { + testkit::Poll::Pending(format!( + "readiness check exited {}; stderr: {}", + output.exit_code, output.stderr + )) + }) + }, + ) + .await?; + + // Use one deadline starting before delete, not a fresh timeout per step. + // Boundary containment may kill the workload before its handler finishes; + // this asserts bounded deletion, not a graceful workload exit. + tokio::time::timeout(Duration::from_secs(10), async { + sandbox.delete().await?; + cx.expect_absent(&sandbox).await + }) + .await + .map_err(|_| { + testkit::Error::msg(format!( + "sandbox {} was not fully deleted within 10 seconds", + sandbox.name() + )) + })? +} + /// A sandbox whose main command exits 0, with restart policy Never, reaches /// Completed with exit code 0 and is not restarted. It is gone from get and /// list after delete. From 3974537eb526d0c123aafa8a640b9db0b8c8e60f Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 5 Nov 2026 09:38:40 -0800 Subject: [PATCH 2/2] test(conformance): describe slow-TERM delete and lengthen delay Signed-off-by: Matthew Grossman --- tests/suites/conformance/README.md | 2 +- tests/suites/conformance/src/lifecycle.rs | 34 +++++++++++++++-------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index befd3e9273..c756107cc1 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -9,7 +9,7 @@ nextest run the tests as the library's unit tests. The crate also builds the | Suite | Covers | |---|---| | `smoke` | Create a sandbox, run `echo` in it, delete it | -| `lifecycle` | SDK: long-running and run-to-completion sandboxes, running-delete within 10 seconds with a one-second TERM delay, the default command, restart of a stopped sandbox, deleting a stopped sandbox | +| `lifecycle` | SDK: long-running and run-to-completion sandboxes, prompt delete when the workload is slow to handle TERM, the default command, restart of a stopped sandbox, deleting a stopped sandbox | | `cli` | The same lifecycle through the `openshell` CLI: status, create, get, list, exec, stop, start, delete | | `persistent-workspace` | Files in `/sandbox` survive stop and start | | `policy-advisor` | `policy.local` accepts a proposed rule; a denied address or hostname yields one draft rule | diff --git a/tests/suites/conformance/src/lifecycle.rs b/tests/suites/conformance/src/lifecycle.rs index 80656647a6..6af1d145b3 100644 --- a/tests/suites/conformance/src/lifecycle.rs +++ b/tests/suites/conformance/src/lifecycle.rs @@ -42,21 +42,31 @@ async fn long_running_exec_delete(cx: &mut Context) -> testkit::Result { cx.expect_absent(&sandbox).await } -/// Deleting a running workload that delays TERM handling by one second finishes -/// within ten seconds, including disappearance from get and list. +/// Long enough that the workload is still handling TERM when sandbox shutdown +/// begins, and below the supervisor's 5s wait before it kills the workload. +const TERM_HANDLING_DELAY: Duration = Duration::from_secs(3); + +/// Well below the 30s termination grace period that a shutdown hang waits out. +const DELETE_DEADLINE: Duration = Duration::from_secs(10); + +/// Many workloads do not exit immediately on TERM, for example while they run +/// cleanup handlers. Deleting such a running sandbox should still complete +/// promptly, rather than waiting out the runtime's termination grace period. #[testkit::test] -async fn running_delete_with_delayed_term_completes_within_ten_seconds( +async fn delete_is_prompt_when_workload_is_slow_to_handle_term( cx: &mut Context, ) -> testkit::Result { + // `sleep & wait` rather than a foreground `sleep`: sh defers traps until a + // foreground child exits, but `wait` returns on the signal. + let script = format!( + "trap 'sleep {}; exit 0' TERM; \ + touch /tmp/delayed-term-ready; \ + while :; do sleep 3600 & wait $!; done", + TERM_HANDLING_DELAY.as_secs() + ); let sandbox = cx .sandbox("dt") - .command([ - "sh", - "-c", - "trap 'sleep 1; exit 0' TERM; \ - touch /tmp/delayed-term-ready; \ - while :; do sleep 3600 & wait $!; done", - ]) + .command(["sh", "-c", script.as_str()]) .create() .await?; sandbox.wait_ready().await?; @@ -85,14 +95,14 @@ async fn running_delete_with_delayed_term_completes_within_ten_seconds( // Use one deadline starting before delete, not a fresh timeout per step. // Boundary containment may kill the workload before its handler finishes; // this asserts bounded deletion, not a graceful workload exit. - tokio::time::timeout(Duration::from_secs(10), async { + tokio::time::timeout(DELETE_DEADLINE, async { sandbox.delete().await?; cx.expect_absent(&sandbox).await }) .await .map_err(|_| { testkit::Error::msg(format!( - "sandbox {} was not fully deleted within 10 seconds", + "sandbox {} was not fully deleted within {DELETE_DEADLINE:?}", sandbox.name() )) })?