feat(e2e): add a chaos experiment harness

CONTRIBUTING.md

@@ -386,6 +386,7 @@ These are the primary `mise` tasks for day-to-day development:
 | `proto/`        | Protocol buffer definitions and [public API conventions](proto/README.md) |
 | `tasks/`        | `mise` task definitions and build scripts     |
 | `tests/testkit/` | Support library for SDK-driven end-to-end suites |
+| `e2e/chaos/`    | [Chaos experiments](e2e/chaos/README.md) against e2e gateways |
 | `deploy/`       | Dockerfiles, Helm chart, Kubernetes manifests |
 | `docs/`         | Published Fern docs source, navigation, and content assets |
 | `fern/`         | Fern site config, components, and theme assets |

e2e/chaos/.gitignore

@@ -0,0 +1,2 @@
+runs/
+.venv/

e2e/chaos/experiments/gateway-graceful-restart.yaml

@@ -0,0 +1,120 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: chaos/v1alpha1
+id: gateway-graceful-restart
+version: 2
+title: Counter instances resume after a graceful gateway restart
+hypothesis: >
+  When the gateway process gets SIGTERM and is started again, its
+  graceful-stop sweep stops the sandbox of every counter instance, the
+  restarted gateway starts each sandbox again, and each instance's script
+  resumes from its saved tick with no tick missing from its journal.
+description: >
+  Two counter instances run. 15s after both have been observed running,
+  gateway-restart stops and restarts the gateway. The checks encode current
+  behaviour, observed on 2026-10-27: each instance's script is started a
+  second time (the wrapper records two attempts), and the instance output
+  directory keeps its files across the restart.
+default-adapter: openshell-docker
+timeout: 15m
+
+workloads:
+  - name: counter
+    description: >
+      Counts from 1 to $TICKS, one tick every $INTERVAL seconds. Each tick
+      appends the line "<tick> <unix time>" to the instance's journal, then
+      saves the tick number atomically. If the script is started again in the
+      same output directory, it continues from the saved tick plus one.
+    kind: shell
+    instances: 2
+    timeout: 6m
+    env: {TICKS: "90", INTERVAL: "1"}
+    run: |
+      out=$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR
+      start=$(cat "$out/counter" 2>/dev/null || echo 0)
+      for i in $(seq $((start + 1)) "$TICKS"); do
+        echo "$i $(date +%s)" >> "$out/journal"
+        echo "$i" > "$out/counter.tmp" && mv "$out/counter.tmp" "$out/counter"
+        sleep "$INTERVAL"
+      done
+    checks:
+      - name: exited-ok
+        description: >
+          For each counter instance: the last attempt of the script exited
+          with code 0. An instance with no recorded exit code fails.
+        kind: exit-code
+        equals: 0
+      - name: journal-complete
+        description: >
+          For each counter instance: the journal has every tick from 1 to
+          $TICKS. A tick may appear twice, because a script that is started
+          again can repeat the tick the previous attempt was stopped on. A
+          missing tick fails.
+        kind: shell
+        run: |
+          journal=$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR/journal
+          if ! missing=$(diff <(seq 1 "$TICKS") <(cut -d" " -f1 "$journal" | sort -un)); then
+            echo "$missing" | head -5
+            echo "want each tick from 1 to $TICKS at least once"
+            exit 1
+          fi
+      - name: restarted
+        description: >
+          For each counter instance: .chaos/attempts has at least 2 lines, so
+          the script was started again after the gateway restart.
+        kind: shell
+        run: |
+          n=$(wc -l < "$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR/.chaos/attempts")
+          [ "$n" -ge 2 ] || { echo "want >= 2 attempts, got $n"; exit 1; }
+
+operations:
+  - name: gateway-restart
+    description: >
+      Sends SIGTERM to the gateway process and, if it has not exited after
+      60s, SIGKILL. Then runs the adapter's start hook, which waits until the
+      gateway's /healthz returns 200, and records target.restarted.
+    uses: restart
+    target: gateway
+    select: one
+    grace-period: 60s
+
+flow:
+  parallel:
+    - counter
+    - serial: [{wait: {workload: counter, condition: started}}, {delay: 15s}, gateway-restart]
+
+checks:
+  - name: all-supervisors-connected
+    description: >
+      At some scrape all 2 counter instances had their supervisor session
+      registered on the gateway, so the metric saw the full workload.
+    kind: metric
+    metric: openshell_server_supervisor_sessions
+    value: max
+    at-least: 2
+  - name: database-healthy-after-restart
+    description: >
+      The gateway's database readiness probe passed in the last scrape of the
+      run, after the restart.
+    kind: metric
+    metric: openshell_server_readiness_database_healthy
+    value: last
+    equals: 1
+
+metrics:
+  - family: openshell_server_supervisor_sessions
+    description: >
+      Supervisor sessions registered on the gateway: one per sandbox whose
+      supervisor is connected. It follows the instance count while sandboxes
+      run, and drops while a sandbox is restarting.
+  - family: openshell_server_readiness_database_healthy
+    description: >
+      1 while the gateway's database probe passes. The gateway restarts with
+      its database; a 0 after the restart would mean the gateway came back
+      before its state did.
+  - family: openshell_supervisor_policy_apply_results_total
+    description: >
+      Policy applies the supervisors reported, by outcome. A sandbox that
+      reconnects to the restarted gateway applies its policy again, so the
+      counter steps once per surviving sandbox.

e2e/chaos/experiments/gateway-kill-keeps-state.yaml

@@ -0,0 +1,146 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: chaos/v1alpha1
+id: gateway-kill-keeps-state
+version: 3
+title: Sandboxes survive a hard gateway kill
+hypothesis: >
+  After the gateway process gets SIGKILL, stays stopped for 15s and is started
+  again, each counter instance's script runs to completion without being
+  started again, and a sandbox created after every counter instance has ended
+  runs its script to exit code 0.
+description: >
+  Three counter instances run with restart-policy never. 20s after all three
+  have been observed running, gateway-down kills the gateway. After every
+  counter instance has ended, one fresh-sandbox instance is created, to show
+  that the restarted gateway can create new sandboxes and not only serve
+  existing ones. The checks encode desired behaviour.
+default-adapter: openshell-docker
+timeout: 15m
+expectations:
+  - issue: NVIDIA/OpenShell#4910
+    # The restarted gateway replaces each live supervisor, the sandbox runtime
+    # rejects the replacement, and the driver stops the sandbox container.
+    checks: [counter/exited-ok, counter/journal-complete]
+
+workloads:
+  - name: counter
+    description: >
+      Counts from 1 to $TICKS, one tick every $INTERVAL seconds. Each tick
+      appends the line "<tick> <unix time>" to the instance's journal, then
+      saves the tick number atomically. If the script is started again in the
+      same output directory, it continues from the saved tick plus one.
+    kind: shell
+    instances: 3
+    timeout: 5m
+    env: {TICKS: "120", INTERVAL: "1"}
+    options: {restart-policy: never}
+    run: |
+      out=$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR
+      start=$(cat "$out/counter" 2>/dev/null || echo 0)
+      for i in $(seq $((start + 1)) "$TICKS"); do
+        echo "$i $(date +%s)" >> "$out/journal"
+        echo "$i" > "$out/counter.tmp" && mv "$out/counter.tmp" "$out/counter"
+        sleep "$INTERVAL"
+      done
+    checks:
+      - name: exited-ok
+        description: >
+          For each counter instance: the last attempt of the script exited
+          with code 0. An instance with no recorded exit code fails.
+        kind: exit-code
+        equals: 0
+      - name: journal-complete
+        description: >
+          For each counter instance: the journal has the ticks 1 to $TICKS,
+          each once and in order, so no tick was skipped or repeated.
+        kind: shell
+        run: |
+          journal=$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR/journal
+          if ! diff=$(diff <(seq 1 "$TICKS") <(cut -d" " -f1 "$journal")); then
+            echo "$diff" | head -5
+            echo "want the ticks 1 to $TICKS, each once, in order"
+            exit 1
+          fi
+      - name: started-once
+        description: >
+          For each counter instance: .chaos/attempts has exactly 1 line, so
+          the script ran once and was not started again.
+        kind: shell
+        run: |
+          n=$(wc -l < "$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR/.chaos/attempts")
+          [ "$n" -eq 1 ] || { echo "want 1 attempt, got $n"; exit 1; }
+
+  - name: fresh-sandbox
+    description: >
+      One instance, created after every counter instance has ended. Its
+      script writes one file and exits 0.
+    kind: shell
+    instances: 1
+    timeout: 1m
+    run: echo ok > "$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR/hello"
+    checks:
+      - name: exited-ok
+        description: >
+          For the fresh-sandbox instance: the last attempt of the script exited
+          with code 0. An instance with no recorded exit code fails.
+        kind: exit-code
+        equals: 0
+
+operations:
+  - name: gateway-down
+    description: >
+      Sends SIGKILL to the gateway process and keeps it stopped for 15s. Then
+      runs the adapter's start hook, which waits up to 120s until the
+      gateway's /healthz returns 200, and records operation.reverted. The gateway
+      is unavailable for longer than 15s.
+    uses: outage
+    target: gateway
+    select: one
+    duration: 15s
+
+flow:
+  serial:
+    - parallel:
+        - counter
+        - serial: [{wait: {workload: counter, condition: started}}, {delay: 20s}, gateway-down]
+    - fresh-sandbox
+
+checks:
+  - name: gateway-recovered
+    description: >
+      The timeline has an operation.reverted event for gateway-down. The harness
+      records it only after the restarted gateway answered /healthz with 200.
+    kind: shell
+    env: {OPERATION: gateway-down}
+    run: |
+      # Slurp: jq 1.6 -e only looks at the last line of a stream.
+      jq -se --arg o "$OPERATION" 'any(.[]; .event == "operation.reverted" and .operation == $o)' \
+        "$CHAOS_EXPERIMENT_RUN_DIR/timeline.jsonl" > /dev/null \
+        || { echo "$OPERATION was never reverted"; exit 1; }
+  - name: database-healthy-after-restart
+    description: >
+      The gateway's database readiness probe passed in the last scrape of the
+      run, after the restart.
+    kind: metric
+    metric: openshell_server_readiness_database_healthy
+    value: last
+    equals: 1
+
+metrics:
+  - family: openshell_server_supervisor_sessions
+    description: >
+      Supervisor sessions registered on the gateway: one per sandbox whose
+      supervisor is connected. It follows the instance count while sandboxes
+      run, and drops while a sandbox is restarting.
+  - family: openshell_server_readiness_database_healthy
+    description: >
+      1 while the gateway's database probe passes. The gateway restarts with
+      its database; a 0 after the restart would mean the gateway came back
+      before its state did.
+  - family: openshell_supervisor_policy_apply_results_total
+    description: >
+      Policy applies the supervisors reported, by outcome. A sandbox that
+      reconnects to the restarted gateway applies its policy again, so the
+      counter steps once per surviving sandbox.

e2e/chaos/experiments/gateway-sigterm-during-recovery.yaml

@@ -0,0 +1,111 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: chaos/v1alpha1
+id: gateway-sigterm-during-recovery
+version: 1
+title: A SIGTERM during the gateway's startup recovery stops it gracefully
+hypothesis: >
+  When the gateway gets SIGTERM while its startup recovery sweep is starting
+  the persisted sandboxes again, it shuts down as it does after startup: it
+  logs the shutdown, leaves no driver operation pending, and the gateway that
+  starts next can start and delete every sandbox.
+description: >
+  Eight counter instances run. 15s after all have been observed running,
+  gateway-restart stops the gateway gracefully, which stops every sandbox,
+  and starts it again. The restarted gateway serves /healthz before its
+  recovery sweep, which starts the eight sandboxes one at a time, so 1s
+  after it is healthy gateway-sigterm-in-recovery sends SIGTERM while the
+  sweep is still running. On main the recovery sweep runs before the
+  SIGTERM handler is installed (crates/openshell-server/src/lib.rs), so
+  the process exits at once and the sandbox whose StartSandbox it had
+  claimed keeps driver_operation_pending. The next gateway retains that
+  operation, and the sandbox can then be neither started nor deleted, so
+  the run also records a failed delete and ends in ERROR.
+default-adapter: openshell-docker
+timeout: 15m
+
+workloads:
+  - name: counter
+    description: >
+      Counts from 1 to $TICKS, one tick every $INTERVAL seconds, saving the
+      tick so that a script started again resumes after it.
+    kind: shell
+    instances: 8
+    timeout: 4m
+    env: {TICKS: "90", INTERVAL: "1"}
+    run: |
+      out=$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR
+      start=$(cat "$out/counter" 2>/dev/null || echo 0)
+      for i in $(seq $((start + 1)) "$TICKS"); do
+        echo "$i $(date +%s)" >> "$out/journal"
+        echo "$i" > "$out/counter.tmp" && mv "$out/counter.tmp" "$out/counter"
+        sleep "$INTERVAL"
+      done
+    checks:
+      - name: exited-ok
+        description: >
+          For each counter instance: the last attempt of the script exited
+          with code 0, so the sandbox was started again after both restarts.
+        kind: exit-code
+        equals: 0
+
+operations:
+  - name: gateway-restart
+    description: >
+      Sends SIGTERM to the gateway process and, if it has not exited after
+      60s, SIGKILL. Then runs the adapter's start hook, which waits until the
+      gateway's /healthz returns 200.
+    uses: restart
+    target: gateway
+    select: one
+    grace-period: 60s
+  - name: gateway-sigterm-in-recovery
+    description: >
+      The same restart, sent while the restarted gateway's recovery sweep is
+      starting the stopped sandboxes.
+    uses: restart
+    target: gateway
+    select: one
+    grace-period: 60s
+
+flow:
+  parallel:
+    - counter
+    - serial:
+        - {wait: {workload: counter, condition: started}}
+        - {delay: 15s}
+        - gateway-restart
+        - {delay: 1s}
+        - gateway-sigterm-in-recovery
+
+checks:
+  - name: every-sigterm-logged
+    description: >
+      The gateway log has "Shutdown signal received; stopping gateway" at
+      least twice, once for each SIGTERM. A gateway that the default SIGTERM
+      disposition kills exits without logging it.
+    kind: shell
+    run: |
+      n=$(grep -c 'Shutdown signal received; stopping gateway' "$CHAOS_EXPERIMENT_RUN_DIR/logs/gateway.log" || true)
+      [ "$n" -ge 2 ] || { echo "want >= 2 graceful shutdowns, got $n"; exit 1; }
+  - name: no-pending-operation-retained
+    description: >
+      No gateway start retained a pending driver operation from the gateway
+      before it ("Retaining pending driver operation during gateway
+      recovery"). A retained operation blocks the sandbox's start, stop and
+      delete.
+    kind: shell
+    run: |
+      log=$CHAOS_EXPERIMENT_RUN_DIR/logs/gateway.log
+      if grep 'Retaining pending driver operation' "$log"; then
+        exit 1
+      fi
+
+expectations:
+  - issue: NVIDIA/OpenShell#4409
+    checks:
+      - experiment/every-sigterm-logged
+      - experiment/no-pending-operation-retained
+      - counter/exited-ok
+    kind: flaky