diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b1034dd23f..e80b56a778 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -386,6 +386,7 @@ These are the primary `mise` tasks for day-to-day development: | `proto/` | Protocol buffer definitions and [public API conventions](proto/README.md) | | `tasks/` | `mise` task definitions and build scripts | | `tests/testkit/` | Support library for SDK-driven end-to-end suites | +| `e2e/chaos/` | [Chaos experiments](e2e/chaos/README.md) against e2e gateways | | `deploy/` | Dockerfiles, Helm chart, Kubernetes manifests | | `docs/` | Published Fern docs source, navigation, and content assets | | `fern/` | Fern site config, components, and theme assets | diff --git a/e2e/chaos/.gitignore b/e2e/chaos/.gitignore new file mode 100644 index 0000000000..8df7f27872 --- /dev/null +++ b/e2e/chaos/.gitignore @@ -0,0 +1,2 @@ +runs/ +.venv/ diff --git a/e2e/chaos/ADAPTER.md b/e2e/chaos/ADAPTER.md new file mode 100644 index 0000000000..9353c24db3 --- /dev/null +++ b/e2e/chaos/ADAPTER.md @@ -0,0 +1,417 @@ + + + +# Adapters + +An adapter maps the harness onto one system on one backend: how to create an +instance, read its state, collect its files and delete it; which processes, +containers, workloads or pods the built-in faults can act on; which +operations experiments can use; and what to observe. `adapters/.yaml` +holds one adapter. An experiment names the one it runs on with +`default-adapter: `, and `--adapter ` runs it on another, so the same +experiment manifest runs on every adapter that has what it uses. Two adapters +exist: `adapters/openshell-docker.yaml`, OpenShell on the Docker compute +driver, and `adapters/openshell-kubernetes.yaml`, OpenShell on the Kubernetes +compute driver. + +The harness knows nothing about OpenShell. Everything OpenShell-specific is in +the adapter's hooks. + +- [Where things run](#where-things-run) +- [Fields](#fields) +- [Hooks](#hooks) +- [Instance hooks](#instance-hooks) +- [Targets](#targets) +- [Operations](#operations) +- [Observers](#observers) +- [openshell-docker](#openshell-docker) +- [openshell-kubernetes](#openshell-kubernetes) +- [Write an adapter](#write-an-adapter) + +## Where things run + +Code runs in three places. + +```mermaid +flowchart LR + host["Harness host
the machine that runs chaos
hooks, checks, built-in faults,
metrics scrapes, watch commands"] + system["System under test
gateway, Docker daemon or cluster
nothing runs here"] + sandbox["Sandbox
the workload's run script"] + host -- "openshell, docker and kubectl CLIs,
signals to process targets" --> system + system -- "openshell sandbox create" --> sandbox +``` + +- The harness host, the machine that runs `chaos`. Every adapter hook runs + here as a host subprocess, `bash -c '\n + + + diff --git a/e2e/chaos/src/chaos/rundir.py b/e2e/chaos/src/chaos/rundir.py new file mode 100644 index 0000000000..cc208c2c3e --- /dev/null +++ b/e2e/chaos/src/chaos/rundir.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The run dir's manifest and its schema version.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from chaos.model import ExperimentError + +# The layout of the run dir and its files, written to manifest.json as +# `schema_version`. Every reader of a run dir refuses any other value; there +# is no compatibility code. +SCHEMA_VERSION = "v1alpha1" + + +def read_manifest(run_dir: Path) -> dict[str, Any]: + """The manifest of a run dir, or ExperimentError if it is missing or was + written under another schema version.""" + path = run_dir / "manifest.json" + if not path.is_file(): + raise ExperimentError(f"{run_dir}: no manifest.json; not a run dir") + manifest = json.loads(path.read_text()) + version = manifest.get("schema_version") + if version != SCHEMA_VERSION: + raise ExperimentError( + f"{path}: schema_version {version!r} is not {SCHEMA_VERSION!r}; " + "this harness cannot read it" + ) + return manifest diff --git a/e2e/chaos/src/chaos/runner.py b/e2e/chaos/src/chaos/runner.py new file mode 100644 index 0000000000..2a18b1a637 --- /dev/null +++ b/e2e/chaos/src/chaos/runner.py @@ -0,0 +1,442 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""One run: the run directory, the flow under a timeout, and the abort path. + +Abort path, in order: undo applied faults; restart process targets that are +down; collect and delete every instance still present; run the adapter +`collect`; run every check; write the verdict, forced to ERROR. A second +SIGINT/SIGTERM skips to deleting instances and exits 2. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import hashlib +import importlib.metadata +import json +import logging +import os +import platform as host +import secrets +import signal +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import yaml + +from chaos import envvars +from chaos.adapter import Adapter, run_hook +from chaos.envvars import RunInfo +from chaos.evaluate import ( + EXIT_CODES, + Result, + append_index, + format_verdict, + run_checks, + run_preflight, + write_verdict, +) +from chaos.flow import Engine +from chaos.inject.base import Injector +from chaos.inject.docker import DockerInjector +from chaos.inject.kubernetes import KubernetesInjector +from chaos.instances import InstanceManager, write_data +from chaos.model import Experiment, RunError, parse_duration +from chaos.observe import Observers +from chaos.operations import Operations +from chaos.report import write_report +from chaos.rundir import SCHEMA_VERSION +from chaos.signals import Signals +from chaos.timeline import Timeline, utc_now + +log = logging.getLogger("chaos") + +PROFILE_COLLECT_TIMEOUT = 300.0 +FORCED_RESTORE_TIMEOUT = 120.0 +INJECTORS: dict[str, type[Injector]] = { + "docker": DockerInjector, + "kubernetes": KubernetesInjector, +} +# Reverting operations, collecting and deleting after an error, all together. +CLEANUP_TIMEOUT = 600.0 + + +def new_run_id() -> str: + return f"{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}-{secrets.token_hex(2)}" + + +def environment() -> dict[str, Any]: + """The harness and its host. `system` is what the adapter's `describe` + hook prints about the system under test, filled in after preflight.""" + return { + "harness": importlib.metadata.version("openshell-chaos"), + "os": host.system(), + "arch": host.machine(), + "python": host.python_version(), + "system": None, + } + + +def descriptions(exp: Experiment) -> dict[str, Any]: + return { + "experiment": exp.description, + "workloads": { + w.name: { + "description": w.description, + "checks": {c.name: c.description for c in w.checks}, + } + for w in exp.workloads + }, + "operations": {o.name: o.description for o in exp.operations}, + "checks": {c.name: c.description for c in exp.checks}, + } + + +def _setup_logging(run_dir: Path) -> logging.Handler: + handler = logging.FileHandler(run_dir / "logs" / "harness.log") + handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s")) + log.addHandler(handler) + return handler + + +class Run: + def __init__( + self, + exp: Experiment, + raw: dict[str, Any], + experiment_path: Path, + adapter: Adapter, + seed: int, + overrides: list[str], + runs_dir: Path, + environ: dict[str, str], + ) -> None: + self.exp = exp + self.adapter = adapter + self.runs_dir = runs_dir + run_id = new_run_id() + run_dir = runs_dir / exp.id / run_id + (run_dir / "logs").mkdir(parents=True) + (run_dir / "system").mkdir() + (run_dir / "workloads").mkdir() + self.run_dir = run_dir + self.info = RunInfo( + experiment_id=exp.id, + experiment_version=exp.version, + adapter=adapter.id, + run_id=run_id, + seed=seed, + run_dir=run_dir, + ) + (run_dir / "experiment.yaml").write_text(yaml.safe_dump(raw, sort_keys=False)) + # The adapter as it ran, so `evaluate` needs nothing outside the run dir. + (run_dir / "adapter.yaml").write_bytes(adapter.path.read_bytes()) + for name, value in exp.data.items(): + write_data(run_dir, name, value) + # The sha256 is of the archived copy, the experiment after --set, so + # an evaluation can check that the copy it reads is what ran. + archived = (run_dir / "experiment.yaml").read_bytes() + self.manifest: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "run_id": run_id, + "experiment": { + "id": exp.id, + "version": exp.version, + "file": str(experiment_path), + "sha256": hashlib.sha256(archived).hexdigest(), + }, + "adapter": { + "id": adapter.id, + "file": str(adapter.path), + "sha256": adapter.sha256, + "backend": adapter.backend, + }, + "seed": seed, + "overrides": overrides, + "environment": environment(), + "started_at": utc_now(), + "finished_at": None, + "descriptions": descriptions(exp), + } + self._write_manifest() + self.timeline = Timeline(run_dir / "timeline.jsonl") + self.instances = InstanceManager(self.info, adapter, self.timeline) + self.injector = INJECTORS[adapter.backend]( + adapter, self.info, self.timeline, self.instances + ) + self.operations = Operations( + adapter, self.info, self.timeline, self.instances, self.injector + ) + self.signal_events = Signals() + self.observers = Observers( + adapter, + self.timeline, + run_dir, + envvars.run_hook_env(self.info), + self.signal_events, + ) + self.errors: list[str] = [] + self.preflight: list[Result] = [] + self.signals = 0 + self.flow_task: asyncio.Task[None] | None = None + self.main_task: asyncio.Task[int] | None = None + + def _write_manifest(self) -> None: + (self.run_dir / "manifest.json").write_text( + json.dumps(self.manifest, indent=2) + "\n" + ) + + def _on_signal(self, sig: signal.Signals) -> None: + self.signals += 1 + if self.signals == 1: + log.warning( + "%s: aborting; send again to skip collection and exit", sig.name + ) + self.errors.append(f"aborted by {sig.name}") + if self.flow_task is not None and not self.flow_task.done(): + self.flow_task.cancel() + elif self.signals == 2 and self.main_task is not None: + log.warning("%s: deleting instances; send again to exit at once", sig.name) + self.main_task.cancel() + else: + log.warning("%s: exiting now; faults and instances are left", sig.name) + for proc in self.observers.procs: + with contextlib.suppress(ProcessLookupError): + os.killpg(proc.pid, signal.SIGKILL) + os._exit(2) + + def execute(self) -> int: + """Run to completion and return the exit code.""" + handler = _setup_logging(self.run_dir) + try: + return asyncio.run(self._outer()) + finally: + log.removeHandler(handler) + self.timeline.close() + + async def _outer(self) -> int: + loop = asyncio.get_running_loop() + for sig in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(sig, self._on_signal, sig) + self.main_task = asyncio.create_task(self._main()) + try: + return await self.main_task + except asyncio.CancelledError: + return await self._forced_exit( + "forced exit: instances deleted without collection; checks not run" + ) + except Exception as err: # noqa: BLE001 (a harness bug is still an ERROR verdict) + log.exception("harness error") + self.errors.append(f"harness error: {err!r}") + return await self._forced_exit( + "instances deleted without collection after a harness error" + ) + + async def _main(self) -> int: + log.info("run %s -> %s", self.info.run_id, self.run_dir) + self.timeline.emit( + "run.started", + experiment=self.exp.id, + version=self.exp.version, + seed=self.info.seed, + adapter=self.info.adapter, + ) + self.preflight = await run_preflight(self.exp, self.adapter, self.info) + failed = [r.check for r in self.preflight if not r.ok] + if failed: + self.timeline.emit("preflight.failed", checks=failed) + self.errors.append(f"preflight failed: {', '.join(failed)}") + return await self._evaluate(checks=False) + await self._describe() + await self._setup() + # A signal before the flow exists only records an error; honor it here. + if not self.errors and not self.signals: + await self.observers.start() + await self._flow() + if not self.errors: + # Adapter operations still applied are put back, newest first, + # before the instances they may act on are gone. + self.errors.extend(await self.injector.undo_all(abort=False)) + if self.errors: + await self._abort() + await self._collect_run() + await self._stop_observers() + return await self._evaluate() + + async def _stop_observers(self) -> None: + """Observers run until adapter collection ends. Their failures are + recorded but never change the verdict.""" + await self.observers.stop() + for error in self.observers.errors: + self.timeline.emit("observer.failed", error=error) + self.observers.errors = [] + + async def _describe(self) -> None: + """Record what the adapter's `describe` hook says about the system + under test, verbatim, as manifest.json `environment.system`.""" + result = await run_hook( + self.adapter, + "describe", + self.adapter.describe, + envvars.run_hook_env(self.info), + ) + if not result.ok: + self.errors.append( + f"describe failed (exit {result.code}): {result.stderr.strip()[-2000:]}" + ) + return + try: + system = json.loads(result.stdout) + except ValueError as err: + self.errors.append(f"describe printed no JSON object: {err}") + return + if not isinstance(system, dict): + self.errors.append( + f"describe printed a JSON {type(system).__name__}, not an object" + ) + return + self.manifest["environment"]["system"] = system + self._write_manifest() + + async def _setup(self) -> None: + if self.adapter.setup is None: + return + result = await run_hook( + self.adapter, "setup", self.adapter.setup, envvars.run_hook_env(self.info) + ) + if not result.ok: + self.errors.append( + f"setup failed (exit {result.code}): {result.stderr.strip()[-2000:]}" + ) + + async def _flow(self) -> None: + engine = Engine( + self.exp, + self.timeline, + self.instances, + self.operations, + self.signal_events, + ) + self.flow_task = asyncio.create_task( + asyncio.wait_for(engine.run(), parse_duration(self.exp.timeout)) + ) + try: + await self.flow_task + except asyncio.CancelledError: + current = asyncio.current_task() + if current is not None and current.cancelling(): + raise + # Cancelled by the first signal; the error is already recorded. + except TimeoutError: + self.errors.append(f"experiment timeout {self.exp.timeout} expired") + except RunError as err: + self.errors.append(str(err)) + except Exception as err: # noqa: BLE001 (any harness bug is an ERROR verdict) + log.exception("harness error") + self.errors.append(f"harness error: {err!r}") + + async def _abort(self) -> None: + """Revert operations, then collect and delete every instance, within + CLEANUP_TIMEOUT. What is left at the deadline is deleted uncollected.""" + self.timeline.emit("abort", errors=self.errors) + loop = asyncio.get_running_loop() + self.instances.deadline = loop.time() + CLEANUP_TIMEOUT + try: + async with asyncio.timeout_at(self.instances.deadline): + await self._cleanup() + except TimeoutError: + self.errors.append( + f"cleanup did not finish in {CLEANUP_TIMEOUT:g}s; " + "instances left were deleted without collection" + ) + self.instances.deadline = loop.time() + FORCED_RESTORE_TIMEOUT + await self._delete_all() + finally: + self.instances.deadline = None + + async def _delete_all(self) -> None: + """One delete attempt per instance left. A failure is an error with + the instance's id, and the verdict says the cleanup is incomplete.""" + results = await asyncio.gather( + *(self.instances.delete_only(i) for i in self.instances.all()), + return_exceptions=True, + ) + failed = [str(r) for r in results if isinstance(r, BaseException)] + if failed: + self.errors.extend(failed) + self.errors.append( + f"cleanup incomplete: {len(failed)} instance(s) not deleted" + ) + + async def _cleanup(self) -> None: + self.errors.extend(await self.injector.undo_all()) + try: + await self.injector.ensure_up() + except Exception as err: # noqa: BLE001 (keep going to delete instances) + self.errors.append(f"restart targets: {err}") + results = await asyncio.gather( + *(self.instances.finish(i) for i in self.instances.all() if i.present), + return_exceptions=True, + ) + self.errors.extend(str(r) for r in results if isinstance(r, BaseException)) + + async def _collect_run(self) -> None: + if self.adapter.collect is None: + return + result = await run_hook( + self.adapter, + "collect", + self.adapter.collect, + envvars.run_hook_env(self.info), + timeout=PROFILE_COLLECT_TIMEOUT, + ) + (self.run_dir / "logs" / "collect.log").write_text( + result.stdout + result.stderr + ) + if not result.ok: + self.errors.append(f"adapter collect failed (exit {result.code})") + + async def _evaluate(self, checks: bool = True) -> int: + results = list(self.preflight) + if checks: + results += await run_checks( + self.exp, self.adapter, self.info, self.timeline + ) + verdict = write_verdict(self.run_dir, results, self.errors) + # The gateway processes seen across restarts, when a fault recorded any. + extra = {"processes": self.injector.pids} if self.injector.pids else {} + self._finish(verdict, **extra) + print(format_verdict(verdict)) + print(self.run_dir) + return EXIT_CODES[verdict["verdict"]] + + def _finish(self, verdict: dict[str, Any], **extra: Any) -> None: + self.timeline.emit("run.finished", verdict=verdict["verdict"], **extra) + self.manifest["finished_at"] = utc_now() + self._write_manifest() + append_index( + self.runs_dir / "index.jsonl", self.manifest, verdict, self.run_dir + ) + try: + write_report(self.run_dir) + except Exception as err: # noqa: BLE001 (`chaos report` can retry it) + log.warning("report.html not written: %r", err) + + async def _forced_exit(self, reason: str) -> int: + """Second signal or harness error: best-effort revert operations and delete + what is left, write an ERROR verdict, exit 2.""" + self.timeline.emit("abort", forced=True) + await self._stop_observers() + try: + async with asyncio.timeout(FORCED_RESTORE_TIMEOUT): + self.errors.extend(await self.injector.undo_all()) + await self.injector.ensure_up() + except Exception as err: # noqa: BLE001 (deleting instances matters more) + self.errors.append(f"restore operations: {err!r}") + await self._delete_all() + self.errors.append(reason) + self._finish(write_verdict(self.run_dir, list(self.preflight), self.errors)) + return 2 diff --git a/e2e/chaos/src/chaos/share.py b/e2e/chaos/src/chaos/share.py new file mode 100644 index 0000000000..becb55bb64 --- /dev/null +++ b/e2e/chaos/src/chaos/share.py @@ -0,0 +1,431 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""`chaos share RUN_DIR`: one Markdown page of a run to paste into an issue +comment or a post. It holds the verdict and the failed checks, a Mermaid +gantt of the flow (workloads, waits, operations, faults) over the instance +states, and the metrics the experiment lists under `metrics:` as a table of +their check results and one Mermaid chart each. GitHub draws the Mermaid +blocks; `--png` renders them with mermaid-cli for places that do not. + +Everything comes from `chaos.report.build`, so the page agrees with +`report.html` and works on a copied run dir. +""" + +from __future__ import annotations + +import math +import re +import shutil +import subprocess +from decimal import Decimal +from pathlib import Path +from typing import Any + +import yaml + +from chaos.report import build + +# Gantt rows before the instance sections are left out: Mermaid draws one +# row per task, and GitHub refuses large diagrams. +ROW_BUDGET = 40 +# A node or fault with the same name more often than this is one row. +COLLAPSE_AFTER = 3 +# Points per chart; longer series are averaged into this many buckets. +MAX_POINTS = 60 +# Charts shown before the rest fold. +CHARTS_SHOWN = 2 +# Mermaid gantt tags by instance state; the others take the default look. +TAG = { + "Running": "active", + "Succeeded": "done", + "Unknown": "crit", + "Failed": "crit", + "Error": "crit", +} +# Instances kept first when the chart cannot hold them all: a check failed +# on them, or they ended badly, then those that went through Unknown. +ENDED_BADLY = {"Failed", "Error"} +# The default xychart line is nearly white on Mermaid's default theme. +CHART_INIT = ( + '%%{init: {"theme": "base", "themeVariables": ' + '{"xyChart": {"plotColorPalette": "#1f3fb5"}}}}%%' +) + + +class ShareError(Exception): + pass + + +def _clock(t: float, hours: bool) -> str: + t = max(0, round(t)) + h, rest = divmod(t, 3600) + m, s = divmod(rest, 60) + return f"{h:02d}:{m:02d}:{s:02d}" if hours else f"{m:02d}:{s:02d}" + + +def _label(text: str) -> str: + """Text safe inside a gantt task or section line.""" + return " ".join(re.sub(r"[:;,#%]+", " ", str(text)).split()) or "-" + + +def _task(label: str, tags: list[str], start: float, end: float, hours: bool) -> str: + a, b = _clock(start, hours), _clock(end, hours) + # A task shorter than a second has no width; a milestone has a diamond. + tags = ["milestone", *tags] if a == b else tags + return f" {_label(label)} :{', '.join([*tags, a, b])}" + + +def flow_spans(timeline: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Every flow node that started, by name: a workload or operation, a + `wait` or a `delay`. A node still open at the end of the timeline ends + there. A node id that starts again (under `repeat`) is a new span.""" + end = max((e["t"] for e in timeline), default=0.0) + open_: dict[str, dict[str, Any]] = {} + pairs: list[tuple[dict[str, Any], float]] = [] + for e in timeline: + if e["event"] == "node.started": + open_[e["node"]] = e + elif e["event"] == "node.finished" and e["node"] in open_: + pairs.append((open_.pop(e["node"]), e["t"])) + pairs.extend((s, end) for s in open_.values()) + spans = [] + for s, to in pairs: + if "name" in s: + kind, name = "node", str(s["name"]) + elif "wait" in s: + kind, name = "wait", f"wait {s['wait']} {s.get('condition', 'started')}" + elif "delay" in s: + kind, name = "delay", f"delay {s['delay']}" + else: + continue + spans.append({"kind": kind, "name": name, "from": s["t"], "to": to}) + return sorted(spans, key=lambda s: (s["from"], s["to"])) + + +def _collapse(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Rows with the same name, more than COLLAPSE_AFTER of them, become one + row from the first start to the last end, named `name ×N`.""" + groups: dict[str, list[dict[str, Any]]] = {} + for row in rows: + groups.setdefault(row["name"], []).append(row) + out = [] + for name, group in groups.items(): + if len(group) > COLLAPSE_AFTER: + out.append( + { + **group[0], + "name": f"{name} ×{len(group)}", + "from": min(r["from"] for r in group), + "to": max(r["to"] for r in group), + } + ) + else: + out.extend(group) + return sorted(out, key=lambda r: (r["from"], r["to"])) + + +def gantt(data: dict[str, Any]) -> tuple[str, str]: + """The gantt text and a note on what it leaves out ('' when nothing).""" + hours = data["end"] >= 3600 + title = " · ".join( + str(x) for x in (data["experiment"]["id"], data["run_id"], data["verdict"]) if x + ) + lines = [ + "gantt", + f" title {_label(title)}", + f" dateFormat {'HH:mm:ss' if hours else 'mm:ss'}", + f" axisFormat {'%H:%M:%S' if hours else '%M:%S'}", + " todayMarker off", + " section flow", + ] + # A built-in operation shows as its fault window, not as its flow node. + fault_names = {f["name"] for f in data["faults"]} + nodes = [s for s in flow_spans(data["timeline"]) if s["name"] not in fault_names] + faults = [ + { + "name": f"fault {f['name']} ({f['uses']})", + "from": f["from"], + "to": f["to"], + "tags": ["crit"], + } + for f in data["faults"] + ] + flow = _collapse([{**n, "tags": []} for n in nodes] + faults) + for row in flow: + lines.append(_task(row["name"], row["tags"], row["from"], row["to"], hours)) + # Instance sections, failed ones first, until the row budget is spent. + budget = ROW_BUDGET - len(flow) + failed = { + r.get("instance") + for r in data["results"] + if r.get("instance") and (not r["ok"] or r.get("error")) + } + lanes = sorted(data["lanes"], key=lambda ln: _lane_rank(ln, failed)) + left_out: list[dict[str, Any]] = [] + for ln in lanes: + if left_out or len(ln["segments"]) > budget: + left_out.append(ln) + continue + budget -= len(ln["segments"]) + lines.append(f" section {_label(ln['name'])}") + for seg in ln["segments"]: + tag = TAG.get(seg["state"]) + lines.append( + _task(seg["state"], [tag] if tag else [], seg["from"], seg["to"], hours) + ) + note = "" + if left_out: + changes = sum(len(ln["segments"]) for ln in left_out) + which = ( + "The instance states" + if len(left_out) == len(lanes) + else (f"{len(left_out)} of {len(lanes)} instances") + ) + note = ( + f"{which} ({changes} state changes) would not fit the chart; " + "`report.html` in the run directory has them." + ) + return "\n".join(lines) + "\n", note + + +def _lane_rank(lane: dict[str, Any], failed: set[str]) -> int: + states = {seg["state"] for seg in lane["segments"]} + if lane["name"] in failed or states & ENDED_BADLY: + return 0 + return 1 if "Unknown" in states else 2 + + +def _bucket(points: list[list[float | None]]) -> list[list[float | None]]: + if len(points) <= MAX_POINTS: + return points + size = math.ceil(len(points) / MAX_POINTS) + out = [] + for i in range(0, len(points), size): + chunk = points[i : i + size] + values = [v for _, v in chunk if v is not None] + out.append([chunk[0][0], sum(values) / len(values) if values else None]) + return out + + +def _num(v: float) -> str: + """`v` to four significant digits in plain decimal. Mermaid's xychart + parser rejects exponent form (`9e-05`, `1.235e+06`), which `g` and + `repr` print for small and large values.""" + v = float(v) + if v == 0: + return "0" + return format(Decimal(f"{v:.4g}"), "f") + + +def xychart(chart: dict[str, Any]) -> tuple[str, str]: + """A Mermaid xychart of one report chart and a note under it ('' when + nothing to say). Scrapes without an answer are drawn at 0, since the + chart has no gaps; leading ones (a counter has no rate at its first + scrape) are dropped instead.""" + points = list(chart["points"]) + while points and points[0][1] is None: + points = points[1:] + if not points: + return "", "no scrapes with a value" + scrapes = len(points) + points = _bucket(points) + values = [0.0 if v is None else v for _, v in points] + low, high = min(0.0, min(values)), max(values) + high = high + (high - low) * 0.1 if high > low else low + 1 + x0, x1 = int(points[0][0]), int(points[-1][0]) + text = "\n".join( + [ + CHART_INIT, + "xychart-beta", + f' title "{chart["title"].replace(chr(34), " ")}"', + f' x-axis "seconds since the run started" {x0} --> {max(x1, x0 + 1)}', + f' y-axis "{chart["subtitle"].replace(chr(34), " ")}" ' + f"{_num(low)} --> {_num(high)}", + f" line [{', '.join(_num(v) for v in values)}]", + ] + ) + notes = [] + if any(v is None for _, v in points): + notes.append("scrapes the endpoint did not answer are drawn at 0") + if len(points) < scrapes: + notes.append(f"{scrapes} scrapes averaged into {len(points)} points") + return text + "\n", "; ".join(notes) + + +def _metric_checks(run_dir: Path, families: list[str]) -> dict[str, list[str]]: + """The names of the experiment's `metric` checks by declared family.""" + path = run_dir / "experiment.yaml" + raw = yaml.safe_load(path.read_text()) if path.is_file() else None + checks = (raw or {}).get("checks") or [] + out: dict[str, list[str]] = {f: [] for f in families} + for check in checks: + if not isinstance(check, dict) or check.get("kind") != "metric": + continue + metric = str(check.get("metric", "")) + for family in families: + if metric == family or ( + metric.startswith(family + "_") + and metric[len(family) :] in ("_count", "_sum", "_bucket") + ): + out[family].append(str(check.get("name"))) + return out + + +def _cell(text: str) -> str: + return " ".join(str(text).split()).replace("|", "\\|") + + +def _status(result: dict[str, Any]) -> str: + if result.get("error"): + return "ERROR" + if result["ok"]: + return "ok" + return "FAILED" + + +def metrics_table(run_dir: Path, data: dict[str, Any]) -> str: + families = [m["family"] for m in data["metrics"]] + checks = _metric_checks(run_dir, families) + results = {r["check"]: r for r in data["results"] if r["scope"] == "experiment"} + lines = ["| Metric | Why it is here | Checks |", "|---|---|---|"] + for m in data["metrics"]: + cells = [] + for name in checks[m["family"]]: + r = results.get(name) + if r is None: + cells.append(f"`{name}`: not run") + continue + # The output starts with the value and the bound; the series + # count after the semicolon is for the report. + summary = r["output"].split(";")[0] + cells.append(f"`{name}` {_status(r)}: {_cell(summary)}") + lines.append( + f"| `{m['family']}` | {_cell(m['description'])} | " + f"{'
'.join(cells) or 'none'} |" + ) + return "\n".join(lines) + "\n" + + +def _failures(results: list[dict[str, Any]]) -> list[str]: + out = [] + for r in results: + if r["ok"] and not r.get("error"): + continue + where = f" on {r['instance']}" if r.get("instance") else "" + verb = "could not decide" if r.get("error") else "failed" + known = f", known {r['issue']}" if r.get("issue") else "" + output = str(r.get("output", "")).strip().splitlines() + detail = f": {_cell(output[0])}" if output else "" + out.append(f"- `{r['check']}`{where} {verb}{known}{detail}") + return out + + +def details_table(data: dict[str, Any]) -> str: + """The report's run details as a two-column table. Digests and image + refs are left out; the report has them.""" + lines = ["| | |", "|---|---|"] + for row in data["details"]: + if row["more"]: + continue + value = f"`{row['value']}`" if row["code"] else row["value"] + note = f", {row['note']}" if row["note"] else "" + lines.append(f"| {_cell(row['key'])} | {_cell(value + note)} |") + return "\n".join(lines) + "\n" + + +def share_markdown(run_dir: Path) -> str: + data = build(run_dir) + exp = data["experiment"] + heading = " · ".join( + str(x) for x in (exp["id"], data["run_id"], data["verdict"]) if x + ) + results = [r for r in data["results"] if r["scope"] != "preflight"] + passed = sum(1 for r in results if r["ok"] and not r.get("error")) + failed = sum(1 for r in results if not r["ok"] and not r.get("error")) + errors = sum(1 for r in results if r.get("error")) + known = sum(1 for r in results if not r["ok"] and r.get("issue")) + counts = [f"{passed} passed"] + if failed: + counts.append(f"{failed} failed" + (f" ({known} known)" if known else "")) + if errors: + counts.append(f"{errors} could not decide") + parts = [f"## {heading}\n"] + if exp["title"]: + parts.append(f"**{_cell(exp['title'])}**\n") + if exp["hypothesis"]: + parts.append(f"Hypothesis: {_cell(exp['hypothesis'])}\n") + parts.append(details_table(data)) + parts.append(f"**Checks:** {', '.join(counts)}.\n") + failures = _failures(results) + if failures: + parts.append("\n".join(failures) + "\n") + for error in data["errors"]: + parts.append(f"Harness error: {_cell(error)}\n") + chart, note = gantt(data) + parts.append("### Flow\n") + parts.append(f"```mermaid\n{chart}```\n") + if note: + parts.append(note + "\n") + if data["metrics"]: + parts.append("### Metrics\n") + parts.append(metrics_table(run_dir, data)) + descriptions = {m["family"]: m["description"] for m in data["metrics"]} + blocks = [] + for c in data["charts"]: + if not c["declared"]: + continue + text, chart_note = xychart(c) + block = [f"#### {_cell(c['title'])}\n"] + if text: + block.append(f"```mermaid\n{text}```\n") + caption = [descriptions.get(c["family"], "")] + if chart_note: + caption.append(chart_note[0].upper() + chart_note[1:] + ".") + if any(caption): + block.append(" ".join(_cell(c) for c in caption if c) + "\n") + blocks.append("\n".join(block)) + parts.extend(blocks[:CHARTS_SHOWN]) + if len(blocks) > CHARTS_SHOWN: + rest = len(blocks) - CHARTS_SHOWN + parts.append( + f"
\n{rest} more chart{'s' if rest > 1 else ''}" + "\n\n" + "\n".join(blocks[CHARTS_SHOWN:]) + "\n
\n" + ) + return "\n".join(parts) + + +def write_share(run_dir: Path, out: Path | None = None) -> Path: + path = out or run_dir / "share.md" + path.write_text(share_markdown(run_dir)) + return path + + +def render_png(markdown: Path) -> Path: + """`markdown` with every Mermaid block replaced by a PNG, written next + to the images under `/`, through mermaid-cli (`mmdc`).""" + out_dir = markdown.with_suffix("") + out = out_dir / markdown.name + command = [ + "mmdc", + "-i", + str(markdown), + "-o", + str(out), + "-e", + "png", + "-s", + "2", + "-b", + "white", + ] + if shutil.which("mmdc") is None: + raise ShareError( + "mermaid-cli is not on PATH (npm install -g @mermaid-js/mermaid-cli); " + "with it, run: " + " ".join(command) + ) + out_dir.mkdir(parents=True, exist_ok=True) + done = subprocess.run(command, capture_output=True, text=True, check=False) + if done.returncode != 0: + raise ShareError(f"mmdc failed ({done.returncode}): {done.stderr.strip()}") + return out diff --git a/e2e/chaos/src/chaos/signals.py b/e2e/chaos/src/chaos/signals.py new file mode 100644 index 0000000000..d323533c34 --- /dev/null +++ b/e2e/chaos/src/chaos/signals.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Signal events: the arrival times of watch lines that match an adapter signal. + +Times are `time.monotonic()` at the moment the harness read the line, not the +time the system under test produced it. +""" + +from __future__ import annotations + +import asyncio +import time +from collections import defaultdict + + +class Signals: + def __init__(self) -> None: + self.events: dict[str, list[float]] = defaultdict(list) + self._changed = asyncio.Event() + + def add(self, name: str, at: float | None = None) -> None: + self.events[name].append(time.monotonic() if at is None else at) + self._changed.set() + self._changed = asyncio.Event() + + async def next( + self, name: str, after: float | None, not_before: float, deadline: float + ) -> float: + """Return the first event of `name` newer than `after` and not before + `not_before`, waiting for one until `deadline`. Raises TimeoutError.""" + while True: + for at in self.events[name]: + if (after is None or at > after) and at >= not_before: + return at + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError + changed = self._changed + try: + await asyncio.wait_for(changed.wait(), remaining) + except TimeoutError: + continue diff --git a/e2e/chaos/src/chaos/timeline.py b/e2e/chaos/src/chaos/timeline.py new file mode 100644 index 0000000000..45218d47ac --- /dev/null +++ b/e2e/chaos/src/chaos/timeline.py @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The run timeline: one JSON object per line in `timeline.jsonl`.""" + +from __future__ import annotations + +import json +import logging +import time +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +log = logging.getLogger("chaos") + + +def utc_now() -> str: + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +class Timeline: + def __init__(self, path: Path) -> None: + self.path = path + self.start = time.monotonic() + self._file = path.open("a") + + def emit(self, event: str, **fields: Any) -> None: + record = { + "t": round(time.monotonic() - self.start, 3), + "ts": utc_now(), + "event": event, + **fields, + } + self._file.write(json.dumps(record, sort_keys=False) + "\n") + self._file.flush() + log.info("%s %s", event, json.dumps(fields) if fields else "") + + def close(self) -> None: + self._file.close() + + +def read_timeline(path: Path) -> list[dict[str, Any]]: + if not path.is_file(): + return [] + return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] diff --git a/e2e/chaos/tests/conftest.py b/e2e/chaos/tests/conftest.py new file mode 100644 index 0000000000..cef70b5de4 --- /dev/null +++ b/e2e/chaos/tests/conftest.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +PACKAGE_DIR = Path(__file__).resolve().parents[1] +FIXTURES = Path(__file__).resolve().parent / "fixtures" +EXPERIMENTS = PACKAGE_DIR / "experiments" +ADAPTERS = PACKAGE_DIR / "adapters" + + +def minimal_adapter(**extra) -> dict: + """A Docker adapter with no hooks that are ever run.""" + adapter = { + "apiVersion": "chaos/v1alpha1", + "id": "openshell-docker", + "backend": "docker", + "limits": {"env-value-bytes": 8192, "run-script-bytes": 28672}, + "describe": 'echo \'{"openshell": "test"}\'', + "instance": { + "output-dir": "/out", + "create": "true", + "status": "true", + "collect": "true", + "delete": "true", + }, + "targets": { + "gateway": {"type": "process", "pid-file": "/pid", "start": "true"}, + "roles": { + "sandbox": {"type": "container", "labels": {"role": "sandbox"}}, + "supervisor": {"type": "container", "labels": {"role": "supervisor"}}, + }, + }, + } + adapter.update(extra) + return adapter + + +@pytest.fixture +def adapters_dir(tmp_path: Path): + """Write an adapter into a temporary adapters dir: `adapters_dir(**extra)`.""" + directory = tmp_path / "adapters" + directory.mkdir() + + def write(**extra) -> Path: + path = directory / "openshell-docker.yaml" + path.write_text(yaml.safe_dump(minimal_adapter(**extra))) + return directory + + return write diff --git a/e2e/chaos/tests/fixtures/invalid-api-version.yaml b/e2e/chaos/tests/fixtures/invalid-api-version.yaml new file mode 100644 index 0000000000..c104041f58 --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-api-version.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v2 +id: invalid-api-version +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: w diff --git a/e2e/chaos/tests/fixtures/invalid-chaos-env.yaml b/e2e/chaos/tests/fixtures/invalid-chaos-env.yaml new file mode 100644 index 0000000000..8b64d40807 --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-chaos-env.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-chaos-env +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + env: {CHAOS_X: "1"} + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: w diff --git a/e2e/chaos/tests/fixtures/invalid-data-name.yaml b/e2e/chaos/tests/fixtures/invalid-data-name.yaml new file mode 100644 index 0000000000..ee2adafb8e --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-data-name.yaml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-data-name +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +data: + p: {version: 1} + Bad_Name: text +workloads: + - name: w + description: d + kind: shell + instances: 2 + timeout: 1m + options: {policy: p} + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +operations: + - {name: g, description: d, uses: openshell-policy-set, options: {policy: p}, scope: global} +flow: {serial: [w, g]} diff --git a/e2e/chaos/tests/fixtures/invalid-expectation-affected-experiment-check.yaml b/e2e/chaos/tests/fixtures/invalid-expectation-affected-experiment-check.yaml new file mode 100644 index 0000000000..bdcf4a3b7b --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-expectation-affected-experiment-check.yaml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-expectation-affected-experiment-check +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +expectations: + - {issue: NVIDIA/OpenShell#1, checks: [experiment/e], instances: affected} +workloads: + - name: w + description: d + kind: shell + instances: 2 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +checks: + - {name: e, description: d, kind: shell, run: "true"} +operations: + - {name: a, description: d, uses: openshell-sandbox-restart, target: {workload: w}, select: one} +flow: {parallel: [w, {serial: [{wait: {workload: w, condition: started}}, a]}]} diff --git a/e2e/chaos/tests/fixtures/invalid-expectation-affected-untargeted.yaml b/e2e/chaos/tests/fixtures/invalid-expectation-affected-untargeted.yaml new file mode 100644 index 0000000000..b9cc4ec834 --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-expectation-affected-untargeted.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-expectation-affected-untargeted +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +expectations: + - {issue: NVIDIA/OpenShell#1, checks: [w/ok], instances: affected} +workloads: + - name: w + description: d + kind: shell + instances: 2 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: w diff --git a/e2e/chaos/tests/fixtures/invalid-expectation-issue.yaml b/e2e/chaos/tests/fixtures/invalid-expectation-issue.yaml new file mode 100644 index 0000000000..3ec53564cc --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-expectation-issue.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-expectation-issue +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +expectations: + - issue: "4910" +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: w diff --git a/e2e/chaos/tests/fixtures/invalid-expectation-shared-check.yaml b/e2e/chaos/tests/fixtures/invalid-expectation-shared-check.yaml new file mode 100644 index 0000000000..7419a470b9 --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-expectation-shared-check.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-expectation-shared-check +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +expectations: + - issue: NVIDIA/OpenShell#1 + checks: [w/ok] + - issue: NVIDIA/OpenShell#2 + checks: [w/ok] +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: w diff --git a/e2e/chaos/tests/fixtures/invalid-expectation-unknown-check.yaml b/e2e/chaos/tests/fixtures/invalid-expectation-unknown-check.yaml new file mode 100644 index 0000000000..6366a73bbd --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-expectation-unknown-check.yaml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-expectation-unknown-check +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +expectations: + - issue: NVIDIA/OpenShell#1 + checks: [w/exited-ok] +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: w diff --git a/e2e/chaos/tests/fixtures/invalid-flow-missing-name.yaml b/e2e/chaos/tests/fixtures/invalid-flow-missing-name.yaml new file mode 100644 index 0000000000..14ea4aa509 --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-flow-missing-name.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-flow-missing-name +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +operations: + - {name: f, description: d, uses: restart, target: gateway, select: one} +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: w diff --git a/e2e/chaos/tests/fixtures/invalid-flow-repeated-name.yaml b/e2e/chaos/tests/fixtures/invalid-flow-repeated-name.yaml new file mode 100644 index 0000000000..ae9af952cd --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-flow-repeated-name.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-flow-repeated-name +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: {serial: [w, w]} diff --git a/e2e/chaos/tests/fixtures/invalid-metric-check-no-bound.yaml b/e2e/chaos/tests/fixtures/invalid-metric-check-no-bound.yaml new file mode 100644 index 0000000000..d94b19df4b --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-metric-check-no-bound.yaml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-metric-check-no-bound +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: w +metrics: [up_gauge] +checks: + - {name: m, description: d, kind: metric, metric: up_gauge} diff --git a/e2e/chaos/tests/fixtures/invalid-metric-check-undeclared.yaml b/e2e/chaos/tests/fixtures/invalid-metric-check-undeclared.yaml new file mode 100644 index 0000000000..98e773915b --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-metric-check-undeclared.yaml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-metric-check-undeclared +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: w +metrics: [up_gauge] +checks: + - {name: m, description: d, kind: metric, metric: other_total, value: increase, at-most: 0} diff --git a/e2e/chaos/tests/fixtures/invalid-metric-entry-both.yaml b/e2e/chaos/tests/fixtures/invalid-metric-entry-both.yaml new file mode 100644 index 0000000000..a3d56e224f --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-metric-entry-both.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-metric-entry-both +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: w +metrics: + - {family: up_gauge, title: Up, expr: "sum(up_gauge)"} diff --git a/e2e/chaos/tests/fixtures/invalid-metric-name.yaml b/e2e/chaos/tests/fixtures/invalid-metric-name.yaml new file mode 100644 index 0000000000..96b007d4d5 --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-metric-name.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-metric-name +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: w +metrics: ["rate(x[1m])"] diff --git a/e2e/chaos/tests/fixtures/invalid-missing-description.yaml b/e2e/chaos/tests/fixtures/invalid-missing-description.yaml new file mode 100644 index 0000000000..f64341f902 --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-missing-description.yaml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-missing-description +version: 1 +title: t +hypothesis: h +default-adapter: openshell-docker +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: w diff --git a/e2e/chaos/tests/fixtures/invalid-no-api-version.yaml b/e2e/chaos/tests/fixtures/invalid-no-api-version.yaml new file mode 100644 index 0000000000..0e509d47fe --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-no-api-version.yaml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +id: invalid-no-api-version +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: w diff --git a/e2e/chaos/tests/fixtures/invalid-no-checks.yaml b/e2e/chaos/tests/fixtures/invalid-no-checks.yaml new file mode 100644 index 0000000000..943622d79e --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-no-checks.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-no-checks +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [] +flow: w diff --git a/e2e/chaos/tests/fixtures/invalid-operation-global-select.yaml b/e2e/chaos/tests/fixtures/invalid-operation-global-select.yaml new file mode 100644 index 0000000000..77cfbd07bb --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-operation-global-select.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-operation-global-select +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +data: + p: {version: 1} +workloads: + - name: w + description: d + kind: shell + instances: 2 + timeout: 1m + options: {policy: p} + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +operations: + - {name: g, description: d, uses: openshell-policy-set, options: {policy: p}, scope: global, select: all} +flow: {serial: [w, g]} diff --git a/e2e/chaos/tests/fixtures/invalid-operation-name-clash.yaml b/e2e/chaos/tests/fixtures/invalid-operation-name-clash.yaml new file mode 100644 index 0000000000..bcd7896400 --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-operation-name-clash.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-operation-name-clash +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +data: + p: {version: 1} +workloads: + - name: w + description: d + kind: shell + instances: 2 + timeout: 1m + options: {policy: p} + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +operations: + - {name: w, description: d, uses: openshell-policy-set, options: {policy: p}, scope: global} +flow: {serial: [w]} diff --git a/e2e/chaos/tests/fixtures/invalid-operation-no-targeting.yaml b/e2e/chaos/tests/fixtures/invalid-operation-no-targeting.yaml new file mode 100644 index 0000000000..6a5bf90199 --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-operation-no-targeting.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-operation-no-targeting +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +data: + p: {version: 1} +workloads: + - name: w + description: d + kind: shell + instances: 2 + timeout: 1m + options: {policy: p} + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +operations: + - {name: g, description: d, uses: openshell-policy-set, options: {policy: p}} +flow: {serial: [w, g]} diff --git a/e2e/chaos/tests/fixtures/invalid-operation-no-wait.yaml b/e2e/chaos/tests/fixtures/invalid-operation-no-wait.yaml new file mode 100644 index 0000000000..39f0d8331f --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-operation-no-wait.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-operation-no-wait +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +data: + p: {version: 1} +workloads: + - name: w + description: d + kind: shell + instances: 2 + timeout: 1m + options: {policy: p} + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +operations: + - {name: a, description: d, uses: openshell-policy-set, options: {policy: p}, target: {workload: w}, select: all} +flow: {parallel: [w, a]} diff --git a/e2e/chaos/tests/fixtures/invalid-outage-no-duration.yaml b/e2e/chaos/tests/fixtures/invalid-outage-no-duration.yaml new file mode 100644 index 0000000000..6d1b7f077b --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-outage-no-duration.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-outage-no-duration +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +operations: + - {name: f, description: d, uses: outage, target: gateway, select: one} +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: {serial: [w, f]} diff --git a/e2e/chaos/tests/fixtures/invalid-repeat-shape.yaml b/e2e/chaos/tests/fixtures/invalid-repeat-shape.yaml new file mode 100644 index 0000000000..b9d72177c8 --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-repeat-shape.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-repeat-shape +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: {serial: [w, {repeat: {times: 2, for: 1m, serial: [{delay: 1s}]}}]} diff --git a/e2e/chaos/tests/fixtures/invalid-repeat-workload.yaml b/e2e/chaos/tests/fixtures/invalid-repeat-workload.yaml new file mode 100644 index 0000000000..4dfe0ee169 --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-repeat-workload.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-repeat-workload +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: {repeat: {times: 2, serial: [w]}} diff --git a/e2e/chaos/tests/fixtures/invalid-restart-duration.yaml b/e2e/chaos/tests/fixtures/invalid-restart-duration.yaml new file mode 100644 index 0000000000..63dbc07ba1 --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-restart-duration.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-restart-duration +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +operations: + - {name: f, description: d, uses: restart, target: gateway, select: one, duration: 10s} +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: {serial: [w, f]} diff --git a/e2e/chaos/tests/fixtures/invalid-role-fault-no-wait.yaml b/e2e/chaos/tests/fixtures/invalid-role-fault-no-wait.yaml new file mode 100644 index 0000000000..6e75fc8e22 --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-role-fault-no-wait.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-role-fault-no-wait +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +operations: + - {name: f, description: d, uses: kill, target: {workload: w, role: supervisor}, select: one} +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: {parallel: [w, f]} diff --git a/e2e/chaos/tests/fixtures/invalid-role-fault-wait-after.yaml b/e2e/chaos/tests/fixtures/invalid-role-fault-wait-after.yaml new file mode 100644 index 0000000000..d081400538 --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-role-fault-wait-after.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-role-fault-wait-after +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +operations: + - {name: f, description: d, uses: kill, target: {workload: w, role: supervisor}, select: one} +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: {parallel: [w, {serial: [f, {wait: {workload: w, condition: started}}]}]} diff --git a/e2e/chaos/tests/fixtures/invalid-unknown-key.yaml b/e2e/chaos/tests/fixtures/invalid-unknown-key.yaml new file mode 100644 index 0000000000..d67ae8903c --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-unknown-key.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-unknown-key +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +colour: red +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: w diff --git a/e2e/chaos/tests/fixtures/invalid-wait-outside-parallel.yaml b/e2e/chaos/tests/fixtures/invalid-wait-outside-parallel.yaml new file mode 100644 index 0000000000..12a1732606 --- /dev/null +++ b/e2e/chaos/tests/fixtures/invalid-wait-outside-parallel.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: invalid-wait-outside-parallel +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: {serial: [w, {wait: {workload: w, condition: started}}]} diff --git a/e2e/chaos/tests/fixtures/valid-expectations.yaml b/e2e/chaos/tests/fixtures/valid-expectations.yaml new file mode 100644 index 0000000000..e3ff23fce4 --- /dev/null +++ b/e2e/chaos/tests/fixtures/valid-expectations.yaml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: valid-expectations +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +expectations: + - {issue: NVIDIA/OpenShell#1, checks: [w/ok], instances: affected, kind: flaky} + - {issue: NVIDIA/OpenShell#2, checks: [experiment/e]} +workloads: + - name: w + description: d + kind: shell + instances: 2 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +checks: + - {name: e, description: d, kind: shell, run: "true"} +operations: + - {name: a, description: d, uses: openshell-sandbox-restart, target: {workload: w}, select: one} +flow: {parallel: [w, {serial: [{wait: {workload: w, condition: started}}, a]}]} diff --git a/e2e/chaos/tests/fixtures/valid-freeze.yaml b/e2e/chaos/tests/fixtures/valid-freeze.yaml new file mode 100644 index 0000000000..8327ae24f5 --- /dev/null +++ b/e2e/chaos/tests/fixtures/valid-freeze.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: valid-freeze +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +operations: + - {name: f, description: d, uses: freeze, target: {workload: w, role: supervisor}, select: all, duration: 5s} +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: {parallel: [w, {serial: [{wait: {workload: w, condition: started}}, f]}]} diff --git a/e2e/chaos/tests/fixtures/valid-minimal.yaml b/e2e/chaos/tests/fixtures/valid-minimal.yaml new file mode 100644 index 0000000000..c4b97b5762 --- /dev/null +++ b/e2e/chaos/tests/fixtures/valid-minimal.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: valid-minimal +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: w diff --git a/e2e/chaos/tests/fixtures/valid-operations.yaml b/e2e/chaos/tests/fixtures/valid-operations.yaml new file mode 100644 index 0000000000..928403c1b0 --- /dev/null +++ b/e2e/chaos/tests/fixtures/valid-operations.yaml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: valid-operations +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +data: + p: {version: 1} +workloads: + - name: w + description: d + kind: shell + instances: 2 + timeout: 1m + options: {policy: p} + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +operations: + - {name: a, description: d, uses: openshell-policy-set, options: {policy: p}, target: {workload: w}, select: all} + - {name: g, description: d, uses: openshell-policy-set, options: {policy: p}, scope: global} +flow: {parallel: [w, {serial: [{wait: {workload: w, condition: started}}, a, g]}]} diff --git a/e2e/chaos/tests/fixtures/valid-role-fault.yaml b/e2e/chaos/tests/fixtures/valid-role-fault.yaml new file mode 100644 index 0000000000..620113e1b7 --- /dev/null +++ b/e2e/chaos/tests/fixtures/valid-role-fault.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: chaos/v1alpha1 +id: valid-role-fault +version: 1 +title: t +hypothesis: h +description: d +default-adapter: openshell-docker +operations: + - {name: f, description: d, uses: kill, target: {workload: w, role: supervisor}, select: one} +workloads: + - name: w + description: d + kind: shell + instances: 1 + timeout: 1m + run: "true" + checks: [{name: ok, description: d, kind: exit-code, equals: 0}] +flow: {parallel: [w, {serial: [{wait: {workload: w, condition: started}}, f]}]} diff --git a/e2e/chaos/tests/test_adapter.py b/e2e/chaos/tests/test_adapter.py new file mode 100644 index 0000000000..c0961b57d4 --- /dev/null +++ b/e2e/chaos/tests/test_adapter.py @@ -0,0 +1,195 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import asyncio +import re +import shutil +import subprocess + +import pytest +from conftest import ADAPTERS + +from chaos.adapter import expand, hook_script, load_adapter, missing_env, run_hook +from chaos.model import ExperimentError + + +@pytest.fixture +def docker_adapter(): + return load_adapter(ADAPTERS, "openshell-docker") + + +def test_openshell_docker_adapter_loads(docker_adapter): + assert docker_adapter.id == "openshell-docker" + assert docker_adapter.backend == "docker" + assert docker_adapter.target("gateway").type == "process" + assert set(docker_adapter.targets.roles) == {"sandbox", "supervisor"} + assert {c.name for c in docker_adapter.checks} == { + "no-leaked-containers", + "nothing-paused", + "gateway-healthy", + } + assert [c.name for c in docker_adapter.preflight] == [ + "docker-reachable", + "pull-otel-collector-image", + "disk-space", + "images-present", + "cli-runs", + "jq-installed", + "gateway-up", + ] + assert [(s.name, s.watch) for s in docker_adapter.signals] == [ + ("reconcile-sweep", "otel") + ] + (sweep,) = docker_adapter.signals + assert re.search(sweep.match, " Name : reconcile.sandboxes") + assert not re.search(sweep.match, " Name : reconcile.sandboxes.one") + assert len(docker_adapter.sha256) == 64 + + +def _hooks(adapter) -> dict[str, str]: + hooks = { + f"instance.{k}": getattr(adapter.instance, k) + for k in ("create", "status", "collect", "delete") + } + hooks["setup"] = adapter.setup + hooks["collect"] = adapter.collect + for name, target in adapter.targets.named.items(): + hooks[f"targets.{name}.start"] = target.start + for name, spec in adapter.operations.items(): + hooks[f"operations.{name}"] = spec.run + if spec.reverts: + hooks[f"operations.{name}.revert"] = spec.revert + return {k: v for k, v in hooks.items() if v} + + +@pytest.mark.skipif(shutil.which("shellcheck") is None, reason="needs shellcheck") +@pytest.mark.parametrize( + ("adapter_id", "count"), [("openshell-docker", 9), ("openshell-kubernetes", 10)] +) +def test_adapter_hooks_pass_shellcheck(adapter_id, count): + adapter = load_adapter(ADAPTERS, adapter_id) + hooks = _hooks(adapter) + assert len(hooks) == count + for name, script in hooks.items(): + result = subprocess.run( + # A hook uses only some of the prelude's helpers. + ["shellcheck", "--shell=bash", "--exclude=SC2317,SC2329", "-"], + input=hook_script(adapter, script), + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, f"{name}:\n{result.stdout}" + + +def test_hook_runs_after_the_prelude_with_its_name(adapters_dir): + adapter = load_adapter( + adapters_dir(prelude='greet() { echo "hi from $0"; }\n'), + "openshell-docker", + ) + result = asyncio.run(run_hook(adapter, "instance.create", "greet", {})) + assert result.ok, result.stderr + assert result.stdout == "hi from instance.create\n" + + +def test_options_defaults_and_overrides(docker_adapter): + assert docker_adapter.resolve_options({}, "w") == { + "image": "", + "policy": "", + "restart-policy": "never", + } + resolved = docker_adapter.resolve_options({"restart-policy": "on-failure"}, "w") + assert resolved["restart-policy"] == "on-failure" + + +def test_option_not_in_enum(docker_adapter): + with pytest.raises( + ExperimentError, match=r"w: option restart-policy='sometimes' not in" + ): + docker_adapter.resolve_options({"restart-policy": "sometimes"}, "w") + + +def test_unknown_option(docker_adapter): + with pytest.raises(ExperimentError, match="w: unknown option 'gpu'"): + docker_adapter.resolve_options({"gpu": "1"}, "w") + + +def test_unknown_target_and_role(docker_adapter): + with pytest.raises(ExperimentError, match="no target 'db'"): + docker_adapter.target("db") + with pytest.raises(ExperimentError, match="no role 'proxy'"): + docker_adapter.role("proxy") + + +def test_missing_env(docker_adapter): + env = dict.fromkeys(docker_adapter.requires_env, "x") + assert missing_env(docker_adapter, env) == [] + env["OPENSHELL_BIN"] = "" + del env["OPENSHELL_GATEWAY"] + assert missing_env(docker_adapter, env) == ["OPENSHELL_BIN", "OPENSHELL_GATEWAY"] + + +def test_missing_adapter(tmp_path): + with pytest.raises(ExperimentError, match="no adapter openshell-kubernetes"): + load_adapter(tmp_path, "openshell-kubernetes") + + +def test_adapter_id_must_match_file_name(adapters_dir): + directory = adapters_dir() + (directory / "openshell-docker.yaml").rename(directory / "other.yaml") + with pytest.raises(ExperimentError, match="does not match the file name"): + load_adapter(directory, "other") + + +def test_backend_must_be_supported(adapters_dir): + with pytest.raises( + ExperimentError, match=r"backend: Input should be 'docker' or 'kubernetes'" + ): + load_adapter(adapters_dir(backend="podman"), "openshell-docker") + + +def test_process_target_needs_pid_file_and_start(adapters_dir): + directory = adapters_dir(targets={"gateway": {"type": "process"}}) + with pytest.raises( + ExperimentError, match="a process target needs pid-file and start" + ): + load_adapter(directory, "openshell-docker") + + +def test_unknown_adapter_key(adapters_dir): + with pytest.raises(ExperimentError, match="colour: Extra inputs are not permitted"): + load_adapter(adapters_dir(colour="red"), "openshell-docker") + + +@pytest.mark.parametrize( + ("operation", "error"), + [ + ({"run": "true"}, r"operations\.op\.revert: Field required"), + ({"run": "true", "revert": "none"}, "revert: none needs a reason"), + ( + {"run": "true", "revert": "true", "reason": "r"}, + "reason goes with revert: none", + ), + ], +) +def test_operation_reverts_or_says_why_not(adapters_dir, operation, error): + directory = adapters_dir(operations={"op": operation}) + with pytest.raises(ExperimentError, match=error): + load_adapter(directory, "openshell-docker") + + +def test_adapter_check_is_shell_only(adapters_dir): + check = {"name": "e", "description": "d", "kind": "exit-code", "equals": 0} + with pytest.raises( + ExperimentError, match=r"checks\.0\.kind: Input should be 'shell'" + ): + load_adapter(adapters_dir(checks=[check]), "openshell-docker") + + +def test_expand(): + assert expand("a-${X}-${Y}", {"X": "1", "Y": "2"}) == "a-1-2" + assert expand("$X", {}) == "$X" + with pytest.raises(ExperimentError, match=r"\$\{MISSING\} is not set"): + expand("${MISSING}", {}) diff --git a/e2e/chaos/tests/test_cli.py b/e2e/chaos/tests/test_cli.py new file mode 100644 index 0000000000..9a0ef0e7c5 --- /dev/null +++ b/e2e/chaos/tests/test_cli.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""`--adapter` picks the adapter instead of the experiment's `default-adapter`.""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import yaml +from conftest import ADAPTERS, PACKAGE_DIR, minimal_adapter + +from chaos.cli import main + +EXPERIMENTS = PACKAGE_DIR / "experiments" + + +def adapters(tmp_path: Path) -> Path: + """The real adapters, and `bare-docker`: a Docker adapter with no + operations, options or signals.""" + directory = tmp_path / "adapters" + shutil.copytree(ADAPTERS, directory) + bare = minimal_adapter(id="bare-docker") + (directory / "bare-docker.yaml").write_text(yaml.safe_dump(bare)) + return directory + + +def test_validate_uses_default_adapter(tmp_path, capsys): + directory = adapters(tmp_path) + path = EXPERIMENTS / "hello-world.yaml" + assert main(["--adapters-dir", str(directory), "validate", str(path)]) == 0 + assert "ok on openshell-docker:" in capsys.readouterr().out + + +def test_validate_with_another_adapter(tmp_path, capsys): + directory = adapters(tmp_path) + path = EXPERIMENTS / "hello-world.yaml" + argv = ["--adapters-dir", str(directory), "validate", "--adapter", "bare-docker"] + assert main([*argv, str(path)]) == 0 + assert "ok on bare-docker:" in capsys.readouterr().out + + +def test_validate_names_what_the_adapter_lacks(tmp_path, capsys): + directory = adapters(tmp_path) + path = EXPERIMENTS / "policy-tighten-survives-gateway-restart.yaml" + argv = ["--adapters-dir", str(directory), "validate", "--adapter", "bare-docker"] + assert main([*argv, str(path)]) == 1 + err = capsys.readouterr().err + assert err.startswith("invalid on bare-docker: ") + assert "unknown option 'policy' for bare-docker" in err + + +def test_unknown_adapter(tmp_path, capsys): + directory = adapters(tmp_path) + path = EXPERIMENTS / "hello-world.yaml" + argv = ["--adapters-dir", str(directory), "preflight", "--adapter", "nope"] + assert main([*argv, str(path)]) == 2 + assert "no adapter nope" in capsys.readouterr().err diff --git a/e2e/chaos/tests/test_dashboard.py b/e2e/chaos/tests/test_dashboard.py new file mode 100644 index 0000000000..4a376dd34e --- /dev/null +++ b/e2e/chaos/tests/test_dashboard.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Per-experiment Grafana dashboards.""" + +from __future__ import annotations + +import json + +import pytest +from conftest import EXPERIMENTS + +from chaos.cli import main +from chaos.dashboard import UID_MAX, build_dashboard, dashboard_uid +from chaos.model import load_experiment, parse_experiment, read_yaml + + +def experiment(**extra): + raw = read_yaml(EXPERIMENTS / "supervisor-kill.yaml") + raw.update(extra) + return parse_experiment(raw, "test") + + +def by_title(dashboard): + return {p["title"]: p for p in dashboard["panels"]} + + +def test_panels_follow_the_experiment(): + exp = experiment(metrics=[], checks=[]) + dash = build_dashboard(exp) + assert dash["uid"] == "exp-supervisor-kill" + panels = by_title(dash) + assert exp.hypothesis.split()[0] in panels[exp.title]["options"]["content"] + for w in exp.workloads: + (target,) = panels[f"{w.name} instances"]["targets"] + assert f'workload="{w.name}"' in target["expr"] + assert "Faults" in panels + assert "chaos_check_result" in panels["Check results"]["targets"][0]["expr"] + run, chart = dash["templating"]["list"] + assert f'experiment="{exp.id}"' in run["query"]["query"] + # No `metrics:`: every chart of the run, repeated over `$chart`. + assert panels["$chart"]["repeat"] == chart["name"] == "chart" + + +def test_listed_metrics_replace_the_chart_repeat(): + dash = build_dashboard( + experiment( + checks=[], + metrics=[ + "openshell_gateway_sessions", + {"title": "Up", "expr": 'up{chaos_run_id="$run"}'}, + ], + ) + ) + panels = by_title(dash) + assert "$chart" not in panels + variable = dash["templating"]["list"][1] + assert variable["name"] == "metric0" and variable["hide"] == 2 + assert 'family="openshell_gateway_sessions"' in variable["query"]["query"] + assert panels["$metric0"]["repeat"] == "metric0" + assert panels["Up"]["targets"][0]["expr"] == 'up{chaos_run_id="$run"}' + + +def test_layout_does_not_overlap(): + cells = set() + for p in build_dashboard(experiment())["panels"]: + g = p["gridPos"] + assert g["x"] + g["w"] <= 24 + for x in range(g["x"], g["x"] + g["w"]): + for y in range(g["y"], g["y"] + g["h"]): + assert (x, y) not in cells, p["title"] + cells.add((x, y)) + + +def test_long_ids_get_a_short_unique_uid(): + a, b = dashboard_uid("a" * 50), dashboard_uid("a" * 49 + "b") + assert len(a) <= UID_MAX and len(b) <= UID_MAX and a != b + + +@pytest.mark.parametrize( + "path", sorted(EXPERIMENTS.glob("*.yaml")), ids=lambda p: p.stem +) +def test_every_shipped_experiment_has_a_dashboard(path, tmp_path): + exp, _ = load_experiment(path) + assert main(["dashboard", str(path), "--dir", str(tmp_path)]) == 0 + dash = json.loads((tmp_path / f"{exp.id}.json").read_text()) + assert dash["uid"] == dashboard_uid(exp.id) diff --git a/e2e/chaos/tests/test_envvars.py b/e2e/chaos/tests/test_envvars.py new file mode 100644 index 0000000000..7abc4fb8ed --- /dev/null +++ b/e2e/chaos/tests/test_envvars.py @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The REFERENCE.md variable tables are the oracle for the `envvars` builders.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +from conftest import PACKAGE_DIR + +from chaos import envvars +from chaos.envvars import InstanceInfo, RunInfo + +ROW_RE = re.compile(r"^\| `(CHAOS_[A-Z_]+)` \|(.*)\|$") +COLUMNS = ["workload_run", "workload_check", "experiment_check"] + +RUN = RunInfo( + "exp", 3, "openshell-docker", "20260101T000000Z-abcd", 42, Path("/runs/exp/r") +) +INST = InstanceInfo( + "counter", + 2, + 1, + "counter-1", + Path("/runs/exp/r/workloads/counter/instances/counter-1"), + "/out", +) + + +def readme_table() -> dict[str, set[str]]: + """Map each column of the variable table to the variables marked in it.""" + table: dict[str, set[str]] = {c: set() for c in COLUMNS} + for line in (PACKAGE_DIR / "REFERENCE.md").read_text().splitlines(): + match = ROW_RE.match(line) + if not match: + continue + cells = [c.strip() for c in match.group(2).split("|")] + assert len(cells) == len(COLUMNS), line + for column, cell in zip(COLUMNS, cells, strict=True): + if cell.startswith("✓"): + table[column].add(match.group(1)) + assert all(table.values()), "REFERENCE.md variable table not found" + return table + + +def builders(exit_code: int | None) -> dict[str, dict[str, str]]: + return { + "workload_run": envvars.workload_run_env(RUN, INST), + "workload_check": envvars.workload_check_env(RUN, INST, "c", exit_code), + "experiment_check": envvars.experiment_check_env(RUN, "c"), + } + + +@pytest.mark.parametrize("column", COLUMNS) +def test_scope_matches_reference(column): + assert set(builders(0)[column]) == readme_table()[column] + + +def test_exit_code_unset_when_none_recorded(): + env = builders(None)["workload_check"] + assert "CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_EXIT_CODE" not in env + assert set(env) == readme_table()["workload_check"] - { + "CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_EXIT_CODE" + } + + +def test_output_dir_is_the_only_value_that_differs(): + run, check = builders(0)["workload_run"], builders(0)["workload_check"] + differ = {k for k in run if run[k] != check[k]} + assert differ == {"CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR"} + assert run["CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR"] == "/out" + assert check["CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR"] == str( + INST.dir / "output" + ) + + +def test_values(): + env = builders(3)["workload_check"] + assert env["CHAOS_EXPERIMENT_VERSION"] == "3" + assert env["CHAOS_EXPERIMENT_RUN_SEED"] == "42" + assert env["CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_COUNT"] == "2" + assert env["CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_INDEX"] == "1" + assert env["CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_EXIT_CODE"] == "3" + assert env["CHAOS_EXPERIMENT_RUN_DIR"] == "/runs/exp/r" + + +def test_instance_hook_env(): + env = envvars.instance_hook_env( + RUN, + INST, + {"restart-policy": "never"}, + Path("/run.sh"), + Path("/wrap.sh"), + Path("/env"), + None, + ) + expected = readme_table()["workload_run"] | { + "CHAOS_EXPERIMENT_RUN_DIR", + "CHAOS_EXPERIMENT_DATA_DIR", + "CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_DIR", + "CHAOS_EXPERIMENT_WORKLOAD_RUN_FILE", + "CHAOS_EXPERIMENT_WORKLOAD_WRAPPER_FILE", + "CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_ENV_FILE", + "CHAOS_EXPERIMENT_WORKLOAD_OPTION_RESTART_POLICY", + } + assert set(env) == expected + with_id = envvars.instance_hook_env( + RUN, INST, {}, Path("/run.sh"), Path("/wrap.sh"), Path("/env"), "sb-1" + ) + assert with_id["CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_SYSTEM_ID"] == "sb-1" + + +def test_operation_env_matches_reference(): + reference = (PACKAGE_DIR / "REFERENCE.md").read_text() + documented = set( + re.findall(r"^- `(CHAOS_EXPERIMENT_OPERATION_[A-Z_]+)`", reference, re.M) + ) + # The runner adds the outcome file and the state dir to both. + documented -= { + "CHAOS_EXPERIMENT_OPERATION_OUTCOME_FILE", + "CHAOS_EXPERIMENT_OPERATION_STATE_DIR", + } + options = {"policy": "/run/data/p"} + targeted = envvars.operation_env(RUN, "a", "openshell-policy-set", options) + option = {"CHAOS_EXPERIMENT_OPERATION_OPTION_POLICY"} + assert ( + set(targeted) + == documented - {"CHAOS_EXPERIMENT_OPERATION_INSTANCES_FILE"} | option + ) + assert targeted["CHAOS_EXPERIMENT_OPERATION_SCOPE"] == "instances" + assert targeted["CHAOS_EXPERIMENT_OPERATION_OPTION_POLICY"] == "/run/data/p" + glob = envvars.operation_env(RUN, "a", "openshell-policy-set", options, Path("/i")) + operations = {k for k in glob if k.startswith("CHAOS_EXPERIMENT_OPERATION_")} + assert operations == documented | option + assert glob["CHAOS_EXPERIMENT_OPERATION_SCOPE"] == "global" + assert glob["CHAOS_EXPERIMENT_RUN_DIR"] == str(RUN.run_dir) diff --git a/e2e/chaos/tests/test_evaluate.py b/e2e/chaos/tests/test_evaluate.py new file mode 100644 index 0000000000..95adbb9fbf --- /dev/null +++ b/e2e/chaos/tests/test_evaluate.py @@ -0,0 +1,568 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Checks, verdicts and `chaos evaluate` against a hand-made run directory.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +from pathlib import Path + +import pytest +import yaml +from conftest import FIXTURES + +from chaos import cli +from chaos.adapter import load_adapter +from chaos.envvars import RunInfo +from chaos.evaluate import ( + Result, + format_verdict, + read_verdict, + run_checks, + selected_instances, + verdict_of, + write_verdict, +) +from chaos.model import load_experiment + +RUN_ID = "20260101T000000Z-abcd" +EXPERIMENT = { + "apiVersion": "chaos/v1alpha1", + "id": "evaluated", + "version": 1, + "title": "t", + "hypothesis": "h", + "description": "d", + "default-adapter": "openshell-docker", + "workloads": [ + { + "name": "w", + "description": "d", + "kind": "shell", + "instances": 2, + "timeout": "1m", + "env": {"WANT": "done"}, + "run": "true", + "checks": [ + { + "name": "exited-ok", + "description": "d", + "kind": "exit-code", + "equals": 0, + }, + { + "name": "wrote-done", + "description": "d", + "kind": "shell", + "run": 'grep -qx "$WANT" "$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR/result"' + ' || { echo "$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_NAME missing"; exit 1; }', + }, + ], + } + ], + "flow": "w", + "checks": [ + { + "name": "timeline-written", + "description": "d", + "kind": "shell", + "run": 'test -f "$CHAOS_EXPERIMENT_RUN_DIR/timeline.jsonl"' + ' && test "$CHAOS_EXPERIMENT_CHECK_NAME" = timeline-written', + } + ], +} + + +def instance_output(run_dir: Path, name: str) -> Path: + path = run_dir / "workloads" / "w" / "instances" / name / "output" + (path / ".chaos").mkdir(parents=True) + return path + + +@pytest.fixture +def run_dir(tmp_path, adapters_dir): + """A finished run: w-0 succeeded, w-1 exited 1 without writing its result.""" + directory = tmp_path / "runs" / "evaluated" / RUN_ID + directory.mkdir(parents=True) + (directory / "timeline.jsonl").write_text( + json.dumps({"event": "run.finished", "verdict": "FAIL"}) + "\n" + ) + (directory / "experiment.yaml").write_text(yaml.safe_dump(EXPERIMENT)) + (directory / "manifest.json").write_text( + json.dumps( + { + "schema_version": "v1alpha1", + "run_id": RUN_ID, + "experiment": {"id": "evaluated", "version": 1}, + "seed": 0, + "adapter": {"id": "openshell-docker"}, + "overrides": [], + "environment": {"system": {"git_sha": "abc"}}, + } + ) + ) + ok = instance_output(directory, "w-0") + (ok / ".chaos" / "exit_code").write_text("0\n") + (ok / "result").write_text("done\n") + bad = instance_output(directory, "w-1") + (bad / ".chaos" / "exit_code").write_text("1\n") + adapters_dir() + return directory + + +def evaluate(run_dir: Path, *extra: str) -> int: + adapters = run_dir.parents[2] / "adapters" + return cli.main(["--adapters-dir", str(adapters), "evaluate", str(run_dir), *extra]) + + +def test_run_dir_commands_refuse_another_schema_version(run_dir, capsys): + manifest = json.loads((run_dir / "manifest.json").read_text()) + manifest["schema_version"] = "v0" + (run_dir / "manifest.json").write_text(json.dumps(manifest)) + assert evaluate(run_dir) == 2 + assert cli.main(["report", str(run_dir)]) == 2 + assert cli.main(["dashboard", str(run_dir)]) == 2 + err = capsys.readouterr().err + assert ( + err.count("schema_version 'v0' is not 'v1alpha1'; this harness cannot read it") + == 3 + ) + assert not (run_dir / "evaluations").exists() + + +def test_evaluate_reports_each_check(run_dir, capsys): + assert evaluate(run_dir) == 1 + verdict = read_verdict(run_dir) + assert verdict["verdict"] == "FAIL" + results = {(r["scope"], r["instance"], r["check"]): r for r in verdict["results"]} + assert set(results) == { + ("workload", "w-0", "exited-ok"), + ("workload", "w-0", "wrote-done"), + ("workload", "w-1", "exited-ok"), + ("workload", "w-1", "wrote-done"), + ("experiment", None, "timeline-written"), + } + assert results["workload", "w-0", "exited-ok"]["ok"] + assert results["workload", "w-0", "wrote-done"]["ok"] + assert results["workload", "w-1", "exited-ok"]["output"] == "exit code 1, want 0" + assert "w-1 missing" in results["workload", "w-1", "wrote-done"]["output"] + assert results["experiment", None, "timeline-written"]["ok"] + *_, last_verdict, path = capsys.readouterr().out.strip().splitlines() + assert last_verdict == "FAIL" + assert Path(path).parent == run_dir / "evaluations" + + +def test_evaluate_is_reproducible_and_appends_index(run_dir): + (run_dir / "verdict.json").write_text(json.dumps({"verdict": "PASS"})) + index = run_dir.parents[1] / "index.jsonl" + evaluate(run_dir, "--index", str(index)) + first = read_verdict(run_dir) + evaluate(run_dir, "--index", str(index)) + assert read_verdict(run_dir)["results"] == first["results"] + assert len(list((run_dir / "evaluations").glob("*.json"))) == 2 + # An evaluation never replaces the verdict the run wrote. + assert json.loads((run_dir / "verdict.json").read_text()) == {"verdict": "PASS"} + lines = [json.loads(line) for line in index.read_text().splitlines()] + assert [line["verdict"] for line in lines] == ["FAIL", "FAIL"] + assert lines[0]["system"] == {"git_sha": "abc"} + assert lines[0]["adapter"] == "openshell-docker" + assert lines[0]["flaky_missed"] == [] + # Paths are relative to the index's directory. + assert lines[0]["path"] == f"evaluated/{RUN_ID}" + assert lines[0]["evaluation"].startswith(f"evaluated/{RUN_ID}/evaluations/") + # Each evaluation says what it read, and the printed verdict starts with it. + archived = run_dir / "experiment.yaml" + assert first["read"] == { + "experiment": { + "file": str(archived), + "sha256": hashlib.sha256(archived.read_bytes()).hexdigest(), + }, + "adapter": { + "id": "openshell-docker", + "file": str(run_dir.parents[2] / "adapters" / "openshell-docker.yaml"), + }, + } + assert format_verdict(first).startswith(f" evaluated with {archived} (sha256 ") + + +def test_evaluate_keeps_run_errors(run_dir): + (run_dir / "verdict.json").write_text(json.dumps({"errors": ["aborted by SIGINT"]})) + assert evaluate(run_dir) == 2 + verdict = read_verdict(run_dir) + assert verdict["verdict"] == "ERROR" + assert verdict["errors"] == ["aborted by SIGINT"] + + +def test_evaluate_of_unfinished_run_is_error(run_dir): + (run_dir / "timeline.jsonl").write_text(json.dumps({"event": "run.started"}) + "\n") + assert evaluate(run_dir) == 2 + assert read_verdict(run_dir)["errors"] == ["run did not finish"] + # The run wrote the same error: it is not repeated. + (run_dir / "verdict.json").write_text( + json.dumps({"errors": ["run did not finish"]}) + ) + evaluate(run_dir) + assert read_verdict(run_dir)["errors"] == ["run did not finish"] + + +def test_evaluate_with_edited_experiment(run_dir, tmp_path): + edited = json.loads(json.dumps(EXPERIMENT)) + edited["workloads"][0]["checks"] = edited["workloads"][0]["checks"][1:] + path = tmp_path / "evaluated.yaml" + path.write_text(yaml.safe_dump(edited)) + ( + run_dir / "workloads" / "w" / "instances" / "w-1" / "output" / "result" + ).write_text("done\n") + assert evaluate(run_dir, "--experiment", str(path)) == 0 + + +CHECKS = ["w/exited-ok", "w/wrote-done"] + + +def test_evaluate_known_failures_are_xfail(run_dir, tmp_path): + path = _edited(tmp_path, expectations=[{"issue": "o/r#1", "checks": CHECKS}]) + assert evaluate(run_dir, "--experiment", path) == 0 + verdict = read_verdict(run_dir) + assert verdict["verdict"] == "XFAIL" + issues = {(r["instance"], r["check"]): r["issue"] for r in verdict["results"]} + assert issues["w-1", "exited-ok"] == issues["w-1", "wrote-done"] == "o/r#1" + # w-0 passed checks that w-1 failed: a race, not a fix. + assert issues["w-0", "exited-ok"] is None and issues["w-0", "wrote-done"] is None + + +def test_evaluate_unexpected_pass_fails(run_dir, tmp_path): + expectations = [ + {"issue": "o/r#1", "checks": CHECKS}, + {"issue": "o/r#2", "checks": ["experiment/timeline-written"]}, + ] + path = _edited(tmp_path, expectations=expectations) + assert evaluate(run_dir, "--experiment", path) == 1 + verdict = read_verdict(run_dir) + assert verdict["verdict"] == "FAIL" + xpass = [r for r in verdict["results"] if r["ok"] and r["issue"]] + assert [(r["check"], r["issue"]) for r in xpass] == [("timeline-written", "o/r#2")] + + +def test_missing_exit_code(run_dir, adapters_dir): + ( + run_dir + / "workloads" + / "w" + / "instances" + / "w-1" + / "output" + / ".chaos" + / "exit_code" + ).unlink() + exp, _ = load_experiment(_write(run_dir.parent, EXPERIMENT)) + adapter = load_adapter(adapters_dir(), "openshell-docker") + info = RunInfo("evaluated", 1, "openshell-docker", RUN_ID, 0, run_dir) + results = asyncio.run(run_checks(exp, adapter, info)) + missing = next(r for r in results if r.instance == "w-1" and r.check == "exited-ok") + assert not missing.ok and missing.output == "no exit code recorded" + + +def _write(directory: Path, raw: dict) -> Path: + path = directory / f"{raw['id']}.yaml" + path.write_text(yaml.safe_dump(raw)) + return path + + +def test_verdict_of(): + ok = Result("workload", "w", "w-0", "c", "d", True, "") + bad = Result("workload", "w", "w-1", "c", "d", False, "boom") + assert verdict_of([ok], []) == "PASS" + assert verdict_of([ok, bad], []) == "FAIL" + assert verdict_of([ok], ["timeout"]) == "ERROR" + assert verdict_of([], []) == "PASS" + known = Result("workload", "w", "w-1", "c", "d", False, "boom", issue="o/r#1") + fixed = Result("workload", "w", "w-0", "c", "d", True, "", issue="o/r#1") + assert verdict_of([ok, known], []) == "XFAIL" + assert verdict_of([known, bad], []) == "FAIL" + assert verdict_of([fixed], []) == "FAIL" + assert verdict_of([known], ["timeout"]) == "ERROR" + + +def test_format_verdict(tmp_path): + bad = Result( + "workload", "w", "w-1", "c", "d", False, "\n".join(f"l{i}" for i in range(9)) + ) + verdict = write_verdict(tmp_path, [bad], ["timeout"]) + text = format_verdict(verdict) + assert " FAIL w-1/c\n" in text and "l8" in text and "l3" not in text + assert text.splitlines()[-2:] == [" error: timeout", "ERROR"] + + +def test_format_verdict_marks_known_failures(tmp_path): + known = Result("workload", "w", "w-1", "c", "d", False, "boom", issue="o/r#1") + fixed = Result("experiment", None, None, "e", "d", True, "", issue="o/r#2") + text = format_verdict(write_verdict(tmp_path, [known, fixed], [])) + assert " xfail w-1/c (known: o/r#1)" in text + assert " XPASS experiment/e (expected to fail: o/r#2; fixed?)" in text + + +def test_validate_command(capsys): + assert cli.main(["validate", str(FIXTURES / "valid-minimal.yaml")]) == 0 + assert cli.main(["validate", str(FIXTURES / "invalid-no-checks.yaml")]) == 1 + assert "invalid-no-checks.yaml" in capsys.readouterr().err + + +def test_list_and_show(capsys): + assert cli.main(["list"]) == 0 + assert "gateway-kill-keeps-state (v" in capsys.readouterr().out + assert cli.main(["show", "smoke"]) == 0 + out = capsys.readouterr().out + assert "check journal-complete:" in out and "hypothesis:" in out + assert cli.main(["list", "--issue", "#4910"]) == 0 + out = capsys.readouterr().out + assert "gateway-kill-keeps-state (v" in out and "smoke (v" not in out + assert cli.main(["show", "gateway-kill-keeps-state"]) == 0 + assert ( + "NVIDIA/OpenShell#4910, known to fail:\n counter/exited-ok\n" + in capsys.readouterr().out + ) + + +def test_list_and_show_fit_80_columns(capsys): + cli.main(["list"]) + ids = [line for line in capsys.readouterr().out.splitlines() if " (v" in line] + assert ids + for line in ids: + cli.main(["show", line.split(" (v")[0]]) + cli.main(["list"]) + lines = capsys.readouterr().out.splitlines() + assert [line for line in lines if len(line) > 80] == [] + + +def test_run_refuses_without_wrapper_env(monkeypatch, tmp_path, capsys): + adapter = load_adapter(cli.DEFAULT_ADAPTERS_DIR, "openshell-docker") + for name in adapter.requires_env: + monkeypatch.delenv(name, raising=False) + smoke = cli.DEFAULT_EXPERIMENTS_DIR / "smoke.yaml" + code = cli.main(["run", str(smoke), "--runs-dir", str(tmp_path)]) + assert code == 2 + assert "needs these environment variables:\n OPENSHELL_BIN\n" in ( + capsys.readouterr().err + ) + assert not any(tmp_path.iterdir()) + + +def _edited(tmp_path: Path, **changes) -> str: + """The experiment with `changes`.""" + edited = {**json.loads(json.dumps(EXPERIMENT)), **changes} + path = tmp_path / "evaluated.yaml" + path.write_text(yaml.safe_dump(edited)) + return str(path) + + +def _results(run_dir: Path) -> dict[tuple[str | None, str], dict]: + return {(r["instance"], r["check"]): r for r in read_verdict(run_dir)["results"]} + + +OPERATION = { + "name": "f", + "description": "d", + "uses": "kill", + "target": {"workload": "w", "role": "supervisor"}, + "select": "one", +} + + +@pytest.mark.parametrize( + ("selected", "code", "verdict"), + [(["w-1"], 0, "XFAIL"), (["w-0"], 1, "FAIL")], +) +def test_evaluate_expectation_on_affected_instances( + run_dir, tmp_path, selected, code, verdict +): + with (run_dir / "timeline.jsonl").open("w") as f: + for event in ( + { + "event": "operation.started", + "operation": "f", + "scope": "instances", + "instances": selected, + }, + {"event": "run.finished", "verdict": "FAIL"}, + ): + f.write(json.dumps(event) + "\n") + started = {"wait": {"workload": "w", "condition": "started"}} + path = _edited( + tmp_path, + operations=[OPERATION], + flow={"parallel": ["w", {"serial": [started, "f"]}]}, + expectations=[{"issue": "o/r#1", "checks": CHECKS, "instances": "affected"}], + ) + assert evaluate(run_dir, "--experiment", path) == code + assert read_verdict(run_dir)["verdict"] == verdict + # Only the selected instance is covered; the other one is the control. + results = _results(run_dir) + assert {i for (i, _), r in results.items() if r["issue"]} == set(selected) + + +def test_evaluate_flaky_expectation(run_dir, tmp_path): + expectation = {"issue": "o/r#1", "checks": CHECKS, "kind": "flaky"} + path = _edited(tmp_path, expectations=[expectation]) + assert evaluate(run_dir, "--experiment", path) == 0 + assert read_verdict(run_dir)["verdict"] == "XFAIL" + # The bug did not trigger anywhere: still a pass, not an XPASS. + w1 = run_dir / "workloads" / "w" / "instances" / "w-1" / "output" + (w1 / ".chaos" / "exit_code").write_text("0\n") + (w1 / "result").write_text("done\n") + index = run_dir.parents[1] / "index.jsonl" + assert evaluate(run_dir, "--experiment", path, "--index", str(index)) == 0 + assert read_verdict(run_dir)["verdict"] == "PASS" + assert all(r["flaky"] for r in _results(run_dir).values() if r["issue"]) + # The index says which flaky known failures did not trigger. + assert read_verdict(run_dir)["read"]["experiment"]["file"] == path + (line,) = [json.loads(line) for line in index.read_text().splitlines()] + assert line["flaky_missed"] == CHECKS + strict = _edited(tmp_path, expectations=[{"issue": "o/r#1", "checks": CHECKS}]) + assert evaluate(run_dir, "--experiment", strict) == 1 + + +@pytest.mark.parametrize( + ("run", "timeout"), [("no-such-command-xyz", "1m"), ("sleep 5", "1s")] +) +def test_evaluate_checker_error_is_error(run_dir, tmp_path, run, timeout): + workloads = json.loads(json.dumps(EXPERIMENT["workloads"])) + workloads[0]["checks"][1].update(run=run, timeout=timeout) + # The expectation does not excuse a check that could not decide. + path = _edited( + tmp_path, + workloads=workloads, + expectations=[{"issue": "o/r#1", "checks": CHECKS}], + ) + assert evaluate(run_dir, "--experiment", path) == 2 + assert read_verdict(run_dir)["verdict"] == "ERROR" + errored = _results(run_dir)["w-0", "wrote-done"] + assert errored["error"] and errored["issue"] is None + + +def test_selected_instances(): + events = [ + { + "event": "operation.started", + "operation": "f", + "scope": "instances", + "instances": ["w-0"], + }, + { + "event": "operation.started", + "operation": "a", + "scope": "instances", + "instances": ["w-1", "w-2"], + }, + {"event": "operation.started", "operation": "g", "scope": "global"}, + {"event": "operation.completed", "operation": "f", "outcome": "ok"}, + ] + assert selected_instances(events) == {"w-0", "w-1", "w-2"} + + +def test_verdict_and_format_of_errors_and_flaky(tmp_path): + error = Result("workload", "w", "w-0", "c", "d", False, "x", error=True) + flaky_ok = Result( + "workload", "w", "w-0", "f", "d", True, "", issue="o/r#1", flaky=True + ) + flaky_bad = Result( + "workload", "w", "w-1", "f", "d", False, "x", issue="o/r#1", flaky=True + ) + assert verdict_of([error], []) == "ERROR" + assert verdict_of([flaky_ok], []) == "PASS" + assert verdict_of([flaky_ok, flaky_bad], []) == "XFAIL" + text = format_verdict(write_verdict(tmp_path, [error, flaky_ok, flaky_bad], [])) + assert " ERROR w-0/c (the check could not decide)" in text + assert " ok w-0/f (flaky: o/r#1 did not trigger)" in text + assert " xfail w-1/f (known, flaky: o/r#1)" in text + + +def _scrapes( + run_dir: Path, samples: list[dict[str, float]], source: str = "gw" +) -> None: + directory = run_dir / "system" / "metrics" + directory.mkdir(parents=True, exist_ok=True) + (directory / f"{source}.jsonl").write_text( + "".join( + json.dumps({"t": float(t), "up": 1, "samples": s}) + "\n" + for t, s in enumerate(samples) + ) + + json.dumps({"t": float(len(samples)), "up": 0, "samples": {}}) + + "\n" + ) + + +def _metric_check(name: str, **fields) -> dict: + fields = {k.replace("_", "-"): v for k, v in fields.items()} + return {"name": name, "description": "d", "kind": "metric", **fields} + + +def test_metric_checks_read_the_recorded_scrapes(run_dir, tmp_path): + _scrapes( + run_dir, + [ + {'req_total{m="a"}': 10, 'req_total{m="b"}': 1, "sessions": 1}, + {'req_total{m="a"}': 16, 'req_total{m="b"}': 1, "sessions": 3}, + # A counter reset: the increase counts 2, not -14. + {'req_total{m="a"}': 2, 'req_total{m="b"}': 0, "sessions": 0}, + ], + ) + ( + run_dir / "workloads" / "w" / "instances" / "w-1" / "output" / "result" + ).write_text("done\n") + path = _edited( + tmp_path, + metrics=["req_total", "sessions", "lat_seconds"], + checks=[ + _metric_check("peak", metric="sessions", value="max", at_least=3), + _metric_check("last", metric="sessions", equals=0), + _metric_check("grew", metric="req_total", value="increase", equals=8), + _metric_check( + "a-only", metric="req_total", labels={"m": "a"}, value="min", at_least=2 + ), + _metric_check("never", metric="lat_seconds_count", at_least=1), + _metric_check("elsewhere", metric="sessions", source="other", equals=0), + ], + ) + # One exit code is 1, so the verdict is FAIL whatever the metric checks say. + assert evaluate(run_dir, "--experiment", path) == 2 + results = _results(run_dir) + assert [ + (results[None, n]["ok"], results[None, n]["error"]) + for n in ("peak", "last", "grew", "a-only", "never", "elsewhere") + ] == [(True, False)] * 4 + [(False, False), (False, True)] + assert results[None, "grew"]["output"] == ( + "increase = 8 (want equals 8); 2 series of req_total summed over 3 scrapes of gw" + ) + assert results[None, "a-only"]["output"].startswith( + 'min = 2 (want at-least 2); 1 series of req_total{m="a"}' + ) + assert results[None, "never"]["output"] == ( + "no series lat_seconds_count in any scrape (gw)" + ) + assert results[None, "elsewhere"]["output"] == ( + "no scrapes of metrics source 'other' in system/metrics" + ) + + +def test_metric_check_needs_one_source(run_dir, tmp_path): + _scrapes(run_dir, [{"sessions": 1}], source="gw") + _scrapes(run_dir, [{"sessions": 2}], source="gw2") + path = _edited( + tmp_path, + metrics=["sessions"], + checks=[ + _metric_check("either", metric="sessions", at_least=1), + _metric_check("second", metric="sessions", source="gw2", equals=2), + ], + ) + evaluate(run_dir, "--experiment", path) + results = _results(run_dir) + assert results[None, "either"]["error"] + assert ( + "more than one source (gw, gw2); set source" + in results[None, "either"]["output"] + ) + assert results[None, "second"]["ok"] diff --git a/e2e/chaos/tests/test_kubernetes.py b/e2e/chaos/tests/test_kubernetes.py new file mode 100644 index 0000000000..82957b4e83 --- /dev/null +++ b/e2e/chaos/tests/test_kubernetes.py @@ -0,0 +1,262 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The kubernetes backend: adapter rules and which experiments validate on +openshell-kubernetes. The adapter has not been run against a cluster, so +nothing here exercises the injector's kubectl calls.""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import pytest +import yaml +from conftest import ADAPTERS, EXPERIMENTS, minimal_adapter + +from chaos.adapter import load_adapter +from chaos.cli import main +from chaos.model import ExperimentError + + +def kubernetes_adapter(**extra) -> dict: + """A Kubernetes adapter whose hooks only record that they ran.""" + adapter = minimal_adapter( + backend="kubernetes", + kubernetes={"context": "${CHAOS_TEST_CONTEXT}"}, + targets={ + "gateway": { + "type": "workload", + "namespace": "openshell", + "resource": "statefulset/openshell", + "start": 'echo started >> "$CHAOS_TEST_LOG"', + }, + "roles": { + "sandbox": { + "type": "pod", + "namespace": "sandboxes", + "labels": { + "pair": "${CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_SYSTEM_ID}", + "role": "workload", + }, + }, + "supervisor": { + "type": "pod", + "namespace": "sandboxes", + "labels": { + "pair": "${CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_SYSTEM_ID}", + "role": "supervisor", + }, + }, + }, + }, + ) + adapter.update(extra) + return adapter + + +def write_adapter(tmp_path: Path, adapter: dict) -> Path: + directory = tmp_path / "adapters" + directory.mkdir(exist_ok=True) + (directory / f"{adapter['id']}.yaml").write_text(yaml.safe_dump(adapter)) + return directory + + +def load(tmp_path: Path, **extra): + return load_adapter( + write_adapter(tmp_path, kubernetes_adapter(**extra)), "openshell-docker" + ) + + +# Adapter rules. + + +def test_openshell_kubernetes_adapter_loads(): + adapter = load_adapter(ADAPTERS, "openshell-kubernetes") + assert adapter.backend == "kubernetes" + assert adapter.kubernetes is not None + assert adapter.kubernetes.context == "${OPENSHELL_E2E_KUBE_CONTEXT}" + assert adapter.target("gateway").type == "workload" + assert {r.type for r in adapter.targets.roles.values()} == {"pod"} + assert {c.name for c in adapter.checks} == {"no-leaked-pods", "gateway-healthy"} + assert adapter.setup + + +def test_kubernetes_needs_a_context(tmp_path): + adapter = kubernetes_adapter() + del adapter["kubernetes"] + with pytest.raises( + ExperimentError, match="the kubernetes backend needs kubernetes.context" + ): + load_adapter(write_adapter(tmp_path, adapter), "openshell-docker") + + +def test_docker_takes_no_kubernetes_block(tmp_path): + adapter = minimal_adapter(kubernetes={"context": "kind-chaos"}) + with pytest.raises(ExperimentError, match="kubernetes"): + load_adapter(write_adapter(tmp_path, adapter), "openshell-docker") + + +def test_workload_target_needs_namespace_resource_and_start(tmp_path): + with pytest.raises( + ExperimentError, match="a workload target needs namespace, resource and start" + ): + load( + tmp_path, + targets={"gateway": {"type": "workload", "namespace": "openshell"}}, + ) + + +def test_pod_target_needs_a_namespace(tmp_path): + targets = {"roles": {"sandbox": {"type": "pod", "labels": {"role": "workload"}}}} + with pytest.raises(ExperimentError, match="a pod target needs a namespace"): + load(tmp_path, targets=targets) + + +def test_process_target_is_not_available_on_kubernetes(tmp_path): + targets = {"gateway": {"type": "process", "pid-file": "/pid", "start": "true"}} + with pytest.raises( + ExperimentError, match="a process target is not available on kubernetes" + ): + load(tmp_path, targets=targets) + + +def test_pod_target_is_not_available_on_docker(tmp_path): + targets = { + "roles": { + "sandbox": { + "type": "pod", + "namespace": "ns", + "labels": {"role": "workload"}, + } + } + } + adapter = minimal_adapter(targets=targets) + with pytest.raises( + ExperimentError, match="a pod target is not available on docker" + ): + load_adapter(write_adapter(tmp_path, adapter), "openshell-docker") + + +# Which shipped experiments validate on openshell-kubernetes. + + +def _validate(tmp_path, capsys, experiment: str) -> tuple[int, str]: + directory = tmp_path / "adapters" + shutil.copytree(ADAPTERS, directory) + argv = [ + "--adapters-dir", + str(directory), + "validate", + "--adapter", + "openshell-kubernetes", + ] + code = main([*argv, str(EXPERIMENTS / f"{experiment}.yaml")]) + out = capsys.readouterr() + return code, out.out + out.err + + +@pytest.mark.parametrize( + "experiment", + [ + "hello-world", + "smoke", + "gateway-graceful-restart", + "gateway-kill-keeps-state", + "policy-tighten-survives-gateway-restart", + "supervisor-kill", + ], +) +def test_experiment_validates_on_kubernetes(tmp_path, capsys, experiment): + code, out = _validate(tmp_path, capsys, experiment) + assert code == 0, out + assert "ok on openshell-kubernetes:" in out + + +@pytest.mark.parametrize( + ("experiment", "reason"), + [ + ( + "supervisor-pause", + "freeze on a pod target (role supervisor) is not supported on " + "kubernetes: freezing a pod's processes needs node-level cgroup access", + ), + ( + "sandbox-container-restart", + "restart-in-place on a pod target (role sandbox) is not supported on " + "kubernetes: the Kubernetes driver runs sandbox pods with " + "restartPolicy: Never", + ), + ("sandbox-stop-start", "openshell-kubernetes has no signal 'reconcile-sweep'"), + ], +) +def test_experiment_is_not_applicable_on_kubernetes( + tmp_path, capsys, experiment, reason +): + code, out = _validate(tmp_path, capsys, experiment) + assert code == 0, out + assert "n/a on openshell-kubernetes:" in out + assert reason in out + + +def test_run_of_a_not_applicable_experiment_exits_0_without_a_run(tmp_path, capsys): + runs = tmp_path / "runs" + code = main( + [ + "run", + "--adapter", + "openshell-kubernetes", + "--runs-dir", + str(runs), + str(EXPERIMENTS / "supervisor-pause.yaml"), + ] + ) + assert code == 0 + assert "n/a on openshell-kubernetes:" in capsys.readouterr().out + assert not runs.exists() + + +def test_not_applicable_on_the_default_adapter_is_invalid(tmp_path, capsys): + raw = yaml.safe_load((EXPERIMENTS / "supervisor-pause.yaml").read_text()) + raw["default-adapter"] = "openshell-kubernetes" + path = tmp_path / "supervisor-pause.yaml" + path.write_text(yaml.safe_dump(raw, sort_keys=False)) + code = main(["validate", str(path)]) + out = capsys.readouterr() + assert code == 1 + assert "invalid on openshell-kubernetes:" in out.err + assert "(on its default-adapter)" in out.err + + +def test_validate_all_adapters_prints_the_matrix(capsys): + code = main(["validate", "--all-adapters"]) + out = capsys.readouterr().out + assert code == 0, out + table, _, reasons = out.partition("\n\n") + lines = table.splitlines() + assert lines[0].split() == [ + "experiment", + "openshell-docker", + "openshell-kubernetes", + ] + rows = {line.split()[0]: line.split()[1:] for line in lines[1:]} + assert rows["supervisor-kill"] == ["ok", "ok"] + assert rows["supervisor-pause"] == ["ok", "n/a"] + assert rows["sandbox-container-restart"] == ["ok", "n/a"] + assert "sandbox-container-restart on openshell-kubernetes: n/a:" in reasons + + +def test_validate_takes_adapter_or_all_adapters_not_both(capsys): + with pytest.raises(SystemExit): + main(["validate", "--adapter", "openshell-docker", "--all-adapters"]) + capsys.readouterr() + + +def test_validate_needs_files_without_all_adapters(capsys): + assert main(["validate"]) == 2 + assert "give experiment files" in capsys.readouterr().err + + +def test_unavailable_names_a_builtin(tmp_path): + with pytest.raises(ExperimentError, match="unavailable.pause: not a built-in"): + load(tmp_path, unavailable={"pause": "no"}) diff --git a/e2e/chaos/tests/test_model.py b/e2e/chaos/tests/test_model.py new file mode 100644 index 0000000000..b71219bb81 --- /dev/null +++ b/e2e/chaos/tests/test_model.py @@ -0,0 +1,410 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml +from conftest import ADAPTERS, EXPERIMENTS, FIXTURES + +from chaos.adapter import load_adapter, validate_adapter +from chaos.model import ( + ExperimentError, + Operation, + Parallel, + Ref, + Repeat, + Serial, + Wait, + apply_overrides, + load_experiment, + parse_duration, + parse_experiment, + read_yaml, +) + +# Each invalid fixture and a pattern its error must contain; the pattern names +# the offending field or flow node. +INVALID = { + "invalid-api-version": r"apiVersion: Input should be 'chaos/v1alpha1'", + "invalid-no-api-version": r"apiVersion: Field required", + "invalid-unknown-key": r"colour: Extra inputs are not permitted", + "invalid-missing-description": r"description: Field required", + "invalid-chaos-env": r"workloads\.0\.env: .*'CHAOS_X' must not start with CHAOS_", + "invalid-flow-missing-name": r"flow: 'f' is never used", + "invalid-flow-repeated-name": r"flow\.serial\[1\]: 'w' appears more than once", + "invalid-metric-name": r"metrics\.0: .*'rate\(x\[1m\]\)' is not a metric family name", + "invalid-metric-entry-both": r"metrics\.0: .*set exactly one of family and expr", + "invalid-metric-check-no-bound": r"checks\.0\.metric: .*set equals, at-least or at-most", + "invalid-metric-check-undeclared": r"checks\.0\.metric: 'other_total' is not a family under metrics:", + "invalid-wait-outside-parallel": r"flow\.serial\[1\]\.wait: wait on 'w' must be in a sibling branch", + "invalid-no-checks": r"workloads\.0\.checks: List should have at least 1 item", + "invalid-outage-no-duration": r"operations\.0: .*outage requires duration", + "invalid-restart-duration": r"operations\.0: .*restart is one-shot and takes no duration", + "invalid-role-fault-no-wait": r"operations\.f: must come after wait: \{workload: w, condition: started\}", + "invalid-role-fault-wait-after": r"operations\.f: must come after wait: \{workload: w, condition: started\}", + "invalid-data-name": r"data\.Bad_Name: want lowercase letters, digits, '-'", + "invalid-operation-no-wait": r"operations\.a: must come after wait: \{workload: w, condition: started\}", + "invalid-operation-global-select": r"operations\.0: .*scope global takes no select", + "invalid-operation-no-targeting": r"operations\.0: .*set exactly one of target and scope", + "invalid-operation-name-clash": r"duplicate workload/operation name 'w'", + "invalid-expectation-issue": r"expectations\.0\.issue: .*must be a URL or look like /#", + "invalid-expectation-unknown-check": r"expectations\.0\.checks: no check 'w/exited-ok'", + "invalid-expectation-shared-check": r"expectations\.1\.checks: 'w/ok' is in more than one expectation", + "invalid-expectation-affected-experiment-check": r"expectations\.0: only workload checks run per instance", + "invalid-expectation-affected-untargeted": r"expectations\.0\.instances: no operation targets w instances", + "invalid-repeat-workload": r"flow\.repeat\.serial\[0\]: only adapter operations can be in a repeat, not 'w'", + "invalid-repeat-shape": r"flow\.serial\[1\]\.repeat: want \{times: , serial: \[\.\.\.\]\}", +} + + +def test_every_invalid_fixture_is_covered(): + assert {p.stem for p in FIXTURES.glob("invalid-*.yaml")} == set(INVALID) + + +@pytest.mark.parametrize("name", sorted(INVALID)) +def test_invalid_fixture_is_rejected(name): + with pytest.raises(ExperimentError, match=INVALID[name]): + load_experiment(FIXTURES / f"{name}.yaml") + + +@pytest.mark.parametrize( + "path", sorted(FIXTURES.glob("valid-*.yaml")), ids=lambda p: p.stem +) +def test_valid_fixture_loads(path): + exp, _ = load_experiment(path) + assert exp.id == path.stem + + +@pytest.mark.parametrize( + "path", sorted(EXPERIMENTS.glob("*.yaml")), ids=lambda p: p.stem +) +def test_shipped_experiment_validates_against_its_adapter(path): + exp, _ = load_experiment(path) + validate_adapter(exp, load_adapter(ADAPTERS, exp.default_adapter)) + + +def test_id_must_equal_file_name(tmp_path): + path = tmp_path / "other-name.yaml" + path.write_text((FIXTURES / "valid-minimal.yaml").read_text()) + with pytest.raises(ExperimentError, match="must equal the file name 'other-name'"): + load_experiment(path) + + +def test_exit_code_check_only_in_workloads(tmp_path): + raw = read_yaml(FIXTURES / "valid-minimal.yaml") + raw["id"] = "exit-code-outside" + raw["checks"] = [ + {"name": "e", "description": "d", "kind": "exit-code", "equals": 0} + ] + path = tmp_path / "exit-code-outside.yaml" + path.write_text(yaml.safe_dump(raw)) + with pytest.raises( + ExperimentError, match=r"checks\.0: Input tag 'exit-code' .* 'shell', 'metric'" + ): + load_experiment(path) + + +def test_keys_are_kebab_case(tmp_path): + raw = read_yaml(FIXTURES / "valid-minimal.yaml") + raw["id"] = "snake-case-key" + raw["requires_env"] = [] + path = tmp_path / "snake-case-key.yaml" + path.write_text(yaml.safe_dump(raw)) + with pytest.raises( + ExperimentError, match=r"requires_env: Extra inputs are not permitted" + ): + load_experiment(path) + + +def test_operation_must_suit_the_target_type(tmp_path, adapters_dir): + exp, _ = load_experiment(FIXTURES / "valid-freeze.yaml") + validate_adapter(exp, load_adapter(adapters_dir(), "openshell-docker")) + text = (FIXTURES / "valid-role-fault.yaml").read_text() + path = tmp_path / "valid-role-fault.yaml" + path.write_text(text.replace("kill", "restart")) + exp, _ = load_experiment(path) + with pytest.raises( + ExperimentError, + match=r"operations\.f: restart on a container target " + r"\(role supervisor\) is not supported on docker", + ): + validate_adapter(exp, load_adapter(adapters_dir(), "openshell-docker")) + + +def test_flow_tree_has_paths(): + exp, _ = load_experiment(FIXTURES / "valid-role-fault.yaml") + assert isinstance(exp.flow, Parallel) + workload, branch = exp.flow.children + assert isinstance(workload, Ref) and workload.path == "flow.parallel[0]" + assert isinstance(branch, Serial) + assert isinstance(branch.children[0], Wait) and branch.children[0].workload == "w" + assert branch.children[1].path == "flow.parallel[1].serial[1]" + + +def test_repeat_parses_times_and_for(tmp_path): + raw = read_yaml(FIXTURES / "valid-operations.yaml") + raw["id"] = "rep" + raw["flow"] = { + "parallel": [ + "w", + { + "serial": [ + {"wait": {"workload": "w", "condition": "started"}}, + {"repeat": {"for": "1m", "serial": ["a", {"delay": "1s"}]}}, + {"repeat": {"times": 2, "serial": ["g"]}}, + ] + }, + ] + } + path = tmp_path / "rep.yaml" + path.write_text(yaml.safe_dump(raw)) + exp, _ = load_experiment(path) + _, branch = exp.flow.children + timed, counted = branch.children[1:] + assert isinstance(timed, Repeat) and (timed.seconds, timed.times) == (60, None) + assert timed.children[0].path == "flow.parallel[1].serial[1].repeat.serial[0]" + assert isinstance(counted, Repeat) and (counted.seconds, counted.times) == (None, 2) + + +@pytest.mark.parametrize( + ("value", "seconds"), [("500ms", 0.5), ("10s", 10), ("6m", 360), ("1h", 3600)] +) +def test_parse_duration(value, seconds): + assert parse_duration(value) == seconds + + +@pytest.mark.parametrize("value", ["10", "1.5s", "10 s", "1d", ""]) +def test_parse_duration_rejects(value): + with pytest.raises(ValueError, match="invalid duration"): + parse_duration(value) + + +# --set overrides. + + +def test_override_addresses_list_items_by_name(): + exp, raw = load_experiment( + EXPERIMENTS / "smoke.yaml", + [ + "workloads.counter.instances=3", + "workloads.counter.env.TICKS=7", + "timeout=20m", + ], + ) + assert exp.workload("counter").instances == 3 + assert exp.workload("counter").env["TICKS"] == "7" + assert exp.timeout == "20m" + assert raw["workloads"][0]["instances"] == 3 + + +def test_override_does_not_mutate_input(): + raw = {"workloads": [{"name": "w", "instances": 1}]} + apply_overrides(raw, ["workloads.w.instances=2"]) + assert raw["workloads"][0]["instances"] == 1 + + +def test_override_creates_missing_mappings(): + data = apply_overrides( + {"workloads": [{"name": "w"}]}, ["workloads.w.options.image=x"] + ) + assert data["workloads"][0]["options"] == {"image": "x"} + + +def test_override_empty_value_is_empty_string(): + assert apply_overrides({"a": "b"}, ["a="]) == {"a": ""} + + +@pytest.mark.parametrize( + ("override", "message"), + [ + ("timeout", "want path=value"), + ("=1", "want path=value"), + ("workloads.nope.instances=2", "no item named 'nope'"), + ("workloads.counter=2", "cannot replace a whole list item"), + ("timeout.minutes=2", "timeout is not a mapping or list"), + ], +) +def test_override_errors(override, message): + with pytest.raises(ExperimentError, match=message): + load_experiment(EXPERIMENTS / "smoke.yaml", [override]) + + +def test_override_result_is_validated(): + with pytest.raises(ExperimentError, match=r"workloads\.0\.instances"): + load_experiment(EXPERIMENTS / "smoke.yaml", ["workloads.counter.instances=0"]) + + +def _with_preflight(tmp_path, names: list[str]): + raw = read_yaml(FIXTURES / "valid-minimal.yaml") + raw["id"] = "pf" + raw["preflight"] = [ + {"name": n, "description": "d", "kind": "shell", "run": "true"} for n in names + ] + path = tmp_path / "pf.yaml" + path.write_text(yaml.safe_dump(raw)) + return path + + +def test_preflight_names(tmp_path): + with pytest.raises(ExperimentError, match="duplicate preflight check"): + load_experiment(_with_preflight(tmp_path, ["a", "a"])) + exp, _ = load_experiment(_with_preflight(tmp_path, ["docker-reachable"])) + with pytest.raises(ExperimentError, match="already has a preflight check"): + validate_adapter(exp, load_adapter(ADAPTERS, exp.default_adapter)) + + +# Data and adapter operations. + + +def _operations_fixture(tmp_path, edit) -> Path: + raw = read_yaml(FIXTURES / "valid-operations.yaml") + raw["id"] = "act" + edit(raw) + path = tmp_path / "act.yaml" + path.write_text(yaml.safe_dump(raw)) + return path + + +@pytest.mark.parametrize( + ("edit", "error"), + [ + ( + lambda raw: raw["workloads"][0]["options"].update(policy="nope"), + r"workloads\.w\.options\.policy: no data entry named 'nope'", + ), + ( + lambda raw: raw["operations"][1]["options"].update(policy="nope"), + r"operations\.g\.options\.policy: no data entry named 'nope'", + ), + ( + lambda raw: raw["operations"][1].update(options={}), + r"operations\.g\.options: option policy is required", + ), + ( + lambda raw: raw["operations"][1]["options"].update(colour="red"), + r"operations\.g\.options: unknown option 'colour' for openshell-policy-set", + ), + ], +) +def test_options_are_checked_against_the_adapter(tmp_path, edit, error): + exp, _ = load_experiment(_operations_fixture(tmp_path, edit)) + with pytest.raises(ExperimentError, match=error): + validate_adapter(exp, load_adapter(ADAPTERS, exp.default_adapter)) + + +def test_adapter_needs_the_version_marker(adapters_dir): + directory = adapters_dir() + path = directory / "openshell-docker.yaml" + path.write_text(path.read_text().replace("apiVersion: chaos/v1alpha1", "")) + with pytest.raises(ExperimentError, match=r"apiVersion: Field required"): + load_adapter(directory, "openshell-docker") + + +def test_limits_are_the_adapters(tmp_path, adapters_dir): + """A `run` script or an `env` value over the adapter's limits is rejected + against that adapter, with the limit named; the harness has no limit.""" + raw = read_yaml(FIXTURES / "valid-minimal.yaml") + raw["workloads"][0]["run"] = "x" * 100 + raw["workloads"][0]["env"] = {"BIG": "y" * 50} + path = tmp_path / "valid-minimal.yaml" + path.write_text(yaml.safe_dump(raw)) + exp, _ = load_experiment(path) + validate_adapter(exp, load_adapter(adapters_dir(), "openshell-docker")) + limits = {"env-value-bytes": 50, "run-script-bytes": 99} + adapter = load_adapter(adapters_dir(limits=limits), "openshell-docker") + with pytest.raises( + ExperimentError, + match=r"workloads\.w\.run: 100 bytes is over limits\.run-script-bytes " + r"\(99\) of openshell-docker", + ): + validate_adapter(exp, adapter) + limits = {"env-value-bytes": 49, "run-script-bytes": 100} + adapter = load_adapter(adapters_dir(limits=limits), "openshell-docker") + with pytest.raises( + ExperimentError, + match=r"workloads\.w\.env\.BIG: 50 bytes is over limits\.env-value-bytes " + r"\(49\) of openshell-docker", + ): + validate_adapter(exp, adapter) + + +def test_operation_must_be_in_the_adapter(adapters_dir): + exp, _ = load_experiment(FIXTURES / "valid-operations.yaml") + validate_adapter(exp, load_adapter(ADAPTERS, exp.default_adapter)) + options = {"policy": {"default": "", "data": True}} + adapter = load_adapter(adapters_dir(options=options), "openshell-docker") + with pytest.raises( + ExperimentError, + match=r"operations\.a: openshell-docker has no operation 'openshell-policy-set'", + ): + validate_adapter(exp, adapter) + + +def test_container_restart_is_one_shot(tmp_path, adapters_dir): + text = (FIXTURES / "valid-role-fault.yaml").read_text() + path = tmp_path / "valid-role-fault.yaml" + path.write_text(text.replace("kill", "restart-in-place")) + exp, _ = load_experiment(path) + validate_adapter(exp, load_adapter(adapters_dir(), "openshell-docker")) + + path.write_text( + text.replace("kill", "restart-in-place").replace( + "select: one}", "select: one, duration: 10s}" + ) + ) + with pytest.raises(ExperimentError, match="restart-in-place is one-shot"): + load_experiment(path) + + +@pytest.mark.parametrize( + ("legacy", "current"), + [ + ("process-restart", "restart"), + ("process-outage", "outage"), + ("container-kill", "kill"), + ("container-restart", "restart-in-place"), + ("container-pause", "freeze"), + ], +) +def test_legacy_operation_names_load_as_their_current_name(legacy, current): + op = Operation.model_validate( + { + "name": "f", + "description": "d", + "uses": legacy, + "target": "gateway", + "select": "all", + } + | ({"duration": "1s"} if current in {"outage", "freeze"} else {}) + ) + assert op.uses == current + + +def test_metrics_entries_and_metric_checks(tmp_path): + raw = read_yaml(FIXTURES / "valid-minimal.yaml") + raw["id"] = "metrics" + raw["metrics"] = [ + "up_gauge", + {"family": "lat_seconds", "description": "d"}, + {"title": "Up", "expr": "sum(up_gauge)"}, + ] + raw["checks"] = [ + { + "name": "slow", + "description": "d", + "kind": "metric", + "metric": "lat_seconds_count", + "value": "increase", + "at-least": 1, + "at-most": 9, + } + ] + exp = parse_experiment(raw, "test") + assert exp.metric_families == ["up_gauge", "lat_seconds"] + assert [m.title for m in exp.metrics if m.expr] == ["Up"] + (check,) = exp.checks + assert check.wants() == "at-least 1 and at-most 9" + assert check.holds(9) and not check.holds(9.5) diff --git a/e2e/chaos/tests/test_observe.py b/e2e/chaos/tests/test_observe.py new file mode 100644 index 0000000000..6ee1a3eda3 --- /dev/null +++ b/e2e/chaos/tests/test_observe.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Metrics scrapes and watch commands, against files and shell commands.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from conftest import minimal_adapter + +from chaos.adapter import Adapter +from chaos.observe import Observers, parse_prometheus +from chaos.timeline import Timeline + +EXPOSITION = """\ +# HELP openshell_server_grpc_requests_total gRPC requests. +# TYPE openshell_server_grpc_requests_total counter +openshell_server_grpc_requests_total{method="Health",code="0"} 3 +openshell_server_grpc_requests_total{method="a b",note="x}y \\"q\\""} 1 1700000000000 +# TYPE openshell_server_supervisor_sessions gauge +openshell_server_supervisor_sessions 2 +broken_sample not-a-number +""" + + +def test_parse_prometheus(): + samples, types = parse_prometheus(EXPOSITION) + assert samples == { + 'openshell_server_grpc_requests_total{method="Health",code="0"}': 3.0, + 'openshell_server_grpc_requests_total{method="a b",note="x}y \\"q\\""}': 1.0, + "openshell_server_supervisor_sessions": 2.0, + } + assert types == { + "openshell_server_grpc_requests_total": "counter", + "openshell_server_supervisor_sessions": "gauge", + } + + +def records(path: Path) -> list[dict]: + return [json.loads(line) for line in path.read_text().splitlines()] + + +def observe(tmp_path: Path, seconds: float, **extra) -> Path: + run_dir = tmp_path / "run" + (run_dir / "logs").mkdir(parents=True) + (run_dir / "system").mkdir() + adapter = Adapter.model_validate(minimal_adapter(**extra)) + timeline = Timeline(run_dir / "timeline.jsonl") + + async def go() -> list[str]: + observers = Observers(adapter, timeline, run_dir, {"RUN": str(run_dir)}) + await observers.start() + await asyncio.sleep(seconds) + await observers.stop() + return observers.errors + + try: + assert asyncio.run(go()) == [] + finally: + timeline.close() + return run_dir + + +def test_scrapes_up_and_down(tmp_path): + exposition = tmp_path / "metrics.txt" + exposition.write_text(EXPOSITION) + run_dir = observe( + tmp_path, + 0.35, + metrics=[ + {"name": "up", "url": f"file://{exposition}", "interval": "100ms"}, + {"name": "down", "url": f"file://{tmp_path}/missing", "interval": "1s"}, + ], + ) + up = records(run_dir / "system" / "metrics" / "up.jsonl") + assert len(up) >= 2 + assert all(r["up"] == 1 for r in up) + assert up[0]["samples"]["openshell_server_supervisor_sessions"] == 2.0 + assert [r["t"] for r in up] == sorted(r["t"] for r in up) + types = json.loads((run_dir / "system" / "metrics" / "up.types.json").read_text()) + assert types["openshell_server_supervisor_sessions"] == "gauge" + (down,) = records(run_dir / "system" / "metrics" / "down.jsonl") + assert down["up"] == 0 and down["error"] + + +def test_watch_records_lines_and_stops(tmp_path): + run_dir = observe( + tmp_path, + 0.5, + watch=[ + {"name": "events", "run": 'echo "{\\"a\\": 1}"; echo "$RUN" >&2; sleep 60'}, + ], + ) + (record,) = records(run_dir / "system" / "watch" / "events.jsonl") + assert json.loads(record["line"]) == {"a": 1} + assert (run_dir / "logs" / "watch-events.log").read_text().strip() == str(run_dir) diff --git a/e2e/chaos/tests/test_recovery.py b/e2e/chaos/tests/test_recovery.py new file mode 100644 index 0000000000..83b8906ca8 --- /dev/null +++ b/e2e/chaos/tests/test_recovery.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""State that must survive a retry: the wrapper's exit code and the undo stack.""" + +from __future__ import annotations + +import asyncio +import os +import subprocess +from typing import cast + +from chaos.inject.base import Undo +from chaos.inject.docker import DockerInjector +from chaos.instances import WRAPPER +from chaos.model import RunError +from chaos.timeline import Timeline + + +def test_wrapper_clears_previous_attempts_exit_code(tmp_path): + chaos = tmp_path / ".chaos" + chaos.mkdir() + (chaos / "exit_code").write_text("0\n") + env = {**os.environ, "CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR": str(tmp_path)} + # The script sees whether the previous attempt's exit code is still there. + script = f'test ! -e "{chaos}/exit_code"' + result = subprocess.run(["sh", "-c", WRAPPER, "chaos", script], env=env) + assert result.returncode == 0 + assert (chaos / "exit_code").read_text() == "0\n" + assert len((chaos / "attempts").read_text().splitlines()) == 1 + + +class _Timeline: + def __init__(self) -> None: + self.events: list[tuple[str, dict]] = [] + + def emit(self, event: str, **fields) -> None: + self.events.append((event, fields)) + + +def _injector() -> tuple[DockerInjector, _Timeline]: + injector = object.__new__(DockerInjector) + injector.undo_stack = [] + timeline = _Timeline() + injector.timeline = cast(Timeline, timeline) + return injector, timeline + + +def test_cancelled_undo_stays_on_the_stack(): + injector, _ = _injector() + started = asyncio.Event() + + async def hang() -> None: + started.set() + await asyncio.sleep(60) + + entry = Undo("pause", "unpause", hang) + injector.undo_stack.append(entry) + + async def go() -> None: + task = asyncio.create_task(injector.undo_all()) + await started.wait() + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + asyncio.run(go()) + assert injector.undo_stack == [entry] + + +def test_failed_undo_is_recorded_and_dropped(): + injector, timeline = _injector() + ran = [] + + async def ok() -> None: + ran.append("older") + + async def fail() -> None: + raise RunError("docker unpause: no such container") + + injector.undo_stack = [Undo("a", "older", ok), Undo("b", "newer", fail)] + errors = asyncio.run(injector.undo_all()) + assert errors == ["undo b (newer): docker unpause: no such container"] + assert ran == ["older"] + assert injector.undo_stack == [] + assert [e for e, _ in timeline.events] == [ + "operation.error", + "operation.reverted", + ] diff --git a/e2e/chaos/tests/test_report.py b/e2e/chaos/tests/test_report.py new file mode 100644 index 0000000000..1ccdaf15f0 --- /dev/null +++ b/e2e/chaos/tests/test_report.py @@ -0,0 +1,467 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The HTML report, built from a synthetic run dir.""" + +from __future__ import annotations + +import itertools +import json +import shutil +from pathlib import Path + +from chaos.export import export +from chaos.report import build, write_report + +TIMELINE = [ + {"t": 0.0, "event": "run.started"}, + { + "t": 1.0, + "event": "instance.created", + "workload": "counter", + "instance": "counter-0", + "system_id": "sb-0", + }, + { + "t": 1.5, + "event": "instance.state", + "workload": "counter", + "instance": "counter-0", + "state": "Running", + }, + { + "t": 4.0, + "event": "operation.started", + "operation": "gateway-down", + "uses": "outage", + "run": 1, + "target": "gateway", + "scope": "target", + "instances": [], + "ids": [], + }, + { + "t": 4.0, + "event": "operation.applied", + "operation": "gateway-down", + "uses": "outage", + }, + { + "t": 9.0, + "event": "operation.reverted", + "operation": "gateway-down", + "uses": "outage", + }, + { + "t": 10.0, + "event": "operation.applied", + "operation": "kill-one", + "uses": "kill", + }, + { + "t": 12.0, + "event": "instance.state", + "workload": "counter", + "instance": "counter-0", + "state": "Succeeded", + }, + { + "t": 12.5, + "event": "instance.collected", + "workload": "counter", + "instance": "counter-0", + "exit_code": 0, + }, + { + "t": 13.0, + "event": "instance.deleted", + "workload": "counter", + "instance": "counter-0", + }, + {"t": 14.0, "event": "run.finished", "verdict": "PASS"}, +] + + +def jsonl(path: Path, records: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(json.dumps(r) + "\n" for r in records)) + + +def run_dir(tmp_path: Path) -> Path: + run = tmp_path / "run" + jsonl( + run / "timeline.jsonl", [{"ts": "2026-10-27T00:00:00Z", **e} for e in TIMELINE] + ) + (run / "manifest.json").write_text( + json.dumps( + { + "schema_version": "v1alpha1", + "run_id": "r1", + "seed": 7, + "adapter": {"id": "openshell-docker"}, + "environment": {"system": {"openshell": "1.2.3"}}, + } + ) + ) + (run / "verdict.json").write_text( + json.dumps({"verdict": "PASS", "errors": [], "results": []}) + ) + (run / "experiment.yaml").write_text( + "id: demo\ntitle: Demo \nhypothesis: |\n It\n holds.\n" + ) + metrics = run / "system" / "metrics" + jsonl( + metrics / "gateway.jsonl", + [ + { + "t": 0.0, + "up": 1, + "samples": { + 'req_total{m="a"}': 10, + 'req_total{m="b"}': 0, + "sessions": 1, + "lat_count": 0, + "lat_sum": 0, + 'lat_bucket{le="1"}': 0, + "idle": 0, + }, + }, + { + "t": 2.0, + "up": 1, + "samples": { + 'req_total{m="a"}': 14, + 'req_total{m="b"}': 2, + "sessions": 2, + "lat_count": 4, + "lat_sum": 2, + 'lat_bucket{le="1"}': 4, + "idle": 0, + }, + }, + {"t": 4.0, "up": 0, "error": "refused"}, + {"t": 6.0, "up": 0, "error": "refused"}, + { + "t": 8.0, + "up": 1, + "samples": { + 'req_total{m="a"}': 1, + "sessions": 1, + "lat_count": 6, + "lat_sum": 3, + "idle": 0, + }, + }, + ], + ) + (metrics / "gateway.types.json").write_text( + json.dumps( + { + "req_total": "counter", + "sessions": "gauge", + "lat": "histogram", + "idle": "gauge", + } + ) + ) + line = { + "Action": "die", + "Actor": { + "Attributes": { + "openshell.ai/sandbox-id": "sb-0", + "openshell.ai/isolation-role": "supervisor", + "exitCode": "137", + } + }, + } + other = { + "Action": "die", + "Actor": {"Attributes": {"openshell.ai/sandbox-id": "elsewhere"}}, + } + jsonl( + run / "system" / "watch" / "docker-events.jsonl", + [ + {"t": 10.1, "line": json.dumps(line)}, + {"t": 10.2, "line": json.dumps(other)}, + {"t": 10.3, "line": "not json"}, + ], + ) + return run + + +def test_lanes_faults_and_marks(tmp_path): + data = build(run_dir(tmp_path)) + (lane,) = data["lanes"] + assert [(s["state"], s["from"], s["to"]) for s in lane["segments"]] == [ + ("Created", 1.0, 1.5), + ("Running", 1.5, 12.0), + ("Succeeded", 12.0, 13.0), + ] + labels = [m["label"] for m in lane["marks"]] + assert "supervisor die (exit 137)" in labels + assert len(labels) == 3 # collected, deleted, one docker event for this run + assert [(f["name"], f["from"], f["to"], f["ended_by"]) for f in data["faults"]] == [ + ("gateway-down", 4.0, 9.0, "operation.reverted"), + ("kill-one", 10.0, 10.0, "operation.applied"), + ] + assert data["experiment"]["hypothesis"] == "It holds." + assert [e["t"] for e in data["events"]] == [0.0, 14.0] + + +def test_metric_charts(tmp_path): + data = build(run_dir(tmp_path)) + charts = {c["title"]: c for c in data["charts"]} + assert set(charts) == { + "req_total", + "sessions", + "lat (observations)", + "lat (mean per observation)", + } + # Counter rate: none at the first scrape, (16 - 10) / 2s, a gap while + # down, then a reset (16 -> 1) read as 1 since the last answer at t=2. + rates = charts["req_total"]["points"] + assert rates == [[0.0, None], [2.0, 3.0], [4.0, None], [6.0, None], [8.0, 0.166667]] + assert charts["req_total"]["subtitle"] == "per second, sum of 2 series" + assert charts["sessions"]["points"][0] == [0.0, 1.0] + assert charts["lat (mean per observation)"]["points"] == [ + [0.0, None], + [2.0, 0.5], + [4.0, None], + [6.0, None], + [8.0, None], + ] + assert data["down"] == [{"source": "gateway", "from": 4.0, "to": 8.0}] + + +def test_listed_metrics_come_first_and_keep_zero_charts(tmp_path): + run = run_dir(tmp_path) + (run / "experiment.yaml").write_text( + "id: demo\ntitle: Demo\nhypothesis: h\n" + "metrics:\n" + " - family: lat\n description: |\n Why it\n matters.\n" + " - sessions\n" + " - family: absent_total\n" + " - {title: Up, expr: 'sum(up)'}\n" + ) + metrics = run / "system" / "metrics" + scrapes = [ + json.loads(line) + for line in (metrics / "gateway.jsonl").read_text().splitlines() + ] + for s in scrapes: + if s["up"]: + s["samples"]["zero_total"] = 0 + jsonl(metrics / "gateway.jsonl", scrapes) + (metrics / "gateway.types.json").write_text( + json.dumps( + {"req_total": "counter", "lat": "histogram", "zero_total": "counter"} + ) + ) + data = build(run) + assert data["metrics"] == [ + {"family": "lat", "description": "Why it matters."}, + {"family": "sessions", "description": ""}, + {"family": "absent_total", "description": ""}, + ] + assert [(c["title"], c["declared"]) for c in data["charts"]] == [ + ("lat (observations)", True), + ("lat (mean per observation)", True), + ("sessions", True), + ("req_total", False), + ] + # An all-zero family is charted only when it is listed. + (run / "experiment.yaml").write_text( + "id: demo\ntitle: Demo\nhypothesis: h\nmetrics: [zero_total]\n" + ) + data = build(run) + assert [(c["title"], c["declared"]) for c in data["charts"]][:1] == [ + ("zero_total", True) + ] + assert not any(v for _, v in data["charts"][0]["points"]) + + +def test_summary_quantiles_not_charted(tmp_path): + run = run_dir(tmp_path) + metrics = run / "system" / "metrics" + jsonl( + metrics / "gateway.jsonl", + [ + { + "t": float(t), + "up": 1, + "samples": { + 'dur{quantile="0.5"}': 0.1, + 'dur{quantile="0.99"}': 0.9, + "dur_count": 2 * t, + "dur_sum": t, + }, + } + for t in range(3) + ], + ) + (metrics / "gateway.types.json").write_text(json.dumps({"dur": "summary"})) + titles = [c["title"] for c in build(run)["charts"]] + assert titles == ["dur (observations)", "dur (mean per observation)"] + + +def test_run_details_table(tmp_path): + run = run_dir(tmp_path) + manifest = json.loads((run / "manifest.json").read_text()) + manifest["experiment"] = {"id": "demo", "version": 2} + manifest["started_at"] = "2026-10-27T00:00:00.123456Z" + manifest["adapter"]["backend"] = "docker" + manifest["environment"]["system"] = { + "openshell": "openshell 1.2.3", + "docker": "29.8.2", + "gateway_sha256": "ab" * 32, + "git_sha": "c0ffee", + "git_dirty": True, + "images": {"supervisor_image": "openshell/supervisor:dev@sha256:00"}, + } + (run / "manifest.json").write_text(json.dumps(manifest)) + (run / "experiment.yaml").write_text( + "id: demo\noperations:\n" + "- {name: gateway-down, uses: outage, target: gateway, select: one}\n" + "- {name: crash, uses: kill-all, scope: global}\n" + ) + # Under `//` the directory names both. + copy = tmp_path / "runs" / "demo" / "r1" + shutil.copytree(run, copy) + rows = build(copy)["details"] + table = [ + (r["key"], r["value"], r["note"], r["code"]) for r in rows if not r["more"] + ] + assert table == [ + ("Experiment", "demo", "version 2", True), + ("Run", "r1", "", True), + ("Run directory", "demo/r1", "", True), + ("Started", "2026-10-27 00:00:00 UTC", "", False), + ("Length", "14s", "", False), + ( + "Seed", + "7", + "picks the instances that gateway-down acts on; " + "the same seed and instance counts pick the same ones", + True, + ), + ("Adapter", "openshell-docker", "docker backend", True), + ("OpenShell", "1.2.3", "", True), + ("Docker", "29.8.2", "", True), + ("Git", "c0ffee", "dirty", True), + ] + # Digests and image refs are folded under the table. + assert [(r["key"], r["value"]) for r in rows if r["more"]] == [ + ("Gateway sha256", "ab" * 32), + ("Images: supervisor image", "openshell/supervisor:dev@sha256:00"), + ] + # A run dir copied elsewhere shows its own name only. + assert build(run)["details"][2]["value"] == "run" + # The page draws the table from the embedded rows. + html = write_report(copy, tmp_path / "page.html").read_text() + data = html.split('")[0])["details"] == rows + assert 'html("table", "", "details")' in html + + +def test_seed_note_without_selecting_operations(tmp_path): + rows = {r["key"]: r for r in build(run_dir(tmp_path))["details"]} + assert rows["Seed"]["note"] == ( + "no operation selects instances; scripts get it as CHAOS_EXPERIMENT_RUN_SEED" + ) + assert "Started" not in rows and "Git" not in rows + + +def test_write_report_embeds_data_safely(tmp_path): + run = run_dir(tmp_path) + hostile = "--> & \u2028\u2029 \u00e9" + (run / "experiment.yaml").write_text( + f"id: demo\ntitle: {json.dumps(hostile)}\nhypothesis: {json.dumps(hostile)}\n" + ) + path = write_report(run) + html = path.read_text() + assert path.name == "report.html" + assert "__DATA__" not in html and "__TITLE__" not in html + assert "demo r1" in html + # The data block is the only script that closes, no HTML tag or comment + # delimiter from the data reaches the parser, and the data round-trips. + assert html.count("") == 2 + data = html.split('" + )[0] + assert not any(c in data for c in "<>&\u2028\u2029") + assert json.loads(data)["experiment"]["title"] == hostile + assert json.loads(data)["experiment"]["hypothesis"] == " ".join(hostile.split()) + + +def test_export_openmetrics(tmp_path): + out = tmp_path / "run.om" + directory = run_dir(tmp_path) + result = {"scope": "workload", "workload": "counter", "instance": "counter-0"} + (directory / "verdict.json").write_text( + json.dumps( + { + "verdict": "FAIL", + "errors": [], + "results": [ + {**result, "check": "a", "ok": True}, + {**result, "check": "b", "ok": False}, + ], + } + ) + ) + run = export(directory, out) + assert (run.run_id, run.experiment) == ("r1", "demo") + assert run.end - run.start == 14.0 + lines = out.read_text().splitlines() + assert lines[-1] == "# EOF" + names = [ln.split("{")[0] for ln in lines if not ln.startswith("#")] + # Each family is contiguous, as OpenMetrics requires. + runs = [name for name, _ in itertools.groupby(names)] + assert len(runs) == len(set(runs)) + raw = [ln for ln in lines if ln.startswith("req_total")] + assert raw[0].startswith( + 'req_total{m="a",chaos_run_id="r1",chaos_source="gateway"} 10 ' + ) + # Gaps are NaN, so Prometheus does not hold the last value across them. + rate = [ln.split()[-2] for ln in lines if 'title="req_total"' in ln] + assert rate == ["NaN", "3.0", "NaN", "NaN", "0.166667", "NaN"] + state = [ln for ln in lines if ln.startswith("chaos_instance_state")] + assert {ln.split()[1] for ln in state} == {"0", "2", "4"} + fault = [ln for ln in lines if 'operation="gateway-down"' in ln] + assert fault[-1].split()[1] == "0" + checks = { + ln.split('check="')[1].split('"')[0]: ln.split()[1] + for ln in lines + if ln.startswith("chaos_check_result") + } + assert checks == {"a": "1", "b": "0"} + assert 'family="lat",title="lat (observations)"' in out.read_text() + (die,) = [ln for ln in lines if "exit 137" in ln] + value, ts = die.split()[-2:] + assert int(value) == round(float(ts) * 1000) + + +def test_fault_windows_end_per_operation(): + from chaos.report import faults + + def ev(t, event, op, uses): + return {"t": t, "event": event, "operation": op, "uses": uses} + + events = [ + ev(1.0, "operation.applied", "bounce", "restart"), + # A revert of another operation does not end the restart window. + ev(2.0, "operation.reverted", "bounce", "restart"), + ev(3.0, "target.restarted", "bounce", "restart"), + ev(4.0, "operation.applied", "in-place", "restart-in-place"), + ev(5.0, "operation.applied", "ice", "freeze"), + ev(6.0, "operation.applied", "down", "outage"), + ev(7.0, "operation.error", "down", "outage"), + ] + assert [ + (f["name"], f["from"], f["to"], f["ended_by"]) for f in faults(events, 9.0) + ] == [ + ("bounce", 1.0, 3.0, "target.restarted"), + ("in-place", 4.0, 4.0, "operation.applied"), + ("ice", 5.0, 9.0, None), + ("down", 6.0, 7.0, "operation.error"), + ] diff --git a/e2e/chaos/tests/test_runner.py b/e2e/chaos/tests/test_runner.py new file mode 100644 index 0000000000..481774f4bf --- /dev/null +++ b/e2e/chaos/tests/test_runner.py @@ -0,0 +1,873 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Whole runs against a fake docker adapter: bash hooks, no Docker, no gateway.""" + +from __future__ import annotations + +import asyncio +import contextlib +import hashlib +import json +import os +import signal +import time +from pathlib import Path + +import pytest +import yaml +from conftest import minimal_adapter + +from chaos import cli, flow, instances, runner +from chaos.adapter import run_shell +from chaos.evaluate import read_verdict +from chaos.inject.docker import DockerInjector +from chaos.runner import Run + +HOOKS = { + # Runs the workload on the host, writing to the instance's output dir. + "create": """ +name=$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_NAME +echo "$name" >> "$FAKE_STATE/created" +[ "${FAKE_CREATE_FAIL:-}" = "$name" ] && exit 1 +while IFS= read -r line; do export "${line?}"; done < "$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_ENV_FILE" +export CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR="$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_DIR/output" +echo "$name ${CHAOS_EXPERIMENT_WORKLOAD_OPTION_POLICY:-}" >> "$FAKE_STATE/options" +if [ -n "${FAKE_BACKGROUND:-}" ]; then + sh -c "$(cat "$CHAOS_EXPERIMENT_WORKLOAD_WRAPPER_FILE")" chaos \\ + "$(cat "$CHAOS_EXPERIMENT_WORKLOAD_RUN_FILE")" > /dev/null 2>&1 & +else + sh -c "$(cat "$CHAOS_EXPERIMENT_WORKLOAD_WRAPPER_FILE")" chaos \\ + "$(cat "$CHAOS_EXPERIMENT_WORKLOAD_RUN_FILE")" > /dev/null 2>&1 +fi +echo "id-$name" +""", + "status": """ +out=$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_DIR/output/.chaos +if [ -f "$out/exit_code" ]; then echo Succeeded +elif [ -f "$out/attempts" ]; then echo Running +else echo Pending; fi +""", + # Records the call, then releases each instance it applies to. + "openshell-policy-set": """ +case "${FAKE_OPERATION_FAIL:-}" in + 1|"$CHAOS_EXPERIMENT_OPERATION_NAME") echo "policy rejected" >&2; exit 3;; +esac +cp "$CHAOS_EXPERIMENT_OPERATION_OPTION_POLICY" "$FAKE_STATE/applied-policy.yaml" +echo "$CHAOS_EXPERIMENT_OPERATION_SCOPE" > "$CHAOS_EXPERIMENT_OPERATION_STATE_DIR/scope" +echo ok > "$CHAOS_EXPERIMENT_OPERATION_OUTCOME_FILE" +release() { + echo "$CHAOS_EXPERIMENT_OPERATION_NAME $CHAOS_EXPERIMENT_OPERATION_SCOPE $1" >> "$FAKE_STATE/operations" + touch "$CHAOS_EXPERIMENT_RUN_DIR/workloads/w/instances/$1/output/go" +} +if [ "$CHAOS_EXPERIMENT_OPERATION_SCOPE" = instances ]; then + release "$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_NAME" +else + while IFS= read -r name; do release "$name"; done < "$CHAOS_EXPERIMENT_OPERATION_INSTANCES_FILE" +fi +""", + "collect": """ +name=$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_NAME +echo "$name $CHAOS_EXPERIMENT_HOOK_TIMEOUT" >> "$FAKE_STATE/collected" +[ "${FAKE_COLLECT_FAIL:-}" = "$name" ] && exit 1 +[ -n "${FAKE_COLLECT_SLOW:-}" ] && sleep 30 +true +""", + "delete": """ +[ "${FAKE_DELETE_FAIL:-}" = "$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_NAME" ] && exit 5 +echo "$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_NAME" >> "$FAKE_STATE/deleted" +""", + "start": """ +nohup sleep 300 > /dev/null 2>&1 & +echo $! > "$FAKE_STATE/gateway.pid" +""", +} + +EXPERIMENT = { + "apiVersion": "chaos/v1alpha1", + "id": "fake", + "version": 1, + "title": "t", + "hypothesis": "h", + "description": "d", + "default-adapter": "openshell-docker", + "workloads": [ + { + "name": "w", + "description": "d", + "kind": "shell", + "instances": 2, + "timeout": "1m", + "run": 'echo ok > "$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR/result"', + "checks": [ + {"name": "ok", "description": "d", "kind": "exit-code", "equals": 0} + ], + } + ], + "flow": "w", +} + + +@pytest.fixture +def fake(tmp_path, monkeypatch): + """A fake docker adapter. Returns the state dir that hooks record into.""" + state = tmp_path / "state" + state.mkdir() + adapters = tmp_path / "adapters" + adapters.mkdir() + adapter = minimal_adapter(prelude="set -u\n") + adapter["instance"] = { + **adapter["instance"], + **{k: HOOKS[k] for k in ("create", "status", "collect", "delete")}, + } + adapter["targets"]["gateway"] = { + "type": "process", + "pid-file": "${FAKE_STATE}/gateway.pid", + "start": HOOKS["start"], + } + (adapters / "openshell-docker.yaml").write_text(yaml.safe_dump(adapter)) + (tmp_path / "fake.yaml").write_text(yaml.safe_dump(EXPERIMENT)) + monkeypatch.setenv("FAKE_STATE", str(state)) + monkeypatch.setattr(instances, "POLL_INTERVAL", 0.05) + monkeypatch.setattr(flow, "POLL_INTERVAL", 0.05) + monkeypatch.setattr(instances, "RETRY_WINDOW", 0.5) + yield state + pid_file = state / "gateway.pid" + if pid_file.exists(): + with contextlib.suppress(ProcessLookupError): + os.kill(int(pid_file.read_text()), signal.SIGKILL) + + +def run(fake: Path) -> tuple[int, dict]: + root = fake.parent + code = cli.main( + [ + "--adapters-dir", + str(root / "adapters"), + "run", + str(root / "fake.yaml"), + "--runs-dir", + str(root / "runs"), + ] + ) + (run_dir,) = (root / "runs" / "fake").iterdir() + return code, json.loads((run_dir / "verdict.json").read_text()) + + +def lines(path: Path) -> list[str]: + return sorted(path.read_text().split()) if path.exists() else [] + + +def index(fake: Path) -> list[str]: + path = fake.parent / "runs" / "index.jsonl" + return [json.loads(line)["verdict"] for line in path.read_text().splitlines()] + + +def test_run_passes_and_deletes_every_instance(fake): + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (0, "PASS"), verdict + assert len(verdict["results"]) == 2 + assert lines(fake / "deleted") == lines(fake / "created") == ["w-0", "w-1"] + assert index(fake) == ["PASS"] + # The run's checks are on the timeline; no gateway fault ran, so the + # `run.finished` event names no processes. + checks = [(e["event"], e["instance"], e.get("ok")) for e in _events(fake, "check.")] + assert checks == [ + ("check.started", "w-0", None), + ("check.finished", "w-0", True), + ("check.started", "w-1", None), + ("check.finished", "w-1", True), + ] + (finished,) = _events(fake, "run.finished") + assert finished == { + "t": finished["t"], + "ts": finished["ts"], + "event": "run.finished", + "verdict": "PASS", + } + (path,) = (fake.parent / "runs" / "fake").iterdir() + (line,) = _index_lines(fake) + assert (line["path"], line["flaky_missed"]) == (f"fake/{path.name}", []) + + +def _index_lines(fake: Path) -> list[dict]: + path = fake.parent / "runs" / "index.jsonl" + return [json.loads(line) for line in path.read_text().splitlines()] + + +def test_state_reason_comes_from_the_status_hook(fake, monkeypatch): + """A second word on the status hook's line is the state's reason.""" + adapters = fake.parent / "adapters" + path = adapters / "openshell-docker.yaml" + adapter = yaml.safe_load(path.read_text()) + adapter["instance"]["status"] = HOOKS["status"].replace( + "echo Succeeded", 'echo "Succeeded MainProcessExited"' + ) + path.write_text(yaml.safe_dump(adapter)) + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (0, "PASS"), verdict + states = [ + (e["instance"], e["state"], e.get("reason")) + for e in _events(fake, "instance.state") + ] + assert ("w-0", "Succeeded", "MainProcessExited") in states + assert all(reason is None for _, state, reason in states if state != "Succeeded") + + +def test_unparsed_status_is_logged_once_per_instance(fake): + """A status hook output that is no state reads as Unknown and is logged + the first time, with the output, so a hook bug is visible without a + debugger.""" + adapters = fake.parent / "adapters" + path = adapters / "openshell-docker.yaml" + adapter = yaml.safe_load(path.read_text()) + adapter["instance"]["status"] = ( + "polls=$FAKE_STATE/polls-$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_NAME\n" + 'n=$(cat "$polls" 2>/dev/null || echo 0); echo $((n + 1)) > "$polls"\n' + 'if [ "$n" -lt 2 ]; then echo "Bogus state"; exit 0; fi\n' + ) + HOOKS["status"] + path.write_text(yaml.safe_dump(adapter)) + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (0, "PASS"), verdict + log = next((fake.parent / "runs" / "fake").glob("*/logs/harness.log")) + lines = [line for line in log.read_text().splitlines() if "read as Unknown" in line] + assert len(lines) == 2, lines + assert sorted( + line.split("status hook for ")[1].split(" ")[0] for line in lines + ) == [ + "w-0", + "w-1", + ] + assert all("stdout 'Bogus state'" in line for line in lines), lines + + +def test_observers_run_beside_the_flow(fake): + path = fake.parent / "adapters" / "openshell-docker.yaml" + adapter = yaml.safe_load(path.read_text()) + (fake / "metrics.txt").write_text("up_gauge 1\n") + adapter["metrics"] = [ + {"name": "m", "url": "file://${FAKE_STATE}/metrics.txt", "interval": "50ms"} + ] + adapter["watch"] = [{"name": "w", "run": "echo started; sleep 60"}] + path.write_text(yaml.safe_dump(adapter)) + code, _ = run(fake) + assert code == 0 + (run_dir,) = (fake.parent / "runs" / "fake").iterdir() + scrapes = (run_dir / "system" / "metrics" / "m.jsonl").read_text().splitlines() + assert json.loads(scrapes[0])["samples"] == {"up_gauge": 1.0} + (line,) = (run_dir / "system" / "watch" / "w.jsonl").read_text().splitlines() + assert json.loads(line)["line"] == "started" + assert '"title": "up_gauge"' in (run_dir / "report.html").read_text() + + +def _with_metrics(fake: Path, families: list[str], text: str | None = "up_gauge 1\n"): + """The experiment lists `families`; the adapter scrapes a file with `text` + (no metrics source when None).""" + path = fake.parent / "adapters" / "openshell-docker.yaml" + adapter = yaml.safe_load(path.read_text()) + if text is not None: + (fake / "metrics.txt").write_text(text) + adapter["metrics"] = [ + {"name": "m", "url": "file://${FAKE_STATE}/metrics.txt", "interval": "50ms"} + ] + path.write_text(yaml.safe_dump(adapter)) + (fake.parent / "fake.yaml").write_text( + yaml.safe_dump({**EXPERIMENT, "metrics": families}) + ) + + +def test_listed_metrics_must_be_exported(fake): + _with_metrics( + fake, + ["up_gauge", "lat_seconds", "gone_total"], + "up_gauge 1\nlat_seconds_count 2\n", + ) + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (2, "ERROR") + assert verdict["errors"] == ["preflight failed: metrics-exported"] + (result,) = verdict["results"] + assert (result["scope"], result["check"], result["ok"]) == ( + "preflight", + "metrics-exported", + False, + ) + assert result["output"] == "m: 2 families, 2 listed\nmissing: gone_total" + assert not (fake / "created").exists() + + +def test_listed_metrics_without_a_source_fail_preflight(fake): + _with_metrics(fake, ["up_gauge"], None) + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (2, "ERROR") + assert verdict["results"][0]["output"] == ( + "adapter openshell-docker has no metrics sources to scrape\nmissing: up_gauge" + ) + + +def test_listed_metrics_are_checked_and_reported(fake): + _with_metrics(fake, ["up_gauge", "zero_total"], "up_gauge 1\nzero_total 0\n") + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (0, "PASS"), verdict + assert [(r["scope"], r["check"]) for r in verdict["results"]][:1] == [ + ("preflight", "metrics-exported") + ] + (run_dir,) = (fake.parent / "runs" / "fake").iterdir() + report = (run_dir / "report.html").read_text() + # A listed family is charted even when it stays at zero. + assert '"title": "zero_total"' in report + assert '"declared": true' in report + + +def test_failed_create_is_still_deleted(fake, monkeypatch): + monkeypatch.setenv("FAKE_CREATE_FAIL", "w-1") + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (2, "ERROR") + assert any("create w-1 failed" in e for e in verdict["errors"]) + assert "w-1" in lines(fake / "deleted") + assert lines(fake / "deleted") == lines(fake / "created") + + +POLICY = {"version": 1, "network_policies": {}} +STARTED = {"wait": {"workload": "w", "condition": "started"}} + + +def _with_operation(fake: Path, monkeypatch, operation: dict) -> None: + """Instances that wait for the operation to release them, then exit 0.""" + root = fake.parent + path = root / "adapters" / "openshell-docker.yaml" + adapter = yaml.safe_load(path.read_text()) + adapter["options"] = {"policy": {"default": "", "data": True}} + adapter["operations"] = { + "openshell-policy-set": { + "run": HOOKS["openshell-policy-set"], + # Records what it was asked to put back and what `run` left behind. + "revert": 'echo "$CHAOS_EXPERIMENT_OPERATION_NAME ' + "$CHAOS_EXPERIMENT_OPERATION_SCOPE " + "${CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_NAME:-} " + '$(cat "$CHAOS_EXPERIMENT_OPERATION_STATE_DIR/scope")" ' + '>> "$FAKE_STATE/reverted"\n' + '[ -z "${FAKE_REVERT_FAIL:-}" ] || { echo "cannot" >&2; exit 4; }\n', + "options": {"policy": {"data": True, "required": True}}, + } + } + path.write_text(yaml.safe_dump(adapter)) + exp = json.loads(json.dumps(EXPERIMENT)) + exp["data"] = {"p": POLICY, "note": "free text\n", "limit": 3} + exp["operations"] = [ + { + "name": "a", + "description": "d", + "uses": "openshell-policy-set", + "options": {"policy": "p"}, + **operation, + } + ] + exp["workloads"][0]["options"] = {"policy": "p"} + exp["workloads"][0]["run"] = ( + 'i=0; until [ -f "$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR/go" ]; do ' + "i=$((i + 1)); [ $i -gt 100 ] && exit 1; sleep 0.05; done" + ) + exp["flow"] = {"parallel": ["w", {"serial": [STARTED, "a"]}]} + (root / "fake.yaml").write_text(yaml.safe_dump(exp)) + monkeypatch.setenv("FAKE_BACKGROUND", "1") + + +def _events(fake: Path, prefix: str) -> list[dict]: + (run_dir,) = (fake.parent / "runs" / "fake").iterdir() + timeline = (run_dir / "timeline.jsonl").read_text().splitlines() + return [e for e in map(json.loads, timeline) if e["event"].startswith(prefix)] + + +def test_targeted_operation_runs_once_per_instance(fake, monkeypatch): + _with_operation(fake, monkeypatch, {"target": {"workload": "w"}, "select": "all"}) + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (0, "PASS"), verdict + assert sorted((fake / "operations").read_text().splitlines()) == [ + "a instances w-0", + "a instances w-1", + ] + assert yaml.safe_load((fake / "applied-policy.yaml").read_text()) == POLICY + # The workload option resolves to the data file the runner wrote. + for line in (fake / "options").read_text().splitlines(): + assert line.split()[1].endswith("/data/p") + (run_dir,) = (fake.parent / "runs" / "fake").iterdir() + assert (run_dir / "data" / "note").read_text() == "free text\n" + assert (run_dir / "data" / "limit").read_text() == "3\n" + started, *rest = _events(fake, "operation.") + assert started["event"] == "operation.started" + assert started["options"] == {"policy": "p"} + assert (started["scope"], started["instances"]) == ("instances", ["w-0", "w-1"]) + completed = [e for e in rest if e["event"] == "operation.completed"] + assert sorted((e["instance"], e["outcome"]) for e in completed) == [ + ("w-0", "ok"), + ("w-1", "ok"), + ] + # The instances exited and were deleted before the end of the flow, so + # there is nothing to put the policy back on: each revert is skipped. + reverted = [e for e in rest if e["event"] == "operation.reverted"] + assert sorted((e["abort"], e["skipped"]) for e in reverted) == [ + (False, "w-0 is deleted"), + (False, "w-1 is deleted"), + ] + assert not (fake / "reverted").exists() + + +def test_global_operation_runs_once_for_running_instances(fake, monkeypatch): + _with_operation(fake, monkeypatch, {"scope": "global"}) + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (0, "PASS"), verdict + assert sorted((fake / "operations").read_text().splitlines()) == [ + "a global w-0", + "a global w-1", + ] + started, completed, reverted = _events(fake, "operation.") + assert (started["scope"], started["instances"]) == ("global", ["w-0", "w-1"]) + assert (completed["event"], completed["outcome"]) == ("operation.completed", "ok") + assert (reverted["event"], reverted["abort"]) == ("operation.reverted", False) + assert reverted["undo"] == "revert a (run 1)" + assert (fake / "reverted").read_text() == "a global global\n" + + +def test_operation_hook_must_report_an_outcome(fake, monkeypatch): + _with_operation(fake, monkeypatch, {"scope": "global"}) + path = fake.parent / "adapters" / "openshell-docker.yaml" + adapter = yaml.safe_load(path.read_text()) + adapter["operations"]["openshell-policy-set"]["run"] = "true\n" + path.write_text(yaml.safe_dump(adapter)) + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (2, "ERROR") + assert any("a: hook wrote no outcome" in e for e in verdict["errors"]), verdict + assert not (fake / "reverted").exists() + + +def test_operations_are_reverted_on_abort_while_instances_are_up(fake, monkeypatch): + """`a` is applied to each instance, then `b` fails: the abort reverts + `a` before the instances are collected and deleted.""" + _with_operation(fake, monkeypatch, {"target": {"workload": "w"}, "select": "all"}) + exp = yaml.safe_load((fake.parent / "fake.yaml").read_text()) + exp["operations"].append({**exp["operations"][0], "name": "b", "scope": "global"}) + del exp["operations"][1]["target"], exp["operations"][1]["select"] + exp["flow"] = {"parallel": ["w", {"serial": [STARTED, "a", "b"]}]} + # Released by `a`, the instances then wait for a second release that + # never comes, so they are still up when `b` fails. + exp["workloads"][0]["run"] += ( + '; until [ -f "$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR/go2" ]; do ' + "i=$((i + 1)); [ $i -gt 100 ] && exit 1; sleep 0.05; done" + ) + (fake.parent / "fake.yaml").write_text(yaml.safe_dump(exp)) + monkeypatch.setenv("FAKE_OPERATION_FAIL", "b") + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (2, "ERROR") + assert any("operation b: " in e for e in verdict["errors"]), verdict + reverted = _events(fake, "operation.reverted") + assert [(e["operation"], e["abort"], e.get("skipped")) for e in reverted] == [ + ("a", True, None), + ("a", True, None), + ] + assert sorted((fake / "reverted").read_text().splitlines()) == [ + "a instances w-0 instances", + "a instances w-1 instances", + ] + assert lines(fake / "deleted") == lines(fake / "created") == ["w-0", "w-1"] + + +def test_failed_revert_is_an_error(fake, monkeypatch): + _with_operation(fake, monkeypatch, {"scope": "global"}) + monkeypatch.setenv("FAKE_REVERT_FAIL", "1") + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (2, "ERROR") + assert any( + e == "undo a (revert a (run 1)): revert a (run 1): " + "operations.openshell-policy-set.revert exited 4: cannot" + for e in verdict["errors"] + ), verdict + assert [e["event"] for e in _events(fake, "operation.")] == [ + "operation.started", + "operation.completed", + "operation.error", + ] + assert lines(fake / "deleted") == lines(fake / "created") == ["w-0", "w-1"] + + +def test_failed_operation_is_an_error_verdict(fake, monkeypatch): + _with_operation(fake, monkeypatch, {"target": {"workload": "w"}, "select": "one"}) + monkeypatch.setenv("FAKE_OPERATION_FAIL", "1") + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (2, "ERROR") + assert any("exited 3: policy rejected" in e for e in verdict["errors"]), verdict + assert [e["event"] for e in _events(fake, "operation.")] == [ + "operation.started", + "operation.error", + ] + assert lines(fake / "deleted") == lines(fake / "created") == ["w-0", "w-1"] + + +def test_failed_collect_is_still_deleted(fake, monkeypatch): + monkeypatch.setenv("FAKE_COLLECT_FAIL", "w-0") + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (2, "ERROR") + assert any("collect w-0 failed" in e for e in verdict["errors"]), verdict + assert lines(fake / "deleted") == lines(fake / "created") == ["w-0", "w-1"] + + +def test_cleanup_is_bounded_and_hooks_get_the_budget(fake, monkeypatch): + _with_operation(fake, monkeypatch, {"target": {"workload": "w"}, "select": "one"}) + monkeypatch.setenv("FAKE_OPERATION_FAIL", "1") + monkeypatch.setenv("FAKE_COLLECT_SLOW", "1") + monkeypatch.setattr(runner, "CLEANUP_TIMEOUT", 1.0) + start = time.monotonic() + code, verdict = run(fake) + assert time.monotonic() - start < 10 + assert (code, verdict["verdict"]) == (2, "ERROR") + assert any(e.startswith("cleanup did not finish in 1s") for e in verdict["errors"]) + assert lines(fake / "deleted") == lines(fake / "created") == ["w-0", "w-1"] + # Each collect was told it had at most the 1s left of the cleanup. + collected = (fake / "collected").read_text().splitlines() + assert {line.split()[1] for line in collected} == {"1"} + + +def _with_hook(fake: Path, hook: str) -> None: + """Replaces the operation hook of the adapter `_with_operation` wrote. The + hook reports `ok`, as every hook must report something.""" + path = fake.parent / "adapters" / "openshell-docker.yaml" + adapter = yaml.safe_load(path.read_text()) + adapter["operations"]["openshell-policy-set"]["run"] = ( + hook + 'echo ok > "$CHAOS_EXPERIMENT_OPERATION_OUTCOME_FILE"\n' + ) + path.write_text(yaml.safe_dump(adapter)) + + +def test_repeat_runs_an_operation_each_iteration(fake, monkeypatch): + _with_operation(fake, monkeypatch, {"target": {"workload": "w"}, "select": "all"}) + path = fake.parent / "fake.yaml" + exp = yaml.safe_load(path.read_text()) + exp["workloads"][0]["run"] = ( + "out=$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR; i=0; " + 'until [ "$(wc -l < "$out/go" 2>/dev/null || echo 0)" -ge 3 ]; do ' + "i=$((i + 1)); [ $i -gt 200 ] && exit 1; sleep 0.05; done" + ) + exp["flow"] = { + "parallel": [ + "w", + { + "serial": [ + STARTED, + {"repeat": {"times": 3, "serial": ["a"]}}, + ] + }, + ] + } + path.write_text(yaml.safe_dump(exp)) + _with_hook( + fake, + 'echo "run" >> "$CHAOS_EXPERIMENT_RUN_DIR/workloads/w/instances/' + '$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_NAME/output/go"\n' + 'echo "applied $CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_NAME"\n', + ) + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (0, "PASS"), verdict + assert [e["iteration"] for e in _events(fake, "repeat.")] == [1, 2, 3] + completed = _events(fake, "operation.completed") + assert [e["run"] for e in completed if e["instance"] == "w-0"] == [1, 2, 3] + (run_dir,) = (fake.parent / "runs" / "fake").iterdir() + log = (run_dir / "logs" / "operations" / "a.w-0.log").read_text() + assert log == "".join(f"--- run {n}\napplied w-0\n" for n in (1, 2, 3)) + + +def test_held_instance_is_not_finished_on_a_terminal_state(fake, monkeypatch): + """An operation that makes the instance read as Succeeded for a while, as a + stop does, does not get it collected and deleted mid-operation.""" + _with_operation(fake, monkeypatch, {"target": {"workload": "w"}, "select": "all"}) + _with_hook( + fake, + "out=$CHAOS_EXPERIMENT_RUN_DIR/workloads/w/instances/" + "$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_NAME/output\n" + 'echo 0 > "$out/.chaos/exit_code"; sleep 0.5; rm "$out/.chaos/exit_code"\n' + 'touch "$out/go"\n', + ) + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (0, "PASS"), verdict + events = _events(fake, "") + names = [(e["event"], e.get("instance")) for e in events] + completed = names.index(("operation.completed", "w-0")) + # Seen Succeeded while held, deleted only after the operation finished. + assert names.index(("instance.state", "w-0")) < completed + assert any( + e["event"] == "instance.state" and e["state"] == "Succeeded" + for e in events[:completed] + ) + assert names.index(("instance.deleted", "w-0")) > completed + + +def _add_preflight(fake: Path, adapter_checks: list, exp_checks: list) -> None: + root = fake.parent + path = root / "adapters" / "openshell-docker.yaml" + adapter = yaml.safe_load(path.read_text()) + adapter["preflight"] = adapter_checks + path.write_text(yaml.safe_dump(adapter)) + (root / "fake.yaml").write_text( + yaml.safe_dump({**EXPERIMENT, "preflight": exp_checks}) + ) + + +def _check(name: str, run: str) -> dict: + return {"name": name, "description": "d", "kind": "shell", "run": run} + + +def test_failed_preflight_creates_nothing(fake): + _add_preflight( + fake, + [_check("up", "true"), _check("disk", "echo full; exit 1")], + [_check("mine", 'echo "$CHAOS_EXPERIMENT_CHECK_NAME"')], + ) + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (2, "ERROR") + assert verdict["errors"] == ["preflight failed: disk"] + assert [(r["scope"], r["check"], r["ok"]) for r in verdict["results"]] == [ + ("preflight", "up", True), + ("preflight", "disk", False), + ("preflight", "mine", True), + ] + assert not (fake / "created").exists() + (run_dir,) = (fake.parent / "runs" / "fake").iterdir() + events = [ + json.loads(line)["event"] + for line in (run_dir / "timeline.jsonl").read_text().splitlines() + ] + assert events == ["run.started", "preflight.failed", "run.finished"] + # Evaluating again keeps the preflight results and runs no other check. + adapters = str(fake.parent / "adapters") + assert cli.main(["--adapters-dir", adapters, "evaluate", str(run_dir)]) == 2 + assert read_verdict(run_dir)["results"] == verdict["results"] + + +def test_passed_preflight_is_part_of_the_verdict(fake): + _add_preflight(fake, [_check("up", "true")], []) + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (0, "PASS"), verdict + assert [r["scope"] for r in verdict["results"]] == [ + "preflight", + "workload", + "workload", + ] + + +def test_preflight_command(fake, capsys): + root = fake.parent + args = [ + "--adapters-dir", + str(root / "adapters"), + "preflight", + str(root / "fake.yaml"), + "--runs-dir", + str(root / "runs"), + ] + _add_preflight( + fake, [_check("up", 'test -f "$CHAOS_EXPERIMENT_RUN_DIR/experiment.yaml"')], [] + ) + assert cli.main(args) == 0 + assert "ok preflight/up" in capsys.readouterr().out + _add_preflight(fake, [_check("up", "exit 1")], []) + assert cli.main(args) == 2 + assert capsys.readouterr().out.splitlines()[-1] == "ERROR" + # Nothing is created and no run dir is left. + assert not (fake / "created").exists() + assert list((root / "runs").iterdir()) == [] + + +def test_describe_output_is_the_manifests_system(fake): + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (0, "PASS"), verdict + (run_dir,) = (fake.parent / "runs" / "fake").iterdir() + manifest = json.loads((run_dir / "manifest.json").read_text()) + assert manifest["schema_version"] == "v1alpha1" + assert manifest["environment"]["system"] == {"openshell": "test"} + assert set(manifest["environment"]) == {"harness", "os", "arch", "python", "system"} + archived = (run_dir / "experiment.yaml").read_bytes() + assert manifest["experiment"]["sha256"] == hashlib.sha256(archived).hexdigest() + + +@pytest.mark.parametrize( + ("describe", "error"), + [ + ("echo nope >&2; exit 3", "describe failed (exit 3): nope"), + ("echo not json", "describe printed no JSON object"), + ("echo '[1]'", "describe printed a JSON list, not an object"), + ], +) +def test_failed_describe_is_an_error_verdict(fake, describe, error): + path = fake.parent / "adapters" / "openshell-docker.yaml" + raw = yaml.safe_load(path.read_text()) + raw["describe"] = describe + path.write_text(yaml.safe_dump(raw)) + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (2, "ERROR") + assert any(error in e for e in verdict["errors"]), verdict["errors"] + assert lines(fake / "created") == [] + + +def test_harness_error_is_an_error_verdict(fake, monkeypatch): + async def broken(self): + raise OSError("disk full") + + monkeypatch.setattr(Run, "_collect_run", broken) + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (2, "ERROR") + assert any("harness error: OSError('disk full')" in e for e in verdict["errors"]) + assert index(fake) == ["ERROR"] + + +def test_forced_exit_reports_an_instance_it_could_not_delete(fake, monkeypatch): + async def broken(self): + raise OSError("disk full") + + monkeypatch.setattr(Run, "_collect_run", broken) + monkeypatch.setenv("FAKE_DELETE_FAIL", "w-1") + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (2, "ERROR") + assert any( + e.startswith("delete w-1 (id-w-1) failed (exit 5)") for e in verdict["errors"] + ), verdict + assert "cleanup incomplete: 1 instance(s) not deleted" in verdict["errors"] + assert lines(fake / "deleted") == ["w-0"] + deleted = _events(fake, "instance.deleted") + assert [e["instance"] for e in deleted] == ["w-0"] + + +def test_forced_restore_is_bounded(fake, monkeypatch): + async def broken(self): + raise OSError("disk full") + + async def stuck(self): + await asyncio.sleep(60) + return [] + + monkeypatch.setattr(Run, "_collect_run", broken) + monkeypatch.setattr(DockerInjector, "undo_all", stuck) + monkeypatch.setattr(runner, "FORCED_RESTORE_TIMEOUT", 0.2) + start = time.monotonic() + code, verdict = run(fake) + assert time.monotonic() - start < 10 + assert (code, verdict["verdict"]) == (2, "ERROR") + assert any( + e.startswith("restore operations: TimeoutError") for e in verdict["errors"] + ) + assert lines(fake / "deleted") == lines(fake / "created") + + +def _alive(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + return True + + +def test_hook_timeout_kills_its_children(tmp_path): + pid_file = tmp_path / "pid" + start = time.monotonic() + result = asyncio.run( + run_shell(["bash", "-c", f"sleep 30 & echo $! > {pid_file}; wait"], {}, 0.5) + ) + assert result.code == 124 + assert time.monotonic() - start < 5 + assert not _alive(int(pid_file.read_text())) + + +def test_cancelled_hook_kills_its_children(tmp_path): + pid_file = tmp_path / "pid" + + async def cancel_hook() -> None: + task = asyncio.create_task( + run_shell(["bash", "-c", f"sleep 30 & echo $! > {pid_file}; wait"], {}, 60) + ) + while not pid_file.exists() or not pid_file.read_text().strip(): + await asyncio.sleep(0.02) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + asyncio.run(cancel_hook()) + assert not _alive(int(pid_file.read_text())) + + +def _with_signal_wait(fake: Path, monkeypatch, watch: str, wait: dict) -> None: + """An operation after two signal waits in a repeat, on a `sweep` signal.""" + _with_operation(fake, monkeypatch, {"target": {"workload": "w"}, "select": "all"}) + path = fake.parent / "adapters" / "openshell-docker.yaml" + adapter = yaml.safe_load(path.read_text()) + adapter["watch"] = [{"name": "otel", "run": watch}] + adapter["signals"] = [ + {"name": "sweep", "watch": "otel", "match": "^Name +: sweep$"} + ] + path.write_text(yaml.safe_dump(adapter)) + exp_path = fake.parent / "fake.yaml" + exp = yaml.safe_load(exp_path.read_text()) + exp["workloads"][0]["run"] = ( + "out=$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_OUTPUT_DIR; i=0; " + 'until [ "$(wc -l < "$out/go" 2>/dev/null || echo 0)" -ge 2 ]; do ' + "i=$((i + 1)); [ $i -gt 200 ] && exit 1; sleep 0.05; done" + ) + exp["flow"] = { + "parallel": [ + "w", + { + "serial": [ + STARTED, + {"repeat": {"times": 2, "serial": [{"wait": wait}, "a"]}}, + ] + }, + ] + } + exp_path.write_text(yaml.safe_dump(exp)) + _with_hook( + fake, + 'echo "run" >> "$CHAOS_EXPERIMENT_RUN_DIR/workloads/w/instances/' + '$CHAOS_EXPERIMENT_WORKLOAD_INSTANCE_NAME/output/go"\n', + ) + + +def test_signal_wait_returns_after_each_new_event(fake, monkeypatch): + _with_signal_wait( + fake, + monkeypatch, + "while true; do echo 'Name : sweep'; echo other; sleep 0.3; done", + {"signal": "sweep", "delay": "100ms", "timeout": "5s"}, + ) + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (0, "PASS"), verdict + waited = _events(fake, "signal.waited") + assert len(waited) == 2 + assert waited[0]["signal_t"] < waited[1]["signal_t"] + # Timeline times are rounded to the millisecond. + for e in waited: + assert e["t"] >= e["signal_t"] + 0.1 - 0.001 + received = _events(fake, "signal.received") + assert received and all(e["signal"] == "sweep" for e in received) + + +def test_signal_wait_timeout_is_an_error(fake, monkeypatch): + _with_signal_wait( + fake, monkeypatch, "sleep 60", {"signal": "sweep", "timeout": "300ms"} + ) + code, verdict = run(fake) + assert (code, verdict["verdict"]) == (2, "ERROR"), verdict + assert any("no sweep signal within 300ms" in e for e in verdict["errors"]) + (timeout,) = _events(fake, "signal.timeout") + assert (timeout["signal"], timeout["timeout"], timeout["last_t"]) == ( + "sweep", + "300ms", + None, + ) + # The workload the wait ran beside was cancelled, and its node says so. + finished = {e["node"]: e.get("status") for e in _events(fake, "node.finished")} + assert finished["flow.parallel[0]"] == "cancelled" + assert "flow.parallel[1].serial[1].repeat[0].wait" not in finished + + +def test_signal_must_exist_in_the_adapter(fake, monkeypatch, capsys): + _with_signal_wait(fake, monkeypatch, "sleep 60", {"signal": "other"}) + argv = ["--adapters-dir", str(fake.parent / "adapters"), "validate"] + code = cli.main([*argv, str(fake.parent / "fake.yaml")]) + assert code != 0 + assert "has no signal 'other'" in capsys.readouterr().err diff --git a/e2e/chaos/tests/test_select.py b/e2e/chaos/tests/test_select.py new file mode 100644 index 0000000000..c6e831f92b --- /dev/null +++ b/e2e/chaos/tests/test_select.py @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import random + +import pytest + +from chaos.inject import Candidate, select +from chaos.model import Operation, RunError + +CANDIDATES = [ + Candidate(f"counter-{i}", i, "supervisor", f"cid{9 - i}") for i in range(6) +] + + +def fault(select_: str | dict, name: str = "f") -> Operation: + return Operation( + name=name, + description="d", + uses="kill", + target={"workload": "counter", "role": "supervisor"}, + select=select_, + ) + + +def names(picks: list[Candidate]) -> list[str]: + return [c.instance for c in picks] + + +@pytest.mark.parametrize("select_", ["one", {"count": 3}]) +def test_same_seed_same_picks_in_any_order(select_): + expected = names(select(fault(select_), 42, CANDIDATES)) + for shuffle_seed in range(10): + shuffled = CANDIDATES[:] + random.Random(shuffle_seed).shuffle(shuffled) + # New container ids, as in a second run: picks follow instance names. + renamed = [ + Candidate(c.instance, c.index, c.role, f"new-{c.id}") for c in shuffled + ] + assert names(select(fault(select_), 42, renamed)) == expected + + +def test_seed_and_fault_name_change_picks(): + picks = { + tuple(names(select(fault({"count": 2}, name), seed, CANDIDATES))) + for seed in range(20) + for name in ("a", "b") + } + assert len(picks) > 1 + + +def test_select_all_returns_every_candidate_sorted(): + assert names(select(fault("all"), 0, list(reversed(CANDIDATES)))) == names( + CANDIDATES + ) + + +def test_select_count_picks_distinct_sorted(): + picks = names(select(fault({"count": 4}), 7, CANDIDATES)) + assert len(picks) == len(set(picks)) == 4 + assert picks == sorted(picks, key=lambda n: int(n.rsplit("-", 1)[1])) + + +def test_select_count_needs_enough_candidates(): + with pytest.raises(RunError, match="wants 7 targets, only 6 found"): + select(fault({"count": 7}), 0, CANDIDATES) + + +def test_no_candidates(): + with pytest.raises(RunError, match="no candidates"): + select(fault("one"), 0, []) + + +def test_adapter_operation_picks_like_a_fault_and_names_itself_in_errors(): + operation = Operation( + name="f", + description="d", + uses="openshell-policy-set", + options={"policy": "p"}, + target={"workload": "counter"}, + select={"count": 2}, + ) + assert names(select(operation, 42, CANDIDATES)) == names( + select(fault({"count": 2}), 42, CANDIDATES) + ) + with pytest.raises(RunError, match="^operation f: no candidates"): + select(operation, 0, []) diff --git a/e2e/chaos/tests/test_share.py b/e2e/chaos/tests/test_share.py new file mode 100644 index 0000000000..9e501f7354 --- /dev/null +++ b/e2e/chaos/tests/test_share.py @@ -0,0 +1,331 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""`chaos share`: the Markdown page, built from a synthetic run dir.""" + +from __future__ import annotations + +import json +import re +import shutil +from pathlib import Path + +from test_report import TIMELINE, jsonl, run_dir + +from chaos.cli import main +from chaos.share import MAX_POINTS, ROW_BUDGET, flow_spans, gantt, xychart + +NODES = [ + {"t": 0.5, "event": "node.started", "node": "flow.serial"}, + {"t": 0.5, "event": "node.started", "node": "flow.serial[0]", "name": "counter"}, + { + "t": 1.0, + "event": "node.started", + "node": "flow.serial[1].wait", + "wait": "counter", + "condition": "started", + }, + { + "t": 1.5, + "event": "node.finished", + "node": "flow.serial[1].wait", + "wait": "counter", + "condition": "started", + }, + {"t": 1.5, "event": "node.started", "node": "flow.serial[2].delay", "delay": "2s"}, + {"t": 3.5, "event": "node.finished", "node": "flow.serial[2].delay", "delay": "2s"}, + { + "t": 4.0, + "event": "node.started", + "node": "flow.serial[3]", + "name": "gateway-down", + }, + { + "t": 9.0, + "event": "node.finished", + "node": "flow.serial[3]", + "name": "gateway-down", + }, + {"t": 13.0, "event": "node.finished", "node": "flow.serial[0]", "name": "counter"}, +] + +RESULTS = [ + { + "scope": "preflight", + "workload": None, + "instance": None, + "check": "gateway-up", + "description": "", + "ok": True, + "output": "", + "issue": None, + "flaky": False, + "error": None, + }, + { + "scope": "workload", + "workload": "counter", + "instance": "counter-0", + "check": "exited-ok", + "description": "", + "ok": True, + "output": "exit code 0", + "issue": None, + "flaky": False, + "error": None, + }, + { + "scope": "workload", + "workload": "counter", + "instance": "counter-0", + "check": "journal-complete", + "description": "", + "ok": False, + "output": "12,60d11\nmore", + "issue": "NVIDIA/OpenShell#1", + "flaky": False, + "error": None, + }, + { + "scope": "experiment", + "workload": None, + "instance": None, + "check": "sessions-max", + "description": "", + "ok": True, + "output": "max = 2 (want at-least 2); 1 series of sessions summed over " + "3 scrapes of gateway", + "issue": None, + "flaky": False, + "error": None, + }, +] + + +def share_run(tmp_path: Path) -> Path: + run = run_dir(tmp_path) + events = sorted(TIMELINE + NODES, key=lambda e: e["t"]) + jsonl(run / "timeline.jsonl", [{"ts": "2026-10-27T00:00:00Z", **e} for e in events]) + (run / "verdict.json").write_text( + json.dumps( + {"verdict": "XFAIL", "errors": ["adapter said no"], "results": RESULTS} + ) + ) + (run / "experiment.yaml").write_text( + "id: demo\ntitle: Demo\nhypothesis: |\n It\n holds.\n" + "metrics:\n" + " - family: sessions\n description: Sessions on the gateway.\n" + " - family: lat\n description: Request latency.\n" + " - req_total\n" + "checks:\n" + " - name: sessions-max\n kind: metric\n metric: sessions\n" + " value: max\n at-least: 2\n" + " - name: exited-ok\n kind: exit-code\n equals: 0\n" + ) + return run + + +def test_share_page(tmp_path): + run = share_run(tmp_path) + out = tmp_path / "page.md" + assert main(["share", str(run), "-o", str(out)]) == 0 + text = out.read_text() + assert text.startswith("## demo · r1 · XFAIL\n") + assert "Hypothesis: It holds.\n" in text + assert ( + "| | |\n|---|---|\n| Experiment | `demo` |\n| Run | `r1` |\n" + "| Run directory | `run` |\n| Length | 14s |\n" + "| Seed | `7`, no operation selects instances; scripts get it as " + "CHAOS_EXPERIMENT_RUN_SEED |\n" + "| Adapter | `openshell-docker` |\n| OpenShell | `1.2.3` |\n" + ) in text + # Preflight checks are not counted; the failed one is listed with its issue. + assert "**Checks:** 2 passed, 1 failed (1 known)." in text + assert ( + "- `journal-complete` on counter-0 failed, known NVIDIA/OpenShell#1: 12,60d11\n" + in text + ) + assert "Harness error: adapter said no" in text + chart = text.split("```mermaid\n")[1].split("```")[0] + assert chart.startswith("gantt\n title demo · r1 · XFAIL\n dateFormat mm:ss\n") + assert " section flow\n counter :00:00, 00:13\n" in chart + assert " wait counter started :00:01, 00:02\n" in chart + assert " delay 2s :00:02, 00:04\n" in chart + # A built-in operation is its fault window, not a second row for its node. + assert " fault gateway-down (outage) :crit, 00:04, 00:09\n" in chart + assert "\n gateway-down :" not in chart + assert " fault kill-one (kill) :milestone, crit, 00:10, 00:10\n" in chart + assert " section counter-0\n" in chart + assert " Running :active, 00:02, 00:12\n" in chart + assert " Succeeded :done, 00:12, 00:13\n" in chart + assert "would not fit" not in text + # The metrics table joins the experiment's metric checks to their family. + assert ( + "| `sessions` | Sessions on the gateway. | " + "`sessions-max` ok: max = 2 (want at-least 2) |\n" in text + ) + assert "| `lat` | Request latency. | none |\n" in text + assert "| `req_total` | | none |\n" in text + # Charts in the declared order; after the first two they fold. + heads = [line for line in text.splitlines() if line.startswith("#### ")] + assert heads == [ + "#### sessions", + "#### lat (observations)", + "#### lat (mean per observation)", + "#### req_total", + ] + assert "2 more charts" in text + assert text.index("#### lat (observations)") < text.index("
") + assert text.index("
") < text.index("#### lat (mean per observation)") + sessions = text.split("#### sessions\n")[1].split("####")[0] + assert ' x-axis "seconds since the run started" 0 --> 8\n' in sessions + assert " line [1, 2, 0, 0, 1]\n" in sessions + assert ( + "Sessions on the gateway. Scrapes the endpoint did not answer are drawn at 0." + in sessions + ) + + +def test_default_output_is_next_to_the_report(tmp_path, capsys): + run = share_run(tmp_path) + assert main(["share", str(run)]) == 0 + assert capsys.readouterr().out.strip() == str(run / "share.md") + assert (run / "share.md").is_file() + + +def test_run_dir_without_manifest_is_refused(tmp_path, capsys): + assert main(["share", str(tmp_path)]) == 2 + assert "manifest" in capsys.readouterr().err + + +def test_png_needs_mermaid_cli(tmp_path, capsys, monkeypatch): + run = share_run(tmp_path) + monkeypatch.setattr(shutil, "which", lambda name: None) + assert main(["share", str(run), "--png"]) == 2 + err = capsys.readouterr().err + assert "mermaid-cli" in err and "mmdc -i" in err + assert (run / "share.md").is_file() + + +def test_repeated_nodes_collapse_into_one_row(): + timeline = [] + for i in range(4): + node = f"flow.repeat[{i}]" + timeline.append( + {"t": i * 10.0, "event": "node.started", "node": node, "name": "restart"} + ) + timeline.append( + { + "t": i * 10.0 + 2, + "event": "node.finished", + "node": node, + "name": "restart", + } + ) + timeline.append( + {"t": 40.0, "event": "node.started", "node": "flow.open", "name": "tail"} + ) + spans = flow_spans(timeline) + assert [s["name"] for s in spans] == ["restart"] * 4 + ["tail"] + # A node still open when the timeline ends runs to its last event. + assert spans[-1]["to"] == 40.0 + data = { + "experiment": {"id": "x"}, + "run_id": "r", + "verdict": "PASS", + "end": 40.0, + "timeline": timeline, + "faults": [], + "lanes": [], + "results": [], + } + chart, note = gantt(data) + assert " restart ×4 :00:00, 00:32\n" in chart + assert "\n restart :" not in chart + assert note == "" + + +def lane(name: str, states: list[str]) -> dict: + return { + "name": name, + "segments": [ + {"from": float(i), "to": float(i + 1), "state": s} + for i, s in enumerate(states) + ], + } + + +def test_instances_beyond_the_row_budget_are_left_out(): + per_lane = ROW_BUDGET // 4 + data = { + "experiment": {"id": "x"}, + "run_id": "r", + "verdict": "FAIL", + "end": 100.0, + "timeline": [], + "faults": [{"name": "f", "uses": "kill", "from": 1.0, "to": 2.0}], + "lanes": [lane(f"c-{i}", ["Running"] * per_lane) for i in range(5)] + + [lane("c-5", ["Running"] * (per_lane - 1) + ["Unknown"])], + "results": [ + {"scope": "workload", "instance": "c-3", "check": "k", "ok": False}, + ], + } + chart, note = gantt(data) + sections = [line.split()[1] for line in chart.splitlines() if "section" in line] + # The instance with a failed check, then the one that went Unknown, then + # the rest in run order, up to the budget. + assert sections == ["flow", "c-3", "c-5", "c-0"] + assert chart.count("\n ") == 1 + 3 * per_lane + assert note.startswith( + f"3 of 6 instances ({3 * per_lane} state changes) would not fit" + ) + data["lanes"] = data["lanes"][:1] + chart, note = gantt(data) + assert note == "" + data["lanes"][0]["segments"] *= 5 + chart, note = gantt(data) + assert "section c-0" not in chart + assert note.startswith(f"The instance states ({5 * per_lane} state changes)") + + +def test_xychart_buckets_long_series_and_drops_leading_gaps(): + points = [ + [float(t), None if t < 2 else float(t % 7)] for t in range(3 * MAX_POINTS) + ] + text, note = xychart( + {"title": 'a "b"', "subtitle": "per second", "points": points, "family": "a"} + ) + assert ' title "a b "\n' in text + values = text.split("line [")[1].split("]")[0].split(", ") + assert len(values) <= MAX_POINTS + assert note == f"{3 * MAX_POINTS - 2} scrapes averaged into {len(values)} points" + assert ' x-axis "seconds since the run started" 2 --> ' in text + assert xychart( + {"title": "t", "subtitle": "s", "points": [[0, None]], "family": "a"} + ) == ( + "", + "no scrapes with a value", + ) + + +def test_mermaid_numbers_are_plain_decimals(tmp_path): + """Mermaid's xychart parser rejects exponent form, which Python prints + for small per-tick latencies (`9e-05`) and for large values.""" + run = share_run(tmp_path) + jsonl( + run / "system" / "metrics" / "gateway.jsonl", + [ + {"t": float(t), "up": 1, "samples": {"sessions": v}} + for t, v in enumerate([0.000235, 0, 0.00009, 0.0001855, 0, 123456.7]) + ], + ) + out = tmp_path / "page.md" + assert main(["share", str(run), "-o", str(out)]) == 0 + text = out.read_text() + fences = [block.split("```")[0] for block in text.split("```mermaid\n")[1:]] + assert any("xychart-beta" in fence for fence in fences) + for fence in fences: + assert not re.search(r"\d[eE][+-]?\d", fence), fence + sessions = text.split("#### sessions\n")[1].split("####")[0] + assert " line [0.000235, 0, 0.00009, 0.0001855, 0, 123500]\n" in sessions + assert '" 0 --> 135800\n line [' in sessions diff --git a/e2e/chaos/tests/test_signals.py b/e2e/chaos/tests/test_signals.py new file mode 100644 index 0000000000..45885cf126 --- /dev/null +++ b/e2e/chaos/tests/test_signals.py @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Adapter signals and the flow's `wait: {signal: ...}` node.""" + +from __future__ import annotations + +import asyncio +import time + +import pytest +from conftest import minimal_adapter +from pydantic import ValidationError + +from chaos.adapter import Adapter +from chaos.model import SignalWait, parse_flow +from chaos.signals import Signals + + +def test_signal_wait_parses(): + node = parse_flow({"wait": {"signal": "sweep", "delay": "55s", "timeout": "3m"}}) + assert node == SignalWait("sweep", delay="55s", timeout="3m", path="flow.wait") + assert (node.delay_seconds, node.timeout_seconds) == (55, 180) + bare = parse_flow({"wait": {"signal": "sweep"}}) + assert (bare.delay_seconds, bare.timeout) == (0, "5m") + + +@pytest.mark.parametrize( + ("value", "error"), + [ + ({"signal": "sweep", "every": "1s"}, r"unknown keys \['every'\]"), + ({"delay": "1s"}, r"want \{workload: , condition: started\} or"), + ({"signal": 1}, r"flow\.wait\.signal: want a signal name"), + ( + {"workload": "w", "condition": "ready"}, + r"flow\.wait\.condition: want started", + ), + ( + {"workload": "w", "condition": "started", "for": "1s"}, + r"unknown keys \['for'\]", + ), + ({"signal": "sweep", "delay": "soon"}, r"flow\.wait\.delay: invalid duration"), + ({"signal": "sweep", "timeout": "3"}, r"flow\.wait\.timeout: invalid duration"), + ("w.running", r"want \{workload: , condition: started\} or"), + ], +) +def test_signal_wait_rejects(value, error): + with pytest.raises(ValueError, match=error): + parse_flow({"wait": value}) + + +def adapter(**extra) -> Adapter: + return Adapter.model_validate( + minimal_adapter(watch=[{"name": "otel", "run": "true"}], **extra) + ) + + +def test_signal_names_a_watch_and_a_regex(): + (signal,) = adapter(signals=[{"name": "s", "watch": "otel", "match": "a+"}]).signals + assert signal.watch == "otel" + with pytest.raises(ValidationError, match="no watch 'logs'"): + adapter(signals=[{"name": "s", "watch": "logs", "match": "a"}]) + with pytest.raises(ValidationError, match="invalid regex"): + adapter(signals=[{"name": "s", "watch": "otel", "match": "("}]) + with pytest.raises(ValidationError, match="signals names must be unique"): + adapter(signals=[{"name": "s", "watch": "otel", "match": "a"}] * 2) + + +def next_event(signals: Signals, after, not_before, timeout=0.5) -> float: + async def go() -> float: + return await signals.next( + "s", after, not_before, deadline=time.monotonic() + timeout + ) + + return asyncio.run(go()) + + +def test_next_skips_old_events(): + now = time.monotonic() + signals = Signals() + for at in (now - 30, now - 10, now - 5): + signals.add("s", at) + # A delay of 20s still reaches the event 10s ago, not the one 30s ago. + assert next_event(signals, None, now - 20) == now - 10 + # The node already returned for it: the next one. + assert next_event(signals, now - 10, now - 20) == now - 5 + with pytest.raises(TimeoutError): + next_event(signals, now - 5, now - 20, timeout=0.1) + + +def test_next_waits_for_a_new_event(): + signals = Signals() + signals.add("s", time.monotonic() - 1) + + async def go() -> tuple[float, float]: + began = time.monotonic() + later = asyncio.get_running_loop().call_later(0.1, signals.add, "s") + try: + return began, await signals.next("s", None, began, began + 2) + finally: + later.cancel() + + began, at = asyncio.run(go()) + assert at >= began + 0.1 diff --git a/e2e/chaos/uv.lock b/e2e/chaos/uv.lock new file mode 100644 index 0000000000..0aa1a8aa0b --- /dev/null +++ b/e2e/chaos/uv.lock @@ -0,0 +1,349 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "openshell-chaos" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "pydantic" }, + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "pydantic", specifier = ">=2.7" }, + { name = "pyyaml", specifier = ">=6.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=9.0.3" }, + { name = "ruff", specifier = ">=0.4" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + +[[package]] +name = "pydantic" +version = "2.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7c/0b/8e10b2e693af8ec54346a14caf36221334775975a747c9ceaa3f8371d96d/pydantic-2.14.1.tar.gz", hash = "sha256:94f478203dd03404682a1ada216965651dd74b1d2d5ffd62e00e0837caab5c26", size = 900362, upload-time = "2026-10-11T18:37:55.396Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/ea/a56b9fe5066f3537b7882f77e9c5dfb26d8c2eefdaed9b5fc73d57b4dc22/pydantic-2.14.1-py3-none-any.whl", hash = "sha256:9195d967ec791692a04438115466764fb8b9a27b31f14a760437694f40d6b454", size = 488106, upload-time = "2026-10-11T18:37:53.437Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.50.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/24/af4ec4be49fbc810f35b0bdcacc3d433b56bbfa469fd3bfd4ae116cd9bf1/pydantic_core-2.50.1.tar.gz", hash = "sha256:e50d7b94baac6c7d09927fa5ca5800a0c7ee5015c7fcff65beb3a1931b5a6e09", size = 513515, upload-time = "2026-10-11T18:35:44.82Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/1b/ca81811da3d2aea724fb3c4ba6b83f50ab8f210576306ccd601973142d35/pydantic_core-2.50.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:c531166c42ea7bdfecc8c50049581f05dd1993b09cc7c52bb36a14e96deaec7d", size = 2152145, upload-time = "2026-10-11T18:31:58.046Z" }, + { url = "https://files.pythonhosted.org/packages/59/20/938ef7e0541af0a314e2f9d63848dbd1656ba2ebfe5ca0469f7ebee00d5e/pydantic_core-2.50.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b6d0c2183008c188e19f4906d426b293bdc4f67ab17df8e180fe16cda208fa71", size = 2045761, upload-time = "2026-10-11T18:31:59.35Z" }, + { url = "https://files.pythonhosted.org/packages/2f/4b/456497fe2affa18f49b226e89bde31e8a204751ee343dd9beaac5438123e/pydantic_core-2.50.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94be440c03fede26969a5ce75468e0e6a9927a1b46d9b679ee8adc1b057b0350", size = 2049557, upload-time = "2026-10-11T18:32:01.121Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9d/cd5dfe13928f55e236dff3a1bee06721d318ca6c955d9ca993b0a5d09f39/pydantic_core-2.50.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:36c426eac0af8d1529ff8467e612b933346caec1fdc0d774f78f67a1a11e16c1", size = 2188153, upload-time = "2026-10-11T18:32:02.442Z" }, + { url = "https://files.pythonhosted.org/packages/fb/8d/d0b25bc484f8d9136ea76ece69e0b2d24ab370cc2e7bbd1b79e61723b763/pydantic_core-2.50.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:028e2f212273d4a39b1ec1e0de8166b1165a65fc0f1111452a9d94fc7c625c63", size = 2388308, upload-time = "2026-10-11T18:32:03.767Z" }, + { url = "https://files.pythonhosted.org/packages/c5/64/6513cac7c9f78492abafa01d57a69de1d246bf3085fd7b0330bf14b025b3/pydantic_core-2.50.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e6f0cc1bb9900dc558960894adeb30b0c083366fc1d69b856209fb2ca5c36fe5", size = 2431187, upload-time = "2026-10-11T18:32:05.159Z" }, + { url = "https://files.pythonhosted.org/packages/ed/60/7f2ae89c37e69a3dc0da9ff57b0388c285542c688d05f36767770429de85/pydantic_core-2.50.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8812592c85d0edf423f10eadcef42716d71e8219085ad9e85b775057b7306133", size = 2157623, upload-time = "2026-10-11T18:32:06.616Z" }, + { url = "https://files.pythonhosted.org/packages/87/02/e3a71373b85a7973099b9a229a4302c7792262717e0b4997e41c01f7dd16/pydantic_core-2.50.1-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:bbce99252ba3167b2b6277f1829d5bf4b43b754524bddf7f944707c3db7d2253", size = 2254539, upload-time = "2026-10-11T18:32:07.968Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d5/5b620aaf40dc631d911565b64d49d3fa8422027c046a7a96dc4e4921ae98/pydantic_core-2.50.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:476f6ed8e43cd1e0b460920e23571700872b284e77331cb30c4faf459cf48a4b", size = 2267344, upload-time = "2026-10-11T18:32:09.399Z" }, + { url = "https://files.pythonhosted.org/packages/e0/48/f46baf4580d8a56ca4782aa65842e16d3cf39736b29fcce8e9c0521b3422/pydantic_core-2.50.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2cbd1b75b09e976ed0d6b6ca297675632ca35df86130088457cdc60ef36970ae", size = 2300877, upload-time = "2026-10-11T18:32:10.903Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9d/e7d28505e6092c1fe075838a4f793ccbe6dd676e44ab16175815ee06cfe0/pydantic_core-2.50.1-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:5958c72adb417c39b12ac87525ac60b0d73315fcdc59e21f44ee4a5e2512c9ef", size = 2464270, upload-time = "2026-10-11T18:32:12.329Z" }, + { url = "https://files.pythonhosted.org/packages/22/b1/3bf63b82efd93ebc9968f9c12fc46037a28bcb452c88e4688f33e5ff7dca/pydantic_core-2.50.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d8f9e8a6c4ab04b78d61f78627370d834eb004b2869dcb28cfffa647b4ea1980", size = 2478653, upload-time = "2026-10-11T18:32:13.735Z" }, + { url = "https://files.pythonhosted.org/packages/28/17/26d1e4b9c2ba6a92b7dc75bbcb99ef44456b7e3af1bced868403667ea7d3/pydantic_core-2.50.1-cp311-cp311-win32.whl", hash = "sha256:4be846f55c9477f5f3ddde8f2ce941137e16862a56d018ed885d422bb6ae02f2", size = 2120246, upload-time = "2026-10-11T18:32:15.591Z" }, + { url = "https://files.pythonhosted.org/packages/24/65/3e875e7991eaecd4d70686293ff0d8a99aa8f93dc295e6a21c3d702912fb/pydantic_core-2.50.1-cp311-cp311-win_amd64.whl", hash = "sha256:0048b6dddc8ef4b64fccaad878bd143b0c3882ea9936279dc11d613f6b7dd1bc", size = 2176332, upload-time = "2026-10-11T18:32:17.153Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5c/58dbd17c61f209d91c70f549eb7a3bbec7023e89ea12dead091462d6d658/pydantic_core-2.50.1-cp311-cp311-win_arm64.whl", hash = "sha256:6a733778df2f7087ec1100ed0b41533e4f3001976e99570fa34f57c66e7f8e3e", size = 2182471, upload-time = "2026-10-11T18:32:18.49Z" }, + { url = "https://files.pythonhosted.org/packages/ee/94/101b49a6c63f99586755690696eae14532772587d9e39e93f7b8d28a49da/pydantic_core-2.50.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:704075d10b74f2f3c6e15407c696d88701df35fc8953f434a431add0d0074db0", size = 2222428, upload-time = "2026-10-11T18:32:20.228Z" }, + { url = "https://files.pythonhosted.org/packages/db/a3/f8b1b39349c33037c73d7a1b794c9c29d8f07d5bd2b74399c1375a529cc6/pydantic_core-2.50.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e8e1d6ce820aa23317e8209a86bd65a540973c12dc7552b48a4f6c8e9926815e", size = 2025779, upload-time = "2026-10-11T18:32:21.723Z" }, + { url = "https://files.pythonhosted.org/packages/d2/2d/862a7d115305cc621358e51ba13f296e0ffdf4fbd984be9f63bc7862730b/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c18db21573bd2c6489f9a544b7499f0df2853958c568e5e783536ee1f690af41", size = 2048429, upload-time = "2026-10-11T18:32:23.3Z" }, + { url = "https://files.pythonhosted.org/packages/c0/7d/fed95d18bf2b5abe2d5ab2bbec4184bfc22b6805994a33ccdae8edffd25a/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cb57f304525a5e3c13333b772bf9a473f36326e9c821b2e8e1b2fd36f80ae2c3", size = 2173274, upload-time = "2026-10-11T18:32:24.697Z" }, + { url = "https://files.pythonhosted.org/packages/93/1e/c6f22ceac65dc3d291ff79bea0eb74cbd1de8b128f01fee48af03bcea1e1/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a27c09d86600f1bf2fe3f37e1ae697faf3143931c09322cd799da94deee923b5", size = 2360403, upload-time = "2026-10-11T18:32:26.281Z" }, + { url = "https://files.pythonhosted.org/packages/2c/97/190b5b6bfe4c72f265feb657913eb30749193927a5ad7b83cfa7866a25a2/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:46b3301d3b5c886f77de7546e47274a5842c622ea2020b8c6524c6b66913b4a6", size = 2439799, upload-time = "2026-10-11T18:32:27.844Z" }, + { url = "https://files.pythonhosted.org/packages/90/24/48cd98388e4ac2c08d0800af36d15413db15e20387f1038123a2c86cb2f6/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93ba4e9d8210d941c200431a56b2c0400b131865947903937ed3ec5404307d2e", size = 2160157, upload-time = "2026-10-11T18:32:29.553Z" }, + { url = "https://files.pythonhosted.org/packages/00/86/d909133d0b9979ed14c20f110f21124a0e346caf071bd66ac5af3009062b/pydantic_core-2.50.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:e5faeaee74a57d32b3ab3aebad2e348f06d3ba946fc5d28c1728455f00a3d13a", size = 2229523, upload-time = "2026-10-11T18:32:31.027Z" }, + { url = "https://files.pythonhosted.org/packages/90/31/4b43f08131b2098e73a852fd932063385f3103b33839fd9d1abdb89e673d/pydantic_core-2.50.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a3cda0e538208e5d722bbf3698b24f19c0a7d05bc8d5f8a7f9b121ea7fa243d9", size = 2242700, upload-time = "2026-10-11T18:32:32.622Z" }, + { url = "https://files.pythonhosted.org/packages/88/77/de39658569dc7a4c293d110a89629f61145e7538f00d1e4280c28debefd3/pydantic_core-2.50.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:57f51b31ff826e2859120cf4737c5a758a48d96f3e97da40ccee1796d58078ff", size = 2278452, upload-time = "2026-10-11T18:32:34.165Z" }, + { url = "https://files.pythonhosted.org/packages/46/bd/1eeb9e4e791e81210ef9096f58080b32ba15b5b13c108d50aaa3f8ce9f73/pydantic_core-2.50.1-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8daa7ee75245d43ad7d747e5c9ecc1b1d06552f72b14887e9276f787d57375f4", size = 2450098, upload-time = "2026-10-11T18:32:35.547Z" }, + { url = "https://files.pythonhosted.org/packages/1c/e6/b2accfdff17e5a88d6358536702146de00f9339adaa4d5bbe1ee93cb4832/pydantic_core-2.50.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:acbf31f37c53a5ac0c34706c80b4f5107ba20b05fdd3816124bf236ef0c57dd2", size = 2476839, upload-time = "2026-10-11T18:32:37.119Z" }, + { url = "https://files.pythonhosted.org/packages/3b/7e/c8a83eb5d2cd42c2e13507150fb777b1edea5a018adc9cbf2858a1d08285/pydantic_core-2.50.1-cp312-cp312-win32.whl", hash = "sha256:45b11cac094aa25725581d9304eee93c9028516b9ea80dd9e175e13a5a2c840e", size = 2101425, upload-time = "2026-10-11T18:32:38.593Z" }, + { url = "https://files.pythonhosted.org/packages/c5/17/8bbd530b8659e9d963f5b16f6cb8e159f4cd174b4d6304e48969ab12dba8/pydantic_core-2.50.1-cp312-cp312-win_amd64.whl", hash = "sha256:132529c83901437ff642f585216831bf5fd7a91df66829907e155192ead62498", size = 2172522, upload-time = "2026-10-11T18:32:40.112Z" }, + { url = "https://files.pythonhosted.org/packages/af/9f/a79d690b127e4e3cff143d031538ab9aa715f6b2fbeecbd07cd6f2bf9fb7/pydantic_core-2.50.1-cp312-cp312-win_arm64.whl", hash = "sha256:4e834f6a8e4ff772dcc34f58ef5504147a3ea5b0f4eeb13b0f8eb2ca75ac57f1", size = 2161366, upload-time = "2026-10-11T18:32:41.606Z" }, + { url = "https://files.pythonhosted.org/packages/ae/57/0e237d7091d2cd44a35d243b7344227f90440166860469cd036afc23a04d/pydantic_core-2.50.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:d5e062c01286d861fd6a1c4ff6e063547b3e713067f2df033c0ff97ac2ca006b", size = 2222774, upload-time = "2026-10-11T18:32:43.103Z" }, + { url = "https://files.pythonhosted.org/packages/f7/b9/c720e56858d4e1539503297ed37063e0c08e0f3541c41b777f6a800f75fa/pydantic_core-2.50.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0c003c3b7f49debb893d2d85ae099ac5959c9839e2f330fadb1fcdf7a6594482", size = 2024738, upload-time = "2026-10-11T18:32:44.587Z" }, + { url = "https://files.pythonhosted.org/packages/4d/b8/fbfc25875219cc060e613170ff10e850c6d8924beb4910da57c2ee3ba1d2/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:409e0ea40ec30d9158f33574fd758e689f6045a0f2596701828c27816ca9687d", size = 2047049, upload-time = "2026-10-11T18:32:46.164Z" }, + { url = "https://files.pythonhosted.org/packages/d7/43/34210124d504c553688f2f04b48500b131237528ac545b445e0d6d30e0d5/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:131059670f1d2444269b8585cb888963994871932447c08b39ac6a51fcfef658", size = 2173415, upload-time = "2026-10-11T18:32:47.722Z" }, + { url = "https://files.pythonhosted.org/packages/65/cf/6e178e8fdc11da5965bef980983bf46326a42f436871dd51ba0a57f39df1/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6dbcbee53bf17196a7f745aa9bf5a9603953a1e365b1f020be3207c676a3e7c4", size = 2361719, upload-time = "2026-10-11T18:32:49.217Z" }, + { url = "https://files.pythonhosted.org/packages/11/14/bd5169d356aa91bf777e28ba0b281c192d286d03c2812c9c9db023a2f00e/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:325c23f3e35cfbf0fe3486fa5f7260d1e45885173002d30a28ca019994124255", size = 2439803, upload-time = "2026-10-11T18:32:51.025Z" }, + { url = "https://files.pythonhosted.org/packages/1c/bc/d79d000e5203ebef39af839f2ce77a777fcad6e08ad26af9a2fcd114ffc8/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17e722e156d0444ecaefbe640bdb60928752bf2013e2b7a11cdb099aaae19bec", size = 2158402, upload-time = "2026-10-11T18:32:52.714Z" }, + { url = "https://files.pythonhosted.org/packages/1b/a5/4902cb5fd599422c130bd3124ab31ee8b771199bd56662702eed07d3fec7/pydantic_core-2.50.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:aa8224f10880d9bf1b5993988ba153d42a8b4f3f4f511f93b1f09c93ff613c72", size = 2229755, upload-time = "2026-10-11T18:32:54.126Z" }, + { url = "https://files.pythonhosted.org/packages/1a/f5/c1481f8669f6060d89110c9b1374173fc8767ca276b84f4870bd8acd5e3f/pydantic_core-2.50.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:41bc8237121bd8dc8d888dfd6279fc166ffc88c1f1bf3a8bf00869680533ca4c", size = 2242303, upload-time = "2026-10-11T18:32:55.641Z" }, + { url = "https://files.pythonhosted.org/packages/04/f9/77fc3c7653ba9b6e42049e17e96b25388187d4a7c274ab1cb07f58ac8419/pydantic_core-2.50.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:45c6266d071c241f2a168d45bf8c54344f0effce35e7e6b73afdec11f3687568", size = 2278345, upload-time = "2026-10-11T18:32:57.38Z" }, + { url = "https://files.pythonhosted.org/packages/95/9b/0579c5d12e7f2b16b27e6782427987341fcce07b0725e02ddb0b74add0c4/pydantic_core-2.50.1-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:1deeacb112d14d3f4fcb16b165f7dbaf76c70ba6e82f37ba042bdab51970a0b8", size = 2449947, upload-time = "2026-10-11T18:32:58.896Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ac/1b677db91eba54cc5922f4de6edc46ba45c2bd712dfdc0130382d158e214/pydantic_core-2.50.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:1c96fd793b73d1b92e65570132505498fe7b21eaef73cdf74e67e5dfba7ac9e4", size = 2476889, upload-time = "2026-10-11T18:33:00.665Z" }, + { url = "https://files.pythonhosted.org/packages/4d/2a/3a9f6624ee3ea9ccba5249dde11418bb4c35780a7a92608f8768bd3fea39/pydantic_core-2.50.1-cp313-cp313-win32.whl", hash = "sha256:06ead20d39ffd6f2f6f2a8f8a6de67ff8bb1b4f14a8a30e058502514ee2ac685", size = 2101118, upload-time = "2026-10-11T18:33:02.329Z" }, + { url = "https://files.pythonhosted.org/packages/2d/1f/323f78ddd9d9938aac420c1abb4e8ba799fc8bdab0acc67c0593b837c979/pydantic_core-2.50.1-cp313-cp313-win_amd64.whl", hash = "sha256:7816e98acc08119dc0f340ab167048ecc54126316330c1f0caf7c6756c88e28f", size = 2170840, upload-time = "2026-10-11T18:33:03.919Z" }, + { url = "https://files.pythonhosted.org/packages/cb/09/8497c52a739ae425c3ac2f7f56414cbc711c67d374346174f40fe2062644/pydantic_core-2.50.1-cp313-cp313-win_arm64.whl", hash = "sha256:c17799a62c142d61b8a3c51752a7cbc87fe2ad4ccfab10e628a77b405075c662", size = 2161633, upload-time = "2026-10-11T18:33:05.518Z" }, + { url = "https://files.pythonhosted.org/packages/49/33/28b96e81677153715e3eafb9f26663a80841e859fde282a380359b0d3fa1/pydantic_core-2.50.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:1cf41f1ae3fa155cf167a72689ad044bcc1e3c97e064123677149bdfb5dafc4a", size = 2224428, upload-time = "2026-10-11T18:33:07.153Z" }, + { url = "https://files.pythonhosted.org/packages/94/40/15c06410c9b7b8da5805d27b64e09bd3f900e986728106db2689dbc51513/pydantic_core-2.50.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4df197990c15b5a37c5a277d131d9f2c67de6133f2e5dafd80d9bba4b99f46f9", size = 2023740, upload-time = "2026-10-11T18:33:08.763Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4e/5eb629f6efc2a27e421d789dd4bfacbb5f6d09c80c28b13d1e87d73163d5/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0036473f5583e6a60e50b8b21651511564277a3f05cc5dab8cf579f552cd5f6c", size = 2049435, upload-time = "2026-10-11T18:33:10.366Z" }, + { url = "https://files.pythonhosted.org/packages/ba/8e/f195aebec49ad12318876ac2368c197a7939f5d52f8e156d2238ef8a4588/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:992c3514ec891fa7858099183e4d64e6bd5a5d4ff452fae29df22faa77a006bb", size = 2172975, upload-time = "2026-10-11T18:33:12.368Z" }, + { url = "https://files.pythonhosted.org/packages/e0/f5/7ad9fb83010cd5ea0948409db105676ed779c4e709e2d3d89b9fe2558794/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:739dc730e6be3bd5ec2f4ab5cfc7eb047cc45fc1497b3bafec74ff2ed07df597", size = 2364197, upload-time = "2026-10-11T18:33:14.244Z" }, + { url = "https://files.pythonhosted.org/packages/ca/fb/bf0aab3e78301d202b82a0322ec968cd703b11fc0623fde9a28708936f62/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:32fad3a91e51b6d2039c572db04a5a873260b399f6bd62c3552671fa7a4a2899", size = 2444259, upload-time = "2026-10-11T18:33:15.79Z" }, + { url = "https://files.pythonhosted.org/packages/45/35/38f6d6564fae57d9b12e5676dfa947e0e7d5c46dbeaa7eb3352261d6d299/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42b54c2c90ad348b5e3a85e03e715d572c1fde357ef104cdfe3b03b697a404ea", size = 2159857, upload-time = "2026-10-11T18:33:17.51Z" }, + { url = "https://files.pythonhosted.org/packages/65/a0/fb0a3ca10f139dcf765b2d312d10cf0f65c57c59993229d00ad12b14ceff/pydantic_core-2.50.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:2df1ff41884de2bc4b307bafd7c40a691094fad2ff8e767e5b45a319257bcf4e", size = 2233954, upload-time = "2026-10-11T18:33:19.591Z" }, + { url = "https://files.pythonhosted.org/packages/8c/6a/e63842252702aa4ec6e6b7178ca85e541a77459076592c23ebe2d9840b33/pydantic_core-2.50.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe90228920fd8ff2be62622b6bb8a2b11acd65046d50c6b130614b5879605a20", size = 2243876, upload-time = "2026-10-11T18:33:21.393Z" }, + { url = "https://files.pythonhosted.org/packages/39/25/5991cf8318b37e0dfab47b87541619a1df8501a793cba0d978846cba37a7/pydantic_core-2.50.1-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:844b869f118e22a41a091bdcedda8a71bc1b0f62c38d1a0c3211cece47e1d8fc", size = 2277908, upload-time = "2026-10-11T18:33:23.324Z" }, + { url = "https://files.pythonhosted.org/packages/a7/3e/3ee8baaa6cc25a6961c69168cf9ff0f002d56f4e0d541ad6724c18fb61f3/pydantic_core-2.50.1-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:2eb75304506894a281d346220a4f7481a1b8729577c5ed2a05395991966a8396", size = 2449285, upload-time = "2026-10-11T18:33:24.942Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c7/acbec6deac13fe697a80c275a9b6661db62c4d323bac8a47347b1f39c7cd/pydantic_core-2.50.1-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:6b20a4bffabdad0db2927ac034ae3b8a681b1f7a0182f3e60b479ad2fde21ebb", size = 2479013, upload-time = "2026-10-11T18:33:26.925Z" }, + { url = "https://files.pythonhosted.org/packages/10/08/21a3f237b264389f6219d053ee78fc1b5c2fd402c1b1cb2b6cb8b85f9834/pydantic_core-2.50.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:99ba9bc2b8062ea0c326a990f7f00e6530c23579de66dd246e72c4cafef950a5", size = 1484400, upload-time = "2026-10-11T18:33:28.693Z" }, + { url = "https://files.pythonhosted.org/packages/09/ce/077a6d262d12ef09108377ac0717f029f420d773ace63cafd6143af75568/pydantic_core-2.50.1-cp314-cp314-win32.whl", hash = "sha256:cf356f70551d40374eaffb1aa63f1eb6d2006681cbd7a9faea173ce0f4dd7cd2", size = 2103222, upload-time = "2026-10-11T18:33:30.326Z" }, + { url = "https://files.pythonhosted.org/packages/14/4c/350a2415209c43d670eb71d3c040f332c04a30583d39e311b05c7ac15762/pydantic_core-2.50.1-cp314-cp314-win_amd64.whl", hash = "sha256:d32f3acc081cc3923386d88f422cde8892335e95f034e0104bb4cf9310d9915f", size = 2171586, upload-time = "2026-10-11T18:33:32.139Z" }, + { url = "https://files.pythonhosted.org/packages/bf/92/9bea6ca96580a0902fed366f064f0829e404f41889b34543279a1162b888/pydantic_core-2.50.1-cp314-cp314-win_arm64.whl", hash = "sha256:bed5163e03b98bc1fa2eb05d74c63d9c5c95d8ed6254985481640fbf5e237dea", size = 2155773, upload-time = "2026-10-11T18:33:33.896Z" }, + { url = "https://files.pythonhosted.org/packages/86/8c/f121f073cf32bdd5cba7e6230b1ce0ac845b0cf43e44dd597d95af272db2/pydantic_core-2.50.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:9572c1369e9c9da2d64a7b7992c786d90ff295abc93964cfe3125e4290768070", size = 2234017, upload-time = "2026-10-11T18:33:35.588Z" }, + { url = "https://files.pythonhosted.org/packages/a4/69/2bb2bcd6146cffe98d760a46c42ae71efd5151d9b2f9c9bf6619a3b32083/pydantic_core-2.50.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2005207aafe1231315718bf6ed5d064a7300fb4772754af35ee72fc68159492e", size = 2001016, upload-time = "2026-10-11T18:33:37.57Z" }, + { url = "https://files.pythonhosted.org/packages/20/b3/fbf854c7d07ec114260c26e9e2071a4381740f9ae09641dbfcbdf2a18c45/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64f6047f62a6c5ae08d0a6afb035667aa2d97c3d20d69762e034c5ea144d92a5", size = 2023015, upload-time = "2026-10-11T18:33:39.433Z" }, + { url = "https://files.pythonhosted.org/packages/65/20/6de55b2f92cdb614b745c6e9ced639fc4fd7e1e77604825a88bdece6fcbd/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1ef800dd7d85bcdadf4c3076e4c94e43939493558a3b69a1ea830c706d4617bb", size = 2192543, upload-time = "2026-10-11T18:33:41.381Z" }, + { url = "https://files.pythonhosted.org/packages/0c/d9/19e91c94bd5c405945ce2f15526808aea37e162c253160de8ed7bf70b406/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b0135bcdcaa0f23573f286e4cb5e0fd2962700964ed13df085b85f2b97aeab9e", size = 2388704, upload-time = "2026-10-11T18:33:43.167Z" }, + { url = "https://files.pythonhosted.org/packages/b6/bc/2e24c8415eae1a25ee5a5484946a917123bad2e9d01689a759236928175a/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0b3a6f334c6a2345ca15318ff894502a90012536404b37c844a976c76c846e0b", size = 2464118, upload-time = "2026-10-11T18:33:44.856Z" }, + { url = "https://files.pythonhosted.org/packages/bd/1f/945b8053cb061c64e102bcaf7bfb9ed740c0bd4349f9bb978a7d4ddff4ab/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06e01fbbfdb9be777b316a71b6c49efaf4a08b615d0a98d678cda3023f79d019", size = 2150142, upload-time = "2026-10-11T18:33:46.661Z" }, + { url = "https://files.pythonhosted.org/packages/2b/78/96a3e50bf0d64aaae781107eb9335b7529d05a85793ed6e7241c4cd1d931/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:a29a061fec0b4e2d714f277e70a3a18125ecff803f2fea6eade2f2e53711d112", size = 2245577, upload-time = "2026-10-11T18:33:48.574Z" }, + { url = "https://files.pythonhosted.org/packages/7a/5d/a6038a0322232758a6ebfa709f4ca14a60bf5b93940cd0b345056a557da4/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f5187624823423e1d1b82b1072ac41dc837389e18d3d0572cc19bbee46cd550a", size = 2263728, upload-time = "2026-10-11T18:33:50.527Z" }, + { url = "https://files.pythonhosted.org/packages/0d/4b/76ded3333a457a9344c4f2d63f2d82d0101654b05178fa4ddfe7d3627674/pydantic_core-2.50.1-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:3e46a9eb0a0901dd6275e6b06ac3a464885ef350ec4121fe486869de8053e4bb", size = 2293630, upload-time = "2026-10-11T18:33:52.362Z" }, + { url = "https://files.pythonhosted.org/packages/a9/00/9eca378335c9c1b72bc779bf6e4c2a82ce784f14ec48a0e87102c9870e05/pydantic_core-2.50.1-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:756d669f04e62ec4148ecfe22be6a4484d9b1181a6ef32e205ebfd200540858b", size = 2468048, upload-time = "2026-10-11T18:33:54.118Z" }, + { url = "https://files.pythonhosted.org/packages/35/ca/e3832e9cf93651251de43c8c5c029ed680a3c1a0a1b17ee26c81bed08cbd/pydantic_core-2.50.1-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:c516cc5367ca3448995d42cb994bf3f4c9002d2a7c22eac9622551269ad1b807", size = 2512159, upload-time = "2026-10-11T18:33:55.99Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f2/773469b5a10a39116a2cb17edaf6d722f017dd8da07161b371465311c542/pydantic_core-2.50.1-cp314-cp314t-win32.whl", hash = "sha256:9d1bed94af6a63835461f3cf7502058eb166c58c4778e11d0f433cfb1bd69e19", size = 2098406, upload-time = "2026-10-11T18:33:58.043Z" }, + { url = "https://files.pythonhosted.org/packages/ee/42/0bb74f8f25204b259b11ab7c12dc7f180893b6bd706118b385147fb6efd5/pydantic_core-2.50.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c8dce1f1e0e5358b682a6ad3fa5e31b31d4560997b8e61417e9217c8d60f8a0c", size = 2148481, upload-time = "2026-10-11T18:33:59.913Z" }, + { url = "https://files.pythonhosted.org/packages/47/0d/d801646c9679a4e630e15cf521d4108b93854b42a6e6e02391bd4c6b1095/pydantic_core-2.50.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ceff0acc940be2715bd6ad17b24c0e5304abf44f6efd0f81ee8499e640f9dc86", size = 2174972, upload-time = "2026-10-11T18:34:01.875Z" }, + { url = "https://files.pythonhosted.org/packages/b2/84/23984b763d8862a02a13d27a44b6e8169428fd85ecfed88f54c29108604d/pydantic_core-2.50.1-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:8a6791afa2245e6c6b180122d105941644f5bd410bb18623b408808cc41a3102", size = 2224599, upload-time = "2026-10-11T18:34:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/4d/90/a63cf8586abc1d1a3f6d9b18f0224789ebf003851f092ebda3c7c863fd32/pydantic_core-2.50.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:84f34323a61a365b4e9295de6028474754829aaddd59c7bf1a040e7487ef8f3c", size = 2024191, upload-time = "2026-10-11T18:34:05.901Z" }, + { url = "https://files.pythonhosted.org/packages/a6/6a/f34bff9808ffb4907fbf5f5040457d253cacf2da7fce9efbc99ca1b1a44d/pydantic_core-2.50.1-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23edad659e8dbd8ca7e4e877fe6c81573abbdf215bd25a68b53e1272f58b80c7", size = 2048252, upload-time = "2026-10-11T18:34:07.757Z" }, + { url = "https://files.pythonhosted.org/packages/b2/54/13f419bf1eb59852003818e25935aaf175687d978f4c4bca70e08fe40a3b/pydantic_core-2.50.1-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a5fce22f1e87d181e924e12da7d81cfe031fb3881a5ddf26ad28f141756ca43", size = 2173376, upload-time = "2026-10-11T18:34:09.592Z" }, + { url = "https://files.pythonhosted.org/packages/6d/6c/b5a34d24cd0c81669d8f8339d74e6815abcf2f8fb48ab4b49b84c09be1d5/pydantic_core-2.50.1-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c73622ef819328873b53109ee4f77ceb598bffedd02daf916102be3228866b78", size = 2364559, upload-time = "2026-10-11T18:34:11.534Z" }, + { url = "https://files.pythonhosted.org/packages/eb/8d/d64d6216a8df365082665927ff923f183056f9049fee08e9777c9ac05296/pydantic_core-2.50.1-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ce8c25ca38cc0e3d7753ba180808de2c0c8cb24eae0df64491e40921454e9831", size = 2444697, upload-time = "2026-10-11T18:34:13.535Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0d/1b1149f60a00ea21ba5f70e28acbd40feb4598af414f80f53c921fac07c9/pydantic_core-2.50.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7689580e72a642ab5ec64d5f55b2e33636fa43b4ebe63c0c2c965ef307c7d1aa", size = 2159961, upload-time = "2026-10-11T18:34:15.56Z" }, + { url = "https://files.pythonhosted.org/packages/1e/35/f236549299dcc78e71e945495d7ec67e78d20844d0999c60601685003031/pydantic_core-2.50.1-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:d5c0e32fdbce7f1e8ef4d11f655694bf5f4175c757a9f1dc2be09b8864e5bcf5", size = 2234581, upload-time = "2026-10-11T18:34:17.502Z" }, + { url = "https://files.pythonhosted.org/packages/6a/85/26901a490522b7f75ef9bb9a7afb73e5bb550f0e1cb5b99a0069183e2eb9/pydantic_core-2.50.1-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:40f523349960fa30f3ea51404308ff50f9997a90df639590f47a057c1f32b415", size = 2244090, upload-time = "2026-10-11T18:34:19.402Z" }, + { url = "https://files.pythonhosted.org/packages/ca/2c/481bcfc70ceeb77a778ca6e5b705fe592cb64735c108157f81b7dd9c280e/pydantic_core-2.50.1-cp315-cp315-musllinux_1_1_aarch64.whl", hash = "sha256:d4193206b6587047437f6f11d7e776df23e1c1e23af2a54d9347275614791e10", size = 2277948, upload-time = "2026-10-11T18:34:21.317Z" }, + { url = "https://files.pythonhosted.org/packages/24/eb/f1e09333faa7ba447cde967310f758430cc7c5e987bdab5228816c70030d/pydantic_core-2.50.1-cp315-cp315-musllinux_1_1_armv7l.whl", hash = "sha256:84bc765b282a9d5b7fe0348b8648904f25a6a04b2139da52b1dd30c8ac3a2c8f", size = 2449688, upload-time = "2026-10-11T18:34:23.321Z" }, + { url = "https://files.pythonhosted.org/packages/d1/b3/036bde636db8f76d92996e81aefc75678ab5cec4a07eea1ad0c72a893fc3/pydantic_core-2.50.1-cp315-cp315-musllinux_1_1_x86_64.whl", hash = "sha256:ed1e728b39a383c81035b2459cfcb35d99dfb01f7d6ebe3a913bc1cc5b81e459", size = 2479286, upload-time = "2026-10-11T18:34:25.214Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a3/07f018294ee18d144afeb6df1a47d9e92960f014be45c5ed13519fa5af95/pydantic_core-2.50.1-cp315-cp315-win32.whl", hash = "sha256:bc94f474417604bd383d2cd445d071b07dd55fedceed3ce33407bf1fcc107290", size = 2103683, upload-time = "2026-10-11T18:34:27.42Z" }, + { url = "https://files.pythonhosted.org/packages/5f/98/f9bd7e1f9b6709f155acb9ef826d9c3884fe925f811fd8e55b9b52280bad/pydantic_core-2.50.1-cp315-cp315-win_amd64.whl", hash = "sha256:983a662de2571cb2502fc8ff47b6770b03d025d2eb314c92f77b3f07c74720ed", size = 2171681, upload-time = "2026-10-11T18:34:29.508Z" }, + { url = "https://files.pythonhosted.org/packages/10/87/4bb3e1e7f385c076ab5af4d6dd0571b22042cd8207eb810d5b9fef15ae31/pydantic_core-2.50.1-cp315-cp315-win_arm64.whl", hash = "sha256:94845ff54dc5193f228cab81b2662a04bfbb892e95bdc15edf7399000ce57d54", size = 2156251, upload-time = "2026-10-11T18:34:31.455Z" }, + { url = "https://files.pythonhosted.org/packages/47/47/83643225b08f2aef6c8cc4bbe6e3f79c5e139c4364a6e450d0f399d87774/pydantic_core-2.50.1-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:4a53d13cdfbedbfa87f08b83c1a0a5efcc767d785a4b41934fa9cb672670493a", size = 2234363, upload-time = "2026-10-11T18:34:33.65Z" }, + { url = "https://files.pythonhosted.org/packages/5e/66/127ca649ba2f2462039dc1e694c6c01e00a3689f023a617364d3507e6d97/pydantic_core-2.50.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:efbecf43d321f7b9281441f1f213f7c21c66988b0e06c2730ba13ed47a46bb08", size = 2000972, upload-time = "2026-10-11T18:34:36.037Z" }, + { url = "https://files.pythonhosted.org/packages/5e/4f/e421e0a5d653b1203b090b2e48745976988d7338a647940704b5b9c2b399/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc1f08f68dac9f9e83845a8039880aba2ab553eb9b2259c3243a313182c253fe", size = 2022925, upload-time = "2026-10-11T18:34:37.972Z" }, + { url = "https://files.pythonhosted.org/packages/d5/4e/ea5568e2491e1a71100f15ae8c2d01ef52db42a184a2d4e716bc79e5eb8f/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5dfe41f232befddb9c4377f6cfc702b51595e2d78ed082672adf8758d2c4619f", size = 2192867, upload-time = "2026-10-11T18:34:39.921Z" }, + { url = "https://files.pythonhosted.org/packages/ae/5e/3b8c3a35acbe219909ada5defad5d7d9fed845fb2b37bd5ec518f453c119/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:adc06d218a1cadfd2ec4628424d7d79ce4eba69c2965e7e7b55106f0da5208c8", size = 2388687, upload-time = "2026-10-11T18:34:42.184Z" }, + { url = "https://files.pythonhosted.org/packages/b6/aa/7889b4e515f91a2e8c0ae6b5081fec0feb30c4398d1434e14793c60f173a/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2cf91809d0721ab81592ba67bea7694821679c10b1a2e3c3460082b286c1918a", size = 2464415, upload-time = "2026-10-11T18:34:44.384Z" }, + { url = "https://files.pythonhosted.org/packages/08/78/93449e628eb8a6fdcce3eff9043081179d1bc7ce6f1bff32dc5006f41e00/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23923ab9292c40da026330b1ecf4dc2618c8e86e0422e5d1fbf50d94d64ca4f8", size = 2150823, upload-time = "2026-10-11T18:34:46.392Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f1/72c5bc129fceb0d00f05dc1e67f518c1728de1928c55f81fc13d7690de39/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:f3377c8c2b3ce898423c5e5dd94c7982e30aa7717a7e6ab2470b9de364963709", size = 2245806, upload-time = "2026-10-11T18:34:48.805Z" }, + { url = "https://files.pythonhosted.org/packages/28/2a/922a0e78f3aa6ab837b59f88190233fb8dad546c17995fde9bb3ed3b9b49/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:455a773617b5913bf5c20d0692e5787b119e52c4d40ea644ca31f5758fd31be2", size = 2264160, upload-time = "2026-10-11T18:34:50.876Z" }, + { url = "https://files.pythonhosted.org/packages/52/a8/0f1449e3e1b20941c9372faa18e7b6a092e30cdfa841902473f7989532ac/pydantic_core-2.50.1-cp315-cp315t-musllinux_1_1_aarch64.whl", hash = "sha256:1a9006395dece0e32e704c315eff8a00bede494f6108546cfc5539c89fef4f9a", size = 2294031, upload-time = "2026-10-11T18:34:52.876Z" }, + { url = "https://files.pythonhosted.org/packages/d1/d0/1031f492857de70355fb16524bbb03efce5fd34c93ed4a1ec60be07e0d4b/pydantic_core-2.50.1-cp315-cp315t-musllinux_1_1_armv7l.whl", hash = "sha256:d2d82aa62521c55ddfb000ae70f88cdd8de974078f6024e821dfe5addd0c818f", size = 2468423, upload-time = "2026-10-11T18:34:54.995Z" }, + { url = "https://files.pythonhosted.org/packages/46/52/269ffffa645b8e47906395a39cf9db1151ae9fa4bcd7f47b960bc31baf37/pydantic_core-2.50.1-cp315-cp315t-musllinux_1_1_x86_64.whl", hash = "sha256:009634b83993777ddcd69cad0ffcace43dabde692109528e35f0fde91e386a8b", size = 2512598, upload-time = "2026-10-11T18:34:57.149Z" }, + { url = "https://files.pythonhosted.org/packages/31/5c/e47e28281f20326ff6f3c31d626f0a83e615d2d94ba41bb0ad6ec237184a/pydantic_core-2.50.1-cp315-cp315t-win32.whl", hash = "sha256:3fde4fdc6487a58d944ca87cf5adc95d5f266e872c19599f5f4c0a8a1b1f9f9f", size = 2098535, upload-time = "2026-10-11T18:34:59.313Z" }, + { url = "https://files.pythonhosted.org/packages/03/ad/759e181e69c1b472c60b2049e5f61d5da1deefdd5ce1df4bb2ebcf771f25/pydantic_core-2.50.1-cp315-cp315t-win_amd64.whl", hash = "sha256:1c8632d4ac04e6f91128fca584b3a8a507d81604c24eeaaad00d4be42765c32b", size = 2148645, upload-time = "2026-10-11T18:35:01.571Z" }, + { url = "https://files.pythonhosted.org/packages/6d/56/8a702c27e5be9f47e5f19d8669227424290e4c024e7c370279cbaf244b4e/pydantic_core-2.50.1-cp315-cp315t-win_arm64.whl", hash = "sha256:c3ede305158e75510be50869b319550ab072008c13d64d4ab1e094fb286b6f44", size = 2175297, upload-time = "2026-10-11T18:35:04.079Z" }, + { url = "https://files.pythonhosted.org/packages/e7/73/a23a327237d9bb985298ce47702e196f995ce850224c12c70886bd66bb5c/pydantic_core-2.50.1-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:062e891facce5ca296a1c37098e5e466780457f86413894b399f0cf22934f769", size = 2162560, upload-time = "2026-10-11T18:35:06.274Z" }, + { url = "https://files.pythonhosted.org/packages/df/b2/33b37b5e82408f402a5e82649a43e7c95cd5dab7beaeef2c35cedbfd18f2/pydantic_core-2.50.1-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:49c2cbb2397fe4d0987e84606e691af6cb87bc0ee1bd3e7b737f7e10b4c142f9", size = 2075250, upload-time = "2026-10-11T18:35:08.455Z" }, + { url = "https://files.pythonhosted.org/packages/04/62/c7d5b9249466ccab90f3ce4790c2399dbfb8172ae9e74a7753dd295fe65a/pydantic_core-2.50.1-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e4472072de0137ee0d8e72d6620e85939c271d2f90f6bbb4b15c24638b79f92", size = 2107891, upload-time = "2026-10-11T18:35:10.918Z" }, + { url = "https://files.pythonhosted.org/packages/d4/7e/28d29bc1657195933147f772dee0558d9e0f045bbc11db7f992f40c37bb5/pydantic_core-2.50.1-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ed4f3cef55164b026fefb41341b7754cc6b624c75dfe7142d2ecceb5ad21c87", size = 2268026, upload-time = "2026-10-11T18:35:13.336Z" }, + { url = "https://files.pythonhosted.org/packages/a5/09/1bcf160f3cd6333e2a76982fae5963e5032de0bcaaba38464223dc266bfe/pydantic_core-2.50.1-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:76e2e83fa6ec8cdc972d438dafc2522b3a47bee4ec0ae668b29cfb1977ab5242", size = 2217070, upload-time = "2026-10-11T18:35:15.688Z" }, + { url = "https://files.pythonhosted.org/packages/3d/81/f02de95eec7794dfc7a5bc69a834f0e834b31b3009014ce6de903855f775/pydantic_core-2.50.1-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:a51eee75939cf811ac09b278745a6cee7dc873ccfbc8b9af3cc88fe4b7ce25b5", size = 2039027, upload-time = "2026-10-11T18:35:18.07Z" }, + { url = "https://files.pythonhosted.org/packages/4d/16/227746ed3771d9bf2304568b55134849ece34e13b38b1d76de763015c77f/pydantic_core-2.50.1-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae28183297fb0d2b8dc46a1f01d51f5e45825fc5afe76a835a6cb7fb34821295", size = 2109009, upload-time = "2026-10-11T18:35:20.218Z" }, + { url = "https://files.pythonhosted.org/packages/e5/b9/7664d1592a0e5a74903f357cd5b28ceb7336f3473256c5a4e8c8432f1881/pydantic_core-2.50.1-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88e492e8b9d0312e7dc13667c30222abf284dc3b79b5302b3607b41a5784ce61", size = 2269883, upload-time = "2026-10-11T18:35:22.48Z" }, + { url = "https://files.pythonhosted.org/packages/bd/35/002a378302ff91d2a7f149bc8d22363faf581ba40ac39d0123444315a32d/pydantic_core-2.50.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7456d699b13954e9c0164dcb267250a10ae0dfb03e6e26d6796ab0d46e189c84", size = 2154076, upload-time = "2026-10-11T18:35:24.743Z" }, + { url = "https://files.pythonhosted.org/packages/4b/5c/7a034330082c00f3c9795052bd9b161aaaf6e1951e897ac00a094823ef7e/pydantic_core-2.50.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b0d955195bbbe489ad343fcc956eacea9357b79cb22192c66cacdefcbc14b32f", size = 2085850, upload-time = "2026-10-11T18:35:27.009Z" }, + { url = "https://files.pythonhosted.org/packages/57/fa/d5b44a7d3d4eceed6ef8713a0003af56fefde6ec75915513bcdd5d730cd9/pydantic_core-2.50.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2c634642694e6a0dad2ab1d375589fa671fd442edd5caf7d9737b8f6ca22906", size = 2259754, upload-time = "2026-10-11T18:35:29.763Z" }, + { url = "https://files.pythonhosted.org/packages/b6/88/e01dd37301710bd8b3b2daf34ec9e2a6e83a333130831c0d468b0292a48c/pydantic_core-2.50.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ee6db2fbed51a7991302e8fac498cd67e336246026d0dfa84cf5166ce1412760", size = 2268712, upload-time = "2026-10-11T18:35:32.654Z" }, + { url = "https://files.pythonhosted.org/packages/3e/43/5bb80d4a4d2b2d84612a30205662cc521a20d209c94abc079ea48b4e0df2/pydantic_core-2.50.1-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:79490e33c4c0fcb933bbbcfc3a62184d8803b99f535863dfbb925e1bcb6945ad", size = 2305049, upload-time = "2026-10-11T18:35:35.392Z" }, + { url = "https://files.pythonhosted.org/packages/e8/7b/e3ef76b269cfe4a5bd3bdcf5d7124cf05e5a8909d443b91639307116bdae/pydantic_core-2.50.1-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:48569b0ade9edfbe065cad1d700175546592aebbb42f02adcebcc26e75b896fe", size = 2466001, upload-time = "2026-10-11T18:35:37.791Z" }, + { url = "https://files.pythonhosted.org/packages/8f/7a/ea0947aadc1d9f72d4be00d2e19939611ec17363bcb5e6eb9ec7d8221c34/pydantic_core-2.50.1-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:5f3cae32fc46121f787cb2486de9cf95a8bf72aec5cc78f64c606fa1735a6ef5", size = 2481777, upload-time = "2026-10-11T18:35:40.287Z" }, + { url = "https://files.pythonhosted.org/packages/24/f3/d15bc0b1fb0c4f1e07b3326ffb7489f7ae70da2b56a0884d22641b0eb47c/pydantic_core-2.50.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:7f476456ac2bb0d937f75191494a09c83a30765fea4f70f3b404942fe25f6cdf", size = 2336787, upload-time = "2026-10-11T18:35:42.558Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "ruff" +version = "0.17.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/59/73/81f140c4edd9cc323de5478e6bbd3bbcfae9a82692df1a258de8cec6a6d2/ruff-0.17.3.tar.gz", hash = "sha256:a57049c8addb3662073859bb55a7ad35aa54f8a7c9ed268911d1333c5aadaaa9", size = 5339444, upload-time = "2026-10-27T00:34:05.914Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/8c/083365310c4e79c842f25bf5a842facbd9b0c0ec336f96dffa5074fe723d/ruff-0.17.3-py3-none-linux_armv6l.whl", hash = "sha256:ab86b84cd9d05e56d329742c0cabdcbe2d1d56c98f4a04aab9b5b66196febfb4", size = 10216522, upload-time = "2026-10-27T00:33:29.822Z" }, + { url = "https://files.pythonhosted.org/packages/72/26/4b2edf31cfebe9605d8f6ed8f1d6f376d3a53d4c9a123e62661c64c440f0/ruff-0.17.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e407b4a6852853a9a2cdce96963e94ad9775578bb44e7561a0f2829f3cc04b11", size = 10217786, upload-time = "2026-10-27T00:33:33.949Z" }, + { url = "https://files.pythonhosted.org/packages/a2/7e/537aba4b7f54397a374d2f6a6fc93e9c8015e18553f31a4a7b7dbced48b5/ruff-0.17.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b75cc113b3aca388933d8296b667b1ac401a80317ea3074c9b0a66541199da29", size = 9440362, upload-time = "2026-10-27T00:33:35.784Z" }, + { url = "https://files.pythonhosted.org/packages/3d/66/283e39734f66c8a5ebaa93c22eb7b7515779a0b1d5a5c0ad20e05c0edb41/ruff-0.17.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab5fa7ed5ce18554ea69d850ad162cbaa0c85fc47c08d83a569f806681c9a6ad", size = 10094281, upload-time = "2026-10-27T00:33:38.216Z" }, + { url = "https://files.pythonhosted.org/packages/21/4c/1ffc768497bd025a32c72152771e40c0b99bddc5cc6e4f50c7949a395346/ruff-0.17.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c673c170e27834d1262e3a9c1e97bdc9b9c3b180c63d89173ce69747232474c1", size = 10022543, upload-time = "2026-10-27T00:33:40.181Z" }, + { url = "https://files.pythonhosted.org/packages/73/c3/f90db547301080fcb82c92abf914c8bd57b89911d9b65a6c4cc6b52d0635/ruff-0.17.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a754c0341bc9ca151b7b335988b3263921c08859f84ca219b70cee825571b6f", size = 10683921, upload-time = "2026-10-27T00:33:42.182Z" }, + { url = "https://files.pythonhosted.org/packages/e5/98/92d4a8a26c6c96cf1a5b34dd59036d8a71e52d650aab065c58867e964e63/ruff-0.17.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4590b9d86c0b1543ebb09094254e191cd43fe8edaf330baa035aab03da251532", size = 11373408, upload-time = "2026-10-27T00:33:44.19Z" }, + { url = "https://files.pythonhosted.org/packages/43/02/1f24d2e3064c9753926deeb901afa57777b37ee46799b09ddd7aee6e5145/ruff-0.17.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bbde18beccd98c29a227c63c99022e3443b99258e08326df6425dfe0443705ef", size = 10977469, upload-time = "2026-10-27T00:33:46.179Z" }, + { url = "https://files.pythonhosted.org/packages/f0/20/fecedea2209f574ab92033b344e01a7f0c2dc76c6d6d19a14c9f8141173f/ruff-0.17.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fa4b0f931cdbcb8a52044f370a3eb429fb42eb88807b0e800fcaa98dd42b24e", size = 10577491, upload-time = "2026-10-27T00:33:48.233Z" }, + { url = "https://files.pythonhosted.org/packages/33/36/b93fdf0bc6f84f0cd487aadfb0354deb40013669ea35ac3d26855de1e959/ruff-0.17.3-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:39405f341deec934b4289634324ad695f72df2970f9121582872eaf5b722f0f5", size = 10608748, upload-time = "2026-10-27T00:33:50.284Z" }, + { url = "https://files.pythonhosted.org/packages/5d/b7/7d3f5f14145daead1efb0b9a8289ed1daa100f419d60774375951723ce00/ruff-0.17.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:68074094f55ca3d29ab19491f640282f7a84061c4c7977aa5094b968ca457e9a", size = 10075057, upload-time = "2026-10-27T00:33:52.074Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ed/93ff7a5909ee56e2bd4b982deae9e0efb2724417874a90c9e41ad66c6dd7/ruff-0.17.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:27bb571ac78fef178fe5118d2827df95c2aae74a48fff2b15f6acd03af5ff5bb", size = 10022431, upload-time = "2026-10-27T00:33:54.061Z" }, + { url = "https://files.pythonhosted.org/packages/9b/07/a83b9eeec8c1c3c0e4eda15f06c3a91001bfd2687ea2f07b750719bf9712/ruff-0.17.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:029c37cbd3de48ca37aa12117bead2d065f15da17d66bf84e0d4274013705db2", size = 10394903, upload-time = "2026-10-27T00:33:56.158Z" }, + { url = "https://files.pythonhosted.org/packages/af/b2/05c39857c2526fcbebe46ad93a8dc97424656dfef14d3ef933abe1ffd331/ruff-0.17.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:692fdb46d0fd9c71e380947dc5fc72f2c3f881073bc1f739e4579376231affbe", size = 10799780, upload-time = "2026-10-27T00:33:58.204Z" }, + { url = "https://files.pythonhosted.org/packages/ec/d9/47171a16739eb7a353936b31c20cf54340531e68c9af7d72fd7a784e235d/ruff-0.17.3-py3-none-win32.whl", hash = "sha256:46539cb031b01fd29ae0a02142483c46640b233ee12bb1c1b690bffa1555d5ca", size = 9908481, upload-time = "2026-10-27T00:34:00.087Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a0/f08c1631bfd81faa6a90eaf70c37c786c11bed2046302b56bb506c25e1a0/ruff-0.17.3-py3-none-win_amd64.whl", hash = "sha256:56b8ffb3f3b3cbccb8b76b71b38261b7ed982ef5d20f5b447b1b61a7f4949b24", size = 10341333, upload-time = "2026-10-27T00:34:02.099Z" }, + { url = "https://files.pythonhosted.org/packages/99/d5/4ad4ed31413c301430918e70d3d38ae6f097af0766cb9ab8c9d502b2a4d6/ruff-0.17.3-py3-none-win_arm64.whl", hash = "sha256:81fce74bcd6be0e9da7c67c7553fd0f1f12964a2052cd728ef917b937a5ca1d9", size = 9854332, upload-time = "2026-10-27T00:34:03.847Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, +] diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index e97d7f961e..fd17b69fb2 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -574,6 +574,8 @@ fi HOST_PORT=$(e2e_pick_port) HEALTH_PORT=$(e2e_pick_port) +# Prometheus /metrics, scraped by the chaos harness. +METRICS_PORT=$(e2e_pick_port) STATE_DIR="${XDG_STATE_HOME}" mkdir -p "${STATE_DIR}" JWT_DIR="${STATE_DIR}/jwt" @@ -588,6 +590,16 @@ ensure_e2e_docker_network "${DOCKER_NETWORK_NAME}" export OPENSHELL_E2E_DOCKER_NETWORK_NAME="${DOCKER_NETWORK_NAME}" export OPENSHELL_E2E_NETWORK_NAME="${DOCKER_NETWORK_NAME}" export OPENSHELL_E2E_SANDBOX_NAMESPACE="${E2E_NAMESPACE}" +export OPENSHELL_E2E_METRICS_PORT="${METRICS_PORT}" +# Opt-in OTLP/gRPC trace export to 127.0.0.1:OPENSHELL_E2E_OTLP_PORT. The chaos +# harness runs the collector there and waits on the gateway's spans. +OTLP_PORT="" +if [ "${OPENSHELL_E2E_GATEWAY_OTLP:-0}" = "1" ]; then + OTLP_PORT=$(e2e_pick_port) + export OPENSHELL_E2E_OTLP_PORT="${OTLP_PORT}" + # Export finished spans within ~200ms instead of the SDK's 5s default. + export OTEL_BSP_SCHEDULE_DELAY="${OTEL_BSP_SCHEDULE_DELAY:-200}" +fi export OPENSHELL_E2E_DRIVER="docker" if [ "${ADDITIONAL_CA_MODE}" = "1" ]; then export OPENSHELL_E2E_ADDITIONAL_CA_ARTIFACT="${XDG_STATE_HOME}/openshell/network-supervisor" @@ -650,10 +662,15 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" printf 'additional_ca_bundle = %s\n\n' \ "$(toml_string "${ADDITIONAL_CA_DIR}/ca.crt")" fi - # Opt-in OTLP export so the otlp_relay e2e can bind a collector stub on - # the referenced port. Supervisors connect lazily, so the gateway may - # start before the stub exists. - if [ -n "${OPENSHELL_E2E_OTLP_ENDPOINT:-}" ]; then + # Opt-in OTLP export. The chaos harness (OPENSHELL_E2E_GATEWAY_OTLP=1) + # runs a collector on the picked port and waits on the gateway's spans. + # Otherwise OPENSHELL_E2E_OTLP_ENDPOINT lets the otlp_relay e2e bind a + # collector stub on the referenced port. Supervisors connect lazily, so + # the gateway may start before the stub exists. + if [ -n "${OTLP_PORT}" ]; then + printf '[openshell.gateway.otlp]\nendpoint = %s\n\n' \ + "$(toml_string "http://127.0.0.1:${OTLP_PORT}")" + elif [ -n "${OPENSHELL_E2E_OTLP_ENDPOINT:-}" ]; then printf '[openshell.gateway.otlp]\n' printf 'endpoint = %s\n\n' "$(toml_string "${OPENSHELL_E2E_OTLP_ENDPOINT}")" # The agent trace relay is opt-in; OPENSHELL_E2E_OTLP_AGENT=0 runs the @@ -710,6 +727,7 @@ GATEWAY_ARGS=( --bind-address "${GATEWAY_BIND_IP}" --port "${HOST_PORT}" --health-port "${HEALTH_PORT}" + --metrics-port "${METRICS_PORT}" --compute-driver docker --tls-cert "${PKI_DIR}/server/tls.crt" --tls-key "${PKI_DIR}/server/tls.key" diff --git a/tasks/test.toml b/tasks/test.toml index 30e8982599..68f1fb4d72 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -194,6 +194,31 @@ run = [ "OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE=openshell/e2e-python:dev OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE_PULL_POLICY=never e2e/with-docker-gateway.sh uv run pytest -o python_files='test_*.py *_test.py' -m 'not gpu' -n ${E2E_PARALLEL:-5} e2e/python", ] +["e2e:chaos"] +description = "Run one chaos experiment against a Docker-backed gateway (pass the file: mise run e2e:chaos e2e/chaos/experiments/smoke.yaml)" +env = { UV_PYTHON_INSTALL_DIR = "{{env.HOME}}/.local/share/uv/python", OPENSHELL_E2E_GATEWAY_OTLP = "1" } +run = "e2e/with-docker-gateway.sh uv run --project e2e/chaos chaos run" + +["e2e:chaos:preflight"] +description = "Run only an experiment's preflight checks against a Docker-backed gateway (pass the file)" +env = { UV_PYTHON_INSTALL_DIR = "{{env.HOME}}/.local/share/uv/python", OPENSHELL_E2E_GATEWAY_OTLP = "1" } +run = "e2e/with-docker-gateway.sh uv run --project e2e/chaos chaos preflight" + +["e2e:chaos:check"] +description = "Run the hello-world and smoke chaos experiments against a Docker-backed gateway and fail unless both verdicts are PASS" +env = { UV_PYTHON_INSTALL_DIR = "{{env.HOME}}/.local/share/uv/python", OPENSHELL_E2E_GATEWAY_OTLP = "1" } +run = "e2e/with-docker-gateway.sh e2e/chaos/scripts/check.sh" + +["e2e:chaos:share"] +description = "Write a Markdown page of a chaos run with Mermaid charts, to paste into an issue or a post (pass the run dir; --png also renders the charts with mermaid-cli)" +env = { UV_PYTHON_INSTALL_DIR = "{{env.HOME}}/.local/share/uv/python" } +run = "uv run --project e2e/chaos chaos share" + +["e2e:chaos:grafana"] +description = "Load chaos runs into a local Prometheus and Grafana (pass run dirs: mise run e2e:chaos:grafana e2e/chaos/runs/smoke/; --down stops it)" +env = { UV_PYTHON_INSTALL_DIR = "{{env.HOME}}/.local/share/uv/python" } +run = "e2e/chaos/grafana/view.sh" + ["e2e:podman"] description = "Run Rust CLI e2e tests against a Podman-backed gateway" run = "e2e/rust/e2e-podman.sh"