From 2e507cf45a6539c8ca0684ae564b6390132309db Mon Sep 17 00:00:00 2001 From: Edward Viaene Date: Wed, 18 Nov 2026 22:42:01 -0600 Subject: [PATCH 1/6] Run folder names always pass the server's file name rule A case slug never starts with a dot, and attachment and part names keep ASCII [A-Za-z0-9._-] only (str.isalnum() let other letters through). Co-Authored-By: Claude Opus 5.5 --- .../embeddedci/src/embeddedci/benchpod/runs/active.py | 10 +++++++++- .../embeddedci/src/embeddedci/benchpod/runs/schema.py | 5 ++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/embeddedci/src/embeddedci/benchpod/runs/active.py b/packages/embeddedci/src/embeddedci/benchpod/runs/active.py index fcf04aa..d0ab138 100644 --- a/packages/embeddedci/src/embeddedci/benchpod/runs/active.py +++ b/packages/embeddedci/src/embeddedci/benchpod/runs/active.py @@ -102,8 +102,16 @@ def default_run_dir(started: float, suite: str = "", base: Union[str, "os.PathLi return Path(base) / "embeddedci-runs" / f"{folder_stamp(started)}-{case_slug_simple(suite or 'pytest')}" +_SIMPLE_OK = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-") + + def case_slug_simple(text: str) -> str: - return "".join(c if c.isalnum() or c in "._-" else "_" for c in text)[:80] or "pytest" + """``text`` as a short file name part: ASCII ``[A-Za-z0-9._-]`` only (the server's file name + rule), every other character ``_``, no leading dot, at most 80 characters.""" + out = "".join(c if c in _SIMPLE_OK else "_" for c in text)[:80] + if out.startswith("."): + out = "_" + out[1:] + return out or "pytest" @dataclass diff --git a/packages/embeddedci/src/embeddedci/benchpod/runs/schema.py b/packages/embeddedci/src/embeddedci/benchpod/runs/schema.py index 7b21445..dfc0413 100644 --- a/packages/embeddedci/src/embeddedci/benchpod/runs/schema.py +++ b/packages/embeddedci/src/embeddedci/benchpod/runs/schema.py @@ -51,8 +51,11 @@ def case_slug(case_id: str) -> str: """The file name stem for a case id: every character outside ``[A-Za-z0-9._-]`` replaced by - ``_``, at most 120 characters, plus ``-`` when it was cut or changed.""" + ``_`` (a leading ``.`` too), at most 120 characters, plus ``-`` when it + was cut or changed.""" slug = _SLUG_BAD.sub("_", case_id) + if slug.startswith("."): # a file name may not start with a dot (hidden; the server refuses it) + slug = "_" + slug[1:] changed = slug != case_id if len(slug) > SLUG_MAX_CHARS: slug = slug[:SLUG_MAX_CHARS] From 4e9f33df62e6684fec0f1a3b3af8a9fea961ac24 Mon Sep 17 00:00:00 2001 From: Edward Viaene Date: Wed, 18 Nov 2026 22:42:01 -0600 Subject: [PATCH 2/6] Upload test runs; build_report links its run to the build --embeddedci-upload (ini embeddedci_upload, EMBEDDEDCI_UPLOAD=1) uploads the recorded run at session end: POST /api/runs, then junit.xml, the traces of failed and error cases with their attachments, and logs/. Credentials: BENCHPOD_API_KEY, else the benchpod login session, else GitHub Actions OIDC. A failed upload is a warning; under benchpod ci the runner uploads instead. In GitHub Actions a session whose tests use build_report records its run and uploads it with inputs.dut_firmware.build_id set to the build the fixture created, using the reporter's credentials. The build status call stays for older servers. BuildReporter gains api_base and session_token. Co-Authored-By: Claude Opus 5.5 --- .../embeddedci/src/embeddedci/benchpod/ci.py | 15 +- .../src/embeddedci/benchpod/pytest_plugin.py | 9 + .../embeddedci/benchpod/runs/pytest_runs.py | 87 ++++++++- .../src/embeddedci/benchpod/runs/upload.py | 167 ++++++++++++++++++ packages/embeddedci/tests/api_surface.json | 2 + 5 files changed, 277 insertions(+), 3 deletions(-) create mode 100644 packages/embeddedci/src/embeddedci/benchpod/runs/upload.py diff --git a/packages/embeddedci/src/embeddedci/benchpod/ci.py b/packages/embeddedci/src/embeddedci/benchpod/ci.py index 5ab9ce4..e57c112 100644 --- a/packages/embeddedci/src/embeddedci/benchpod/ci.py +++ b/packages/embeddedci/src/embeddedci/benchpod/ci.py @@ -94,6 +94,16 @@ def active(self) -> bool: def build_id(self) -> Optional[str]: return self._build_id + @property + def api_base(self) -> str: + """The embeddedci server this reporter talks to.""" + return self._api_base + + @property + def session_token(self) -> str: + """The session token the reporter uses (the test run is uploaded with it too).""" + return self._token + # -- public API used by tests ------------------------------------------------- def record_wiring( @@ -169,7 +179,10 @@ def set_result(self, success: bool, reason: str = "") -> None: self._result = (bool(success), reason) def finalize(self) -> None: - """Create the build if needed and post the final pytest status. Idempotent.""" + """Create the build if needed and post the final pytest status. Idempotent. + + The status call is kept for servers that predate test runs; newer servers keep the build's + own status and show the test results from the run linked to the build instead.""" if self._finalized: return self._finalized = True diff --git a/packages/embeddedci/src/embeddedci/benchpod/pytest_plugin.py b/packages/embeddedci/src/embeddedci/benchpod/pytest_plugin.py index 7f7b00d..8bb05f6 100644 --- a/packages/embeddedci/src/embeddedci/benchpod/pytest_plugin.py +++ b/packages/embeddedci/src/embeddedci/benchpod/pytest_plugin.py @@ -533,6 +533,10 @@ def test_boots(benchpod, firmware, build_report): ... # the pytest pass/fail is captured automatically The build status (pass/fail) is captured automatically from the test result at teardown. + + The session's test run is recorded too and uploaded at the end, linked to the build: every + test's outcome, measurements and pod-call timings, and the traces of the failed tests (see + ``--embeddedci-upload``). The build page shows it under Tests. """ from .ci import make_build_reporter @@ -556,6 +560,11 @@ def test_boots(benchpod, firmware, build_report): if log_text: reporter.upload_logs("pytest.log", log_text) reporter.finalize() + # The session's test run is uploaded at the end, linked to this build. + runs = request.config.pluginmanager.get_plugin(_runs.PLUGIN_NAME) + build_id = getattr(reporter, "build_id", None) + if runs is not None and getattr(reporter, "active", False) and build_id: + runs.link_build(build_id, reporter.api_base, reporter.session_token) def _collect_pytest_log(node: "pytest.Item") -> str: diff --git a/packages/embeddedci/src/embeddedci/benchpod/runs/pytest_runs.py b/packages/embeddedci/src/embeddedci/benchpod/runs/pytest_runs.py index 1bb610b..c0c8ca6 100644 --- a/packages/embeddedci/src/embeddedci/benchpod/runs/pytest_runs.py +++ b/packages/embeddedci/src/embeddedci/benchpod/runs/pytest_runs.py @@ -38,6 +38,8 @@ ) PLUGIN_NAME = "embeddedci-runs" +#: ``EMBEDDEDCI_UPLOAD=1`` uploads the recorded run (like ``--embeddedci-upload``). +UPLOAD_ENV = "EMBEDDEDCI_UPLOAD" #: ``--embeddedci-run-dir auto``: the default folder, ``./embeddedci-runs/-/``. AUTO_RUN_DIR = "auto" @@ -59,6 +61,15 @@ def add_options(parser: "pytest.Parser") -> None: "--embeddedci-dut", action="store", default=None, dest="embeddedci_dut", help="Name of the board under test recorded in run.json (key.dut; default '').", ) + group.addoption( + "--embeddedci-upload", action="store_true", default=False, dest="embeddedci_upload", + help="Upload the recorded run to embeddedci.com at the end (records into " + "./embeddedci-runs/ when --embeddedci-run-dir is not given). Credentials: BENCHPOD_API_KEY, " + "else the benchpod login session, else GitHub Actions OIDC. Also the embeddedci_upload ini " + "option and EMBEDDEDCI_UPLOAD=1.", + ) + parser.addini("embeddedci_upload", type="bool", default=False, + help="Upload the recorded run to embeddedci.com (see --embeddedci-upload).") group.addoption( "--embeddedci-keep-traces", action="store", default=None, dest="embeddedci_keep_traces", choices=list(KEEP_POLICIES), @@ -67,6 +78,13 @@ def add_options(parser: "pytest.Parser") -> None: ) +def _ini_bool(config: Any, name: str) -> bool: + try: + return bool(config.getini(name)) + except (ValueError, KeyError): + return False + + # -- measure ------------------------------------------------------------------------------------- class Measure: @@ -220,6 +238,13 @@ def __init__(self, config: "pytest.Config") -> None: self.dut = "" self.record: Optional[Dict[str, Any]] = None self._reports: Dict[str, Dict[str, Any]] = {} + #: Upload the run at the end (--embeddedci-upload, the ini option, EMBEDDEDCI_UPLOAD=1). + self.upload = False + #: The build a build_report fixture created: (build id, api base, session token). + self.build_link: Optional[Tuple[str, str, str]] = None + #: The upload's result ({"id", "url", "build_id"}) or the warning it ended with. + self.uploaded: Optional[Dict[str, Any]] = None + self.upload_warning = "" # -- start -------------------------------------------------------------------------------- @@ -236,15 +261,19 @@ def _settings(self) -> Optional[Dict[str, Any]]: or os.environ.get(KEEP_TRACES_ENV, "").strip().lower() or "all") if keep not in KEEP_POLICIES: raise pytest.UsageError(f"{KEEP_TRACES_ENV} must be one of {', '.join(KEEP_POLICIES)}, got {keep!r}") + self.upload = bool(config.getoption("embeddedci_upload", False) + or _ini_bool(config, "embeddedci_upload") + or os.environ.get(UPLOAD_ENV, "").strip().lower() in ("1", "true", "yes")) t0_text = os.environ.get(RUN_T0_ENV, "").strip() try: t0 = float(t0_text) if t0_text else None except ValueError: raise pytest.UsageError(f"{RUN_T0_ENV} must be unix seconds, got {t0_text!r}") from None if env_dir and env_step: + self.upload = False # benchpod ci uploads the whole run itself return {"run_dir": env_dir, "step": env_step, "handover": True, "keep": keep, "t0": t0, "run_id": os.environ.get(RUN_ID_ENV, "").strip()} - run_dir = opt_dir or env_dir + run_dir = opt_dir or env_dir or (AUTO_RUN_DIR if self.upload else "") if not run_dir: return None return {"run_dir": run_dir, "step": "pytest", "handover": False, "keep": keep, "t0": t0, @@ -253,8 +282,34 @@ def _settings(self) -> Optional[Dict[str, Any]]: @pytest.hookimpl(tryfirst=True) def pytest_sessionstart(self, session: "pytest.Session") -> None: settings = self._settings() - if settings is None: + if settings is not None: + self._start(settings) + + def pytest_collection_finish(self, session: "pytest.Session") -> None: + """A suite whose tests use ``build_report`` records its run in GitHub Actions (when nothing + else asked for it), so the run can be uploaded linked to the build the fixture reports.""" + if self.run is not None or hasattr(self.config, "workerinput"): + return + if os.environ.get(RUN_STEP_ENV, "").strip(): return + from ..ci import running_in_github_actions + + if not running_in_github_actions(): + return + if not any("build_report" in getattr(item, "fixturenames", ()) for item in session.items): + return + self._settings() + self._start({"run_dir": AUTO_RUN_DIR, "step": "pytest", "handover": False, + "keep": os.environ.get(KEEP_TRACES_ENV, "").strip().lower() or "failed", + "t0": None, "run_id": ""}) + + def link_build(self, build_id: str, api_base: str, token: str) -> None: + """The ``build_report`` fixture created ``build_id``: link the run to it (the first one) and + upload it at the end with the fixture's credentials.""" + if build_id and self.build_link is None: + self.build_link = (build_id, api_base, token) + + def _start(self, settings: Dict[str, Any]) -> None: run_dir = settings["run_dir"] if run_dir == AUTO_RUN_DIR: import time @@ -335,8 +390,32 @@ def pytest_sessionfinish(self, session: "pytest.Session", exitstatus: int) -> No "tools": tools, "github": provenance.github_info(), } + if self.build_link is not None: + inputs["dut_firmware"] = {**inputs["dut_firmware"], "build_id": self.build_link[0]} key = {"suite": self.suite, "dut": self.dut, "definition_sha256": ""} self.record = run.write_run(cases=self.cases, key=key, inputs=inputs, suite=self.suite) + if self.upload or self.build_link is not None: + self._upload(run) + + def _upload(self, run: RunRecording) -> None: + from .upload import RunUploadError, upload_run + + problems: List[str] = [] + api_base = (self.config.getoption("benchpod_api_base", None) + or os.environ.get("BENCHPOD_API_BASE") or None) + token = None + build_id = None + if self.build_link is not None: + build_id, base, token = self.build_link + api_base = api_base or base + try: + self.uploaded = upload_run(run.run_dir, api_base=api_base, token=token, + build_id=build_id, warn=problems.append) + except RunUploadError as exc: + self.upload_warning = str(exc) + return + if problems: + self.upload_warning = "some files were not uploaded: " + "; ".join(problems[:5]) def pytest_terminal_summary(self, terminalreporter: Any) -> None: run = self.run @@ -347,6 +426,10 @@ def pytest_terminal_summary(self, terminalreporter: Any) -> None: elif self.record is not None: outcome = self.record["results"]["outcome"] terminalreporter.write_line(f"embeddedci: run {outcome}, recorded in {run.run_dir}") + if self.uploaded is not None: + terminalreporter.write_line(f"embeddedci: run uploaded: {self.uploaded.get('url') or self.uploaded.get('id')}") + if self.upload_warning: + terminalreporter.write_line(f"embeddedci: warning: run upload: {self.upload_warning}", yellow=True) def pytest_unconfigure(self, config: "pytest.Config") -> None: if self.run is not None and self.run.active: diff --git a/packages/embeddedci/src/embeddedci/benchpod/runs/upload.py b/packages/embeddedci/src/embeddedci/benchpod/runs/upload.py new file mode 100644 index 0000000..0c2410b --- /dev/null +++ b/packages/embeddedci/src/embeddedci/benchpod/runs/upload.py @@ -0,0 +1,167 @@ +"""Upload a run folder to embeddedci.com: ``POST /api/runs`` with ``run.json``, then its files with +``POST /api/runs/{id}/files?name=``. + +What goes up: ``run.json``, ``junit.xml``, the traces of the failed and error cases (the ``failed`` +keep policy, whatever the folder kept locally) with the attachments those traces point to, and +``logs/``. The uploaded ``run.json`` says so: its ``keep.traces`` is ``failed`` (or ``none``) and the +cases whose trace stays local have ``trace: ""``. The folder on disk is not changed. +""" + +from __future__ import annotations + +import json +import os +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Union + +from embeddedci import _http + +from ..errors import BenchPodError + +_HTTP_TIMEOUT = 60.0 +#: The case outcomes whose traces are uploaded. +_UPLOADED_TRACES = frozenset({"failed", "error"}) + + +class RunUploadError(BenchPodError): + """Uploading a run failed (no credentials, the server refused, the network).""" + + +def upload_files(run_dir: Path, record: Dict[str, Any]) -> List[str]: + """The files of ``run_dir`` to upload next to ``record``, as names relative to the folder.""" + names: List[str] = [] + if (run_dir / "junit.xml").is_file(): + names.append("junit.xml") + for case in record.get("results", {}).get("cases", []) or []: + trace = case.get("trace") or "" + if not trace or case.get("outcome") not in _UPLOADED_TRACES: + continue + path = run_dir / trace + if not path.is_file(): + continue + names.append(trace) + names.extend(sorted(_attachments(path) - set(names))) + logs = run_dir / "logs" + if logs.is_dir(): + names.extend(f"logs/{p.name}" for p in sorted(logs.iterdir()) if p.is_file()) + return [n for n in names if (run_dir / n).is_file()] + + +def _attachments(trace: Path) -> Set[str]: + out: Set[str] = set() + try: + with trace.open("r", encoding="utf-8") as f: + for line in f: + if '"att"' not in line: + continue + try: + att = json.loads(line).get("att") + except ValueError: + continue + if isinstance(att, str) and att.startswith("att/"): + out.add(att) + except OSError: + pass + return out + + +def upload_record(record: Dict[str, Any], *, build_id: Optional[str] = None) -> Dict[str, Any]: + """The ``run.json`` as uploaded: traces only for failed and error cases, and ``build_id`` (when + given) in ``inputs.dut_firmware`` so the server links the run to that build.""" + doc = json.loads(json.dumps(record)) + keep = (doc.get("keep") or {}).get("traces", "all") + if keep == "all": + doc["keep"] = {"traces": "failed"} + for case in doc.get("results", {}).get("cases", []) or []: + if case.get("outcome") not in _UPLOADED_TRACES: + case["trace"] = "" + if build_id: + inputs = doc.setdefault("inputs", {}) + firmware = inputs.get("dut_firmware") + if not isinstance(firmware, dict): + firmware = {} + firmware["build_id"] = str(build_id) + inputs["dut_firmware"] = firmware + return doc + + +def _token(api_base: str) -> str: + """A credential for the runs API: ``BENCHPOD_API_KEY``, else the ``benchpod login`` session, + else GitHub Actions OIDC (see :func:`~embeddedci.benchpod.cloud_auth.mint_session_token`).""" + from ..cloud_auth import mint_session_token + + try: + return mint_session_token(api_base)[0] + except BenchPodError as exc: + raise RunUploadError(f"no credentials to upload the run: {exc}") from exc + + +def _request(url: str, token: str, body: bytes, content_type: str) -> Dict[str, Any]: + from ..cloud_auth import USER_AGENT + + req = urllib.request.Request(url, data=body, method="POST", headers={ + "Authorization": f"Bearer {token}", "Content-Type": content_type, + "Accept": "application/json", "User-Agent": USER_AGENT}) + try: + with _http.urlopen(req, timeout=_HTTP_TIMEOUT) as resp: + raw = resp.read().decode("utf-8") + except urllib.error.HTTPError as exc: + msg = exc.read().decode("utf-8", "replace") + try: + msg = json.loads(msg).get("error", msg) + except ValueError: + pass + raise RunUploadError(f"run upload failed (HTTP {exc.code}): {msg}") from exc + except Exception as exc: # noqa: BLE001 - network, TLS + raise RunUploadError(f"run upload failed: {exc}") from exc + try: + data = json.loads(raw) if raw.strip() else {} + except ValueError: + data = {} + return data if isinstance(data, dict) else {} + + +def upload_run(run_dir: Union[str, "os.PathLike[str]"], *, api_base: Optional[str] = None, + token: Optional[str] = None, build_id: Optional[str] = None, + warn: Optional[Callable[[str], None]] = None) -> Dict[str, Any]: + """Upload the run folder ``run_dir``. Returns the server's ``{"id", "url", "build_id"}``. + + ``token`` is the credential (default: ``BENCHPOD_API_KEY``, else the ``benchpod login`` + session, else GitHub Actions OIDC); ``api_base`` the server (default ``BENCHPOD_API_BASE``, + else ``https://www.embeddedci.com``). A file the server refuses is reported to ``warn`` and + skipped; a refused ``run.json`` raises :class:`RunUploadError`.""" + from ..cloud_auth import DEFAULT_API_BASE + + folder = Path(run_dir) + try: + record = json.loads((folder / "run.json").read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise RunUploadError(f"{folder}: no readable run.json ({exc})") from exc + base = (api_base or os.environ.get("BENCHPOD_API_BASE") or DEFAULT_API_BASE).rstrip("/") + tok = token or _token(base) + doc = upload_record(record, build_id=build_id) + created = _request(f"{base}/api/runs", tok, json.dumps(doc).encode("utf-8"), "application/json") + run_id = str(created.get("id") or "") + if not run_id: + raise RunUploadError("run upload: the server returned no run id") + for name in upload_files(folder, doc): + url = f"{base}/api/runs/{urllib.parse.quote(run_id, safe='')}/files?name={urllib.parse.quote(name, safe='')}" + try: + _request(url, tok, (folder / name).read_bytes(), "application/octet-stream") + except (RunUploadError, OSError) as exc: + if warn is not None: + warn(f"{name}: {exc}") + return created + + +def names_valid(names: Iterable[str]) -> bool: + """Whether every name passes the server's file name rule (``run.json``, ``junit.xml``, or one + level of ``trace/``, ``att/``, ``logs/`` with ``[A-Za-z0-9._-]`` names not starting with a dot, + at most 200 characters).""" + import re + + rule = re.compile(r"^(?:run\.json|junit\.xml|(?:trace|att|logs)/[A-Za-z0-9_-][A-Za-z0-9._-]*)$") + return all(len(n) <= 200 and rule.match(n) for n in names) diff --git a/packages/embeddedci/tests/api_surface.json b/packages/embeddedci/tests/api_surface.json index 42726b2..4a4e04f 100644 --- a/packages/embeddedci/tests/api_surface.json +++ b/packages/embeddedci/tests/api_surface.json @@ -204,9 +204,11 @@ "BuildReporter": { "__init__": "(self, *, api_base: 'str', session_token: 'str', metadata: 'Dict[str, str]', target: 'str' = '', name: 'str' = '') -> 'None'", "active": "property", + "api_base": "property", "build_id": "property", "finalize": "(self) -> 'None'", "record_wiring": "(self, *, target: 'Optional[str]' = None, swclk: 'Optional[int]' = None, swdio: 'Optional[int]' = None, nreset: 'Optional[bool]' = None, efuse: 'Optional[int]' = None) -> 'None'", + "session_token": "property", "set_result": "(self, success: 'bool', reason: 'str' = '') -> 'None'", "upload_artifact": "(self, path: 'str') -> 'None'", "upload_artifacts": "(self, paths: 'Iterable[str]') -> 'None'", From f8118ceb2aa7ee73f45131e165b9a09dda5fc28e Mon Sep 17 00:00:00 2001 From: Edward Viaene Date: Wed, 18 Nov 2026 22:42:01 -0600 Subject: [PATCH 3/6] Tests for run upload, build_report runs and file names A fake runs API: only failed traces and their attachments go up, the upload order, a refused upload, the plugin option and env, a failed upload as a warning, no upload under benchpod ci, build_report linking the run to its build, and trace/attachment names against the server's rule. Co-Authored-By: Claude Opus 5.5 --- packages/embeddedci/tests/test_runs_upload.py | 316 ++++++++++++++++++ 1 file changed, 316 insertions(+) create mode 100644 packages/embeddedci/tests/test_runs_upload.py diff --git a/packages/embeddedci/tests/test_runs_upload.py b/packages/embeddedci/tests/test_runs_upload.py new file mode 100644 index 0000000..470912d --- /dev/null +++ b/packages/embeddedci/tests/test_runs_upload.py @@ -0,0 +1,316 @@ +"""Uploading a run folder (``--embeddedci-upload``) and ``build_report`` linking the run to its +build, against an in-process fake runs API.""" + +from __future__ import annotations + +import json +import threading +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Dict, List + +import pytest + +from embeddedci.benchpod.runs import case_slug, set_active_run +from embeddedci.benchpod.runs import upload as upload_mod +from embeddedci.benchpod.runs.upload import ( + RunUploadError, + names_valid, + upload_files, + upload_record, + upload_run, +) + +RUN_ENV = ("EMBEDDEDCI_RUN_DIR", "EMBEDDEDCI_RUN_STEP", "EMBEDDEDCI_RUN_ID", "EMBEDDEDCI_RUN_T0", + "EMBEDDEDCI_KEEP_TRACES", "EMBEDDEDCI_UPLOAD", "GITHUB_ACTIONS", "BENCHPOD_API_BASE") + + +@pytest.fixture(autouse=True) +def _no_env(monkeypatch): + for name in RUN_ENV: + monkeypatch.delenv(name, raising=False) + yield + set_active_run(None) + + +class RunsAPI: + """POST /api/runs and /api/runs/{id}/files; ``fail`` answers run creation with HTTP 500.""" + + def __init__(self) -> None: + self.requests: List[Dict[str, Any]] = [] + self.fail = False + api = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *a: Any) -> None: + pass + + def do_POST(self) -> None: # noqa: N802 - http.server naming + body = self.rfile.read(int(self.headers.get("Content-Length") or 0)) + url = urllib.parse.urlsplit(self.path) + api.requests.append({"path": url.path, "query": urllib.parse.parse_qs(url.query), + "auth": self.headers.get("Authorization"), "body": body}) + if url.path == "/api/runs": + if api.fail: + self._send(500, {"error": "database down"}) + return + doc = json.loads(body) + self._send(201, {"id": "r-1", "url": "https://example.test/run/r-1", + "build_id": doc["inputs"].get("dut_firmware", {}).get("build_id", "")}) + elif url.path == "/api/runs/r-1/files": + name = api.requests[-1]["query"]["name"][0] + if not names_valid([name]): + self._send(400, {"error": "bad name"}) + else: + self._send(201, {"name": name, "size_bytes": len(body)}) + else: + self._send(404, {"error": "no route"}) + + def _send(self, code: int, obj: Dict[str, Any]) -> None: + data = json.dumps(obj).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.base = "http://127.0.0.1:%d" % self.server.server_address[1] + threading.Thread(target=self.server.serve_forever, daemon=True).start() + + def run_doc(self) -> Dict[str, Any]: + return json.loads(next(r for r in self.requests if r["path"] == "/api/runs")["body"]) + + def file_names(self) -> List[str]: + return [r["query"]["name"][0] for r in self.requests if r["path"].endswith("/files")] + + def close(self) -> None: + self.server.shutdown() + + +@pytest.fixture +def api(): + a = RunsAPI() + yield a + a.close() + + +def _folder(tmp_path: Path) -> Path: + run = tmp_path / "run" + (run / "trace").mkdir(parents=True) + (run / "att").mkdir() + (run / "logs").mkdir() + ok_trace, bad_trace = f"trace/{case_slug('t.py::test_ok')}.ndjson", f"trace/{case_slug('t.py::test_bad')}.ndjson" + (run / ok_trace).write_text('{"t":1,"op":"command","cmd":"big","att":"att/0001-big.json"}\n') + (run / bad_trace).write_text('{"t":2,"op":"command","cmd":"x","ok":true}\n' + '{"t":3,"op":"samples","cmd":"la","att":"att/0002-la.json"}\n') + (run / "att" / "0001-big.json").write_text("{}") + (run / "att" / "0002-la.json").write_text("{}") + (run / "logs" / "flash.log").write_text("ok") + (run / "junit.xml").write_text("") + record = {"schema_version": 1, "keep": {"traces": "all"}, + "inputs": {"dut_firmware": {}}, + "results": {"cases": [ + {"id": "t.py::test_ok", "outcome": "passed", "trace": ok_trace}, + {"id": "t.py::test_bad", "outcome": "failed", "trace": bad_trace}, + {"id": "t.py::test_skip", "outcome": "skipped", "trace": ""}]}} + (run / "run.json").write_text(json.dumps(record)) + return run + + +def test_only_failed_traces_and_their_attachments_go_up(tmp_path): + run = _folder(tmp_path) + record = json.loads((run / "run.json").read_text()) + doc = upload_record(record, build_id="b-9") + assert doc["keep"] == {"traces": "failed"} + cases = {c["id"]: c for c in doc["results"]["cases"]} + assert cases["t.py::test_ok"]["trace"] == "" and cases["t.py::test_bad"]["trace"] + assert doc["inputs"]["dut_firmware"]["build_id"] == "b-9" + assert record["results"]["cases"][0]["trace"] # the input is not changed + names = upload_files(run, doc) + assert names == ["junit.xml", cases["t.py::test_bad"]["trace"], "att/0002-la.json", "logs/flash.log"] + assert names_valid(names) + + +@pytest.mark.parametrize("case_id", [ + "tests/test_can_hil.py::test_ack", + "tests/test_a.py::TestX::test_param[1.8 V-ünïcode/../x]", + "x" * 500, + ".hidden", + "", +]) +def test_trace_names_pass_the_server_rule(case_id): + assert names_valid([f"trace/{case_slug(case_id)}.ndjson", f"trace/{case_slug(case_id)}-2.ndjson"]) + + +def test_server_rule_refuses_bad_names(): + assert not names_valid(["trace/.x", "trace/a/b", "parts/hil.json", "trace/a b", "x" * 201]) + assert names_valid(["run.json", "junit.xml", "trace/_session.ndjson", "att/0001-la.json.zst"]) + + +def test_upload_run_posts_the_run_then_its_files(tmp_path, api): + run = _folder(tmp_path) + got = upload_run(run, api_base=api.base, token="tok") + assert got == {"id": "r-1", "url": "https://example.test/run/r-1", "build_id": ""} + assert api.requests[0]["path"] == "/api/runs" and api.requests[0]["auth"] == "Bearer tok" + assert api.run_doc()["keep"] == {"traces": "failed"} + assert api.file_names() == upload_files(run, api.run_doc()) + + +def test_upload_run_refused(tmp_path, api): + api.fail = True + with pytest.raises(RunUploadError, match="HTTP 500.*database down"): + upload_run(_folder(tmp_path), api_base=api.base, token="tok") + with pytest.raises(RunUploadError, match="run.json"): + upload_run(tmp_path / "missing", api_base=api.base, token="tok") + + +# -- the pytest plugin ----------------------------------------------------------------------------- + +SUITE = ''' +def test_ok(): + pass + + +def test_bad(): + assert False +''' + + +def test_plugin_uploads_with_the_option(pytester, api, monkeypatch): + monkeypatch.setattr(upload_mod, "_token", lambda base: "tok") + monkeypatch.setenv("BENCHPOD_API_BASE", api.base) + pytester.makepyfile(test_suite=SUITE) + out = pytester.path / "out" + res = pytester.runpytest(f"--embeddedci-run-dir={out}", "--embeddedci-upload", "-p", "no:cacheprovider") + res.assert_outcomes(passed=1, failed=1) + res.stdout.fnmatch_lines(["*embeddedci: run uploaded: https://example.test/run/r-1*"]) + doc = api.run_doc() + assert len(doc["results"]["cases"]) == 2 and "junit.xml" in api.file_names() + assert (out / "run.json").exists() + + +def test_plugin_upload_env_records_into_the_default_folder(pytester, api, monkeypatch): + monkeypatch.setattr(upload_mod, "_token", lambda base: "tok") + monkeypatch.setenv("BENCHPOD_API_BASE", api.base) + monkeypatch.setenv("EMBEDDEDCI_UPLOAD", "1") + pytester.makepyfile(test_suite=SUITE) + res = pytester.runpytest("-p", "no:cacheprovider") + res.assert_outcomes(passed=1, failed=1) + assert len(list((pytester.path / "embeddedci-runs").iterdir())) == 1 + assert api.run_doc()["schema_version"] == 1 + + +def test_plugin_upload_failure_is_a_warning(pytester, api, monkeypatch): + monkeypatch.setattr(upload_mod, "_token", lambda base: "tok") + monkeypatch.setenv("BENCHPOD_API_BASE", api.base) + api.fail = True + pytester.makepyfile(test_suite="def test_ok():\n pass\n") + res = pytester.runpytest("--embeddedci-upload", "-p", "no:cacheprovider") + res.assert_outcomes(passed=1) + assert res.ret == 0 + res.stdout.fnmatch_lines(["*embeddedci: warning: run upload: run upload failed (HTTP 500)*"]) + + +def test_plugin_never_uploads_under_benchpod_ci(pytester, api, monkeypatch): + monkeypatch.setattr(upload_mod, "_token", lambda base: "tok") + monkeypatch.setenv("BENCHPOD_API_BASE", api.base) + monkeypatch.setenv("EMBEDDEDCI_RUN_DIR", str(pytester.path / "ci")) + monkeypatch.setenv("EMBEDDEDCI_RUN_STEP", "hil") + pytester.makepyfile(test_suite="def test_ok():\n pass\n") + res = pytester.runpytest("--embeddedci-upload", "-p", "no:cacheprovider") + res.assert_outcomes(passed=1) + assert api.requests == [] and (pytester.path / "ci" / "parts" / "hil.json").exists() + + +class _FakeReporter: + active = True + build_id = "b-42" + + def __init__(self, base: str) -> None: + self.api_base = base + self.session_token = "oidc-tok" + self.result = None + + def record_wiring(self, **_k: Any) -> None: + pass + + def upload_logs(self, *_a: Any) -> None: + pass + + def set_result(self, success: bool, reason: str = "") -> None: + self.result = (success, reason) + + def finalize(self) -> None: + pass + + +def test_build_report_uploads_a_run_linked_to_the_build(pytester, api, monkeypatch): + from embeddedci.benchpod import ci + + reporters: List[_FakeReporter] = [] + + def make(**_k: Any) -> _FakeReporter: + reporters.append(_FakeReporter(api.base)) + return reporters[-1] + + monkeypatch.setattr(ci, "make_build_reporter", make) + monkeypatch.setenv("GITHUB_ACTIONS", "true") + pytester.makepyfile(test_suite=''' +def test_flash(build_report): + build_report.record_wiring(target="t.cfg") + + +def test_other(): + assert False +''') + res = pytester.runpytest("-p", "no:cacheprovider") + res.assert_outcomes(passed=1, failed=1) + assert reporters[0].result == (True, "") # the old status call still runs + doc = api.run_doc() + assert doc["inputs"]["dut_firmware"]["build_id"] == "b-42" + assert {c["id"].split("::")[-1] for c in doc["results"]["cases"]} == {"test_flash", "test_other"} + assert next(r for r in api.requests if r["path"] == "/api/runs")["auth"] == "Bearer oidc-tok" + local = json.loads(next((pytester.path / "embeddedci-runs").iterdir()).joinpath("run.json").read_text()) + assert local["inputs"]["dut_firmware"]["build_id"] == "b-42" + res.stdout.fnmatch_lines(["*embeddedci: run uploaded: https://example.test/run/r-1*"]) + + +def test_build_report_outside_github_actions_records_nothing(pytester, api): + pytester.makepyfile(test_suite=''' +def test_flash(build_report): + assert not build_report.active +''') + res = pytester.runpytest("-p", "no:cacheprovider") + res.assert_outcomes(passed=1) + assert not (pytester.path / "embeddedci-runs").exists() and api.requests == [] + + +def test_the_repos_benchpod_ci_file(): + yaml = pytest.importorskip("yaml") + here = Path(__file__).parent + cfg = yaml.safe_load((here / "benchpod-ci.yaml").read_text()) + assert cfg["version"] == 1 and cfg["suite"] and cfg["dut"] == "none" + step = cfg["steps"][0]["pytest"] + assert (here / step["path"]).is_dir() and (here / step["args"][0]).is_dir() + + +@pytest.mark.parametrize("what", ["la_capture", "ünïcode_cmd", "温度", ".dot", "a/b c", ""]) +def test_attachment_and_part_names_pass_the_server_rule(what): + from embeddedci.benchpod.runs.active import case_slug_simple + + part = case_slug_simple(what) + assert names_valid([f"att/0001-{part}.json", f"att/0001-{part}.json.zst", f"logs/{part}.log"]) + assert all(c.isascii() for c in part) + + +def test_recorded_attachment_names_are_ascii(tmp_path): + from embeddedci.benchpod.runs import RunRecording + + run = RunRecording(tmp_path / "run") + run.begin_case("c") + run.record(t=0.0, op="command", cmd="ünï", req={"cmd": "ünï", "x": "y" * 5000}, ms=1.0, ok=True) + run.end_case("c", "failed") + names = [f"att/{p.name}" for p in (tmp_path / "run" / "att").iterdir()] + assert names and names_valid(names) From 8d867e138e15a772d32a68c82e90700c4eb1a1fa Mon Sep 17 00:00:00 2001 From: Edward Viaene Date: Wed, 18 Nov 2026 22:42:01 -0600 Subject: [PATCH 4/6] benchpod-ci.yaml for the SDK's hardware tests One pytest step over e2e/ and examples/ (the make e2e tiers), dut: none: the DUT tier skips without --benchpod-firmware. Mentioned in CONTRIBUTING and the e2e README. Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 4 +++- packages/embeddedci/tests/benchpod-ci.yaml | 23 ++++++++++++++++++++++ packages/embeddedci/tests/e2e/README.md | 10 ++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 packages/embeddedci/tests/benchpod-ci.yaml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ffc45c7..780f70d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,7 +13,9 @@ Thanks for helping out. Issues and pull requests are welcome. make test PYTHON=.venv/bin/python ``` -- If you have a BenchPod, run the hardware tier too: `make e2e POD=`. +- If you have a BenchPod, run the hardware tier too: `make e2e POD=`, or record it as a + test run with `cd packages/embeddedci/tests && benchpod ci --pod ` + (`packages/embeddedci/tests/benchpod-ci.yaml`; see `packages/embeddedci/tests/e2e/README.md`). - The 2.x SDK API and MCP tool surface are frozen (snapshot tests fail on any change). Additions are fine; renames and removals wait for a major version. - Keep commits small and focused, and describe what changed and why. diff --git a/packages/embeddedci/tests/benchpod-ci.yaml b/packages/embeddedci/tests/benchpod-ci.yaml new file mode 100644 index 0000000..b69ad07 --- /dev/null +++ b/packages/embeddedci/tests/benchpod-ci.yaml @@ -0,0 +1,23 @@ +# The SDK's own hardware tests with `benchpod ci` (needs a benchpod-cli that has `ci`): +# +# cd packages/embeddedci/tests && benchpod ci --pod +# +# The same tests as `make e2e POD=...` from the repository root: the pod, GPIO, rev3, digital and +# USB tiers of e2e/ and the examples. Without --benchpod-firmware the DUT tier (test_e2e_dut.py) +# skips, so the suite needs no DUT firmware; add `--benchpod-firmware=` to `args` (and name +# the board in `dut`) to run it. The bench wiring comes from BENCHPOD_E2E_* (see e2e/README.md). +# Results land in ./embeddedci-runs/