From 17c5d88c2a2ade132140649d1f8e134f13ff460f Mon Sep 17 00:00:00 2001 From: Edward Viaene Date: Thu, 19 Nov 2026 06:43:32 -0600 Subject: [PATCH 1/3] SDK: read test runs back (embeddedci.benchpod.runs.api) RunsApi over a ServerApi: list_runs (suite, dut, build, pod, ref, limit, before), get_run, compare_runs (previous, a run id, baseline:), list_run_files, get_run_file and case_trace; summarize_run, find_case and trace_lines shape the documents. Module-level shortcuts use BENCHPOD_API_KEY, else the benchpod login session, else GitHub Actions OIDC. Tests against a fake runs API. Co-Authored-By: Claude Opus 5.5 --- .../src/embeddedci/benchpod/runs/api.py | 233 ++++++++++++++++++ packages/embeddedci/tests/runs_api_fake.py | 81 ++++++ packages/embeddedci/tests/test_runs_api.py | 76 ++++++ 3 files changed, 390 insertions(+) create mode 100644 packages/embeddedci/src/embeddedci/benchpod/runs/api.py create mode 100644 packages/embeddedci/tests/runs_api_fake.py create mode 100644 packages/embeddedci/tests/test_runs_api.py diff --git a/packages/embeddedci/src/embeddedci/benchpod/runs/api.py b/packages/embeddedci/src/embeddedci/benchpod/runs/api.py new file mode 100644 index 0000000..5afa4f1 --- /dev/null +++ b/packages/embeddedci/src/embeddedci/benchpod/runs/api.py @@ -0,0 +1,233 @@ +"""Read test runs back from embeddedci.com: list them, read one, compare two, fetch a run's files. + +:: + + from embeddedci.benchpod.runs import api + + runs = api.list_runs(suite="process-node-hil", limit=5)["runs"] + run = api.get_run(runs[0]["id"]) + print(api.summarize_run(run)["failed_cases"]) + print(api.compare_runs(run["id"], "previous")["cases"]["newly_failing"]) + for line in api.case_trace(run, "tests/test_can.py::test_ack", errors_only=True)["lines"]: + print(line) + +Credentials: ``BENCHPOD_API_KEY``, else the ``benchpod login`` session, else GitHub Actions OIDC. +:class:`RunsApi` takes a :class:`~embeddedci.benchpod.server_api.ServerApi` for anything else. +""" + +from __future__ import annotations + +import json +import os +from typing import Any, Dict, List, Optional +from urllib.parse import quote + +from ..server_api import ServerApi + +#: How many runs one list call returns at most (the server's cap). +LIST_MAX = 200 +#: A trace line's ``req``/``resp`` longer than this (as JSON) is shortened in :func:`trace_lines`. +TRACE_FIELD_MAX = 300 + + +def default_server(api_base: Optional[str] = None, api_key: Optional[str] = None) -> ServerApi: + """A server client: ``api_key`` / ``BENCHPOD_API_KEY``, else the ``benchpod login`` session, + else (in GitHub Actions) a session token from the job's OIDC token.""" + from ..cloud_auth import DEFAULT_API_BASE, login_session_token, mint_session_token + + base = (api_base or os.environ.get("BENCHPOD_API_BASE") or DEFAULT_API_BASE).rstrip("/") + key = (api_key or os.environ.get("BENCHPOD_API_KEY") or "").strip() + if key: + return ServerApi(api_base=base, api_key=key) + login = login_session_token(base) + if login is not None: + return ServerApi(api_base=base, token_provider=login) + token = mint_session_token(base)[0] # GitHub Actions OIDC, else CloudAuthError listing the options + return ServerApi(api_base=base, token_provider=lambda: token) + + +class RunsApi: + """The runs API of an embeddedci server (``/api/runs``).""" + + def __init__(self, server: Optional[ServerApi] = None) -> None: + self.server = server if server is not None else default_server() + + def list_runs(self, *, suite: Optional[str] = None, dut: Optional[str] = None, + build: Optional[str] = None, pod: Optional[str] = None, ref: Optional[str] = None, + limit: int = 20, before: Optional[str] = None) -> Dict[str, Any]: + """Runs newest first: ``{"runs": [...], "next_before": ..., "org_pods": N}``. ``pod`` is a + pod's name or id; ``before`` the ``next_before`` of the previous page.""" + query = {"suite": suite or None, "dut": dut or None, "build": build or None, + "device": pod or None, "ref": ref or None, + "limit": max(1, min(int(limit), LIST_MAX)), "before": before or None} + _, data = self.server.request("GET", "/runs", query=query) + data = dict(data or {}) + data.setdefault("runs", []) + return data + + def get_run(self, run_id: str) -> Dict[str, Any]: + """One run as stored: the run.json document with ``id``, ``url``, ``build_id``, ``files``.""" + _, data = self.server.request("GET", f"/runs/{quote(run_id, safe='')}") + return dict(data or {}) + + def compare_runs(self, run_id: str, with_: str = "previous") -> Dict[str, Any]: + """The diff of an earlier run (``a``) against ``run_id`` (``b``). ``with_``: ``previous`` + (the newest earlier valid run of the same suite and DUT), a run id, or + ``baseline:``. ``found`` is false when there is nothing to compare with.""" + _, data = self.server.request("GET", f"/runs/{quote(run_id, safe='')}/compare", + query={"with": with_ or "previous"}) + return dict(data or {}) + + def list_run_files(self, run_id: str) -> List[Dict[str, Any]]: + """The run's uploaded files: ``name``, ``size_bytes``, ``expires_at``, ``expired``.""" + _, data = self.server.request("GET", f"/runs/{quote(run_id, safe='')}/files") + return list((data or {}).get("files") or []) + + def get_run_file(self, run_id: str, name: str) -> bytes: + """One uploaded file of the run (``junit.xml``, ``trace/.ndjson``, ``att/...``).""" + _, raw = self.server.request("GET", f"/runs/{quote(run_id, safe='')}/files", + query={"name": name}, parse_json=False) + return bytes(raw or b"") + + def case_trace(self, run: Any, case_id: str, *, errors_only: bool = False, + limit: int = 200) -> Dict[str, Any]: + """The pod-call trace of one case (``run``: a run id or a :meth:`get_run` document), as + trimmed lines: see :func:`trace_lines`. Raises ``LookupError`` when the run has no such + case, or no trace was uploaded for it (only failed and error cases' traces are).""" + doc = self.get_run(run) if isinstance(run, str) else run + case = find_case(doc, case_id) + trace = str(case.get("trace") or "") + if not trace: + raise LookupError(f"no trace was uploaded for {case['id']} ({case.get('outcome')}): only " + "failed and error cases' traces are uploaded") + data = self.get_run_file(str(doc.get("id") or doc.get("run_id")), trace) + out = trace_lines(data, errors_only=errors_only, limit=limit) + out["case"] = case["id"] + out["trace"] = trace + return out + + +# -- helpers on the documents ------------------------------------------------------------------------ + +def find_case(run: Dict[str, Any], case_id: str) -> Dict[str, Any]: + """The case ``case_id`` of a run document: an exact id, else the one case whose id ends with it + (``test_ack`` finds ``tests/test_can.py::test_ack``).""" + cases = list((run.get("results") or {}).get("cases") or []) + for c in cases: + if c.get("id") == case_id: + return c + tail = [c for c in cases if str(c.get("id", "")).endswith(case_id)] + if len(tail) == 1: + return tail[0] + if tail: + raise LookupError(f"{case_id!r} matches {len(tail)} cases: " + + ", ".join(str(c.get("id")) for c in tail[:5])) + raise LookupError(f"the run has no case {case_id!r}") + + +def first_line(text: str) -> str: + for line in (text or "").splitlines(): + if line.strip(): + return line.strip()[:500] + return "" + + +def summarize_run(run: Dict[str, Any]) -> Dict[str, Any]: + """The essentials of a run document: who and what, outcome and counts, health, the failed and + error cases with the first line of their message, and every measurement outside its limits.""" + results = run.get("results") or {} + inputs = run.get("inputs") or {} + health = run.get("health") or {} + failed = [] + out_of_limits = [] + for c in results.get("cases") or []: + if c.get("outcome") in ("failed", "error"): + failed.append({"id": c.get("id"), "outcome": c.get("outcome"), + "message": first_line(str(c.get("message") or "")), + "trace": bool(c.get("trace"))}) + for m in c.get("measurements") or []: + if m.get("outcome") == "failed": + out_of_limits.append({"case": c.get("id"), **{k: m.get(k) for k in + ("name", "value", "unit", "min", "max")}}) + pod = inputs.get("pod") or {} + return { + "id": run.get("id") or run.get("run_id"), + "url": run.get("url"), + "suite": (run.get("key") or {}).get("suite"), + "dut": (run.get("key") or {}).get("dut"), + "outcome": results.get("outcome"), + "counts": results.get("counts") or {}, + "duration_ms": results.get("duration_ms"), + "started": run.get("started"), + "trigger": run.get("trigger"), + "build_id": run.get("build_id") or (inputs.get("dut_firmware") or {}).get("build_id") or "", + "pod": {k: pod.get(k) for k in ("name", "board", "firmware", "gateware", "connection")}, + "tests": inputs.get("tests") or {}, + "valid": health.get("valid", True), + "health_reasons": list(health.get("reasons") or []), + "failed_cases": failed, + "measurements_out_of_limits": out_of_limits, + } + + +def _short(value: Any) -> Any: + text = json.dumps(value, separators=(",", ":"), default=str) + if len(text) <= TRACE_FIELD_MAX: + return value + return f"<{len(text)} bytes: {text[:TRACE_FIELD_MAX]}...>" + + +def trace_lines(data: bytes, *, errors_only: bool = False, limit: int = 200) -> Dict[str, Any]: + """A trace file's lines (NDJSON) as objects, ``req``/``resp`` over 300 characters shortened: + ``{"lines": [...], "total": N, "errors": N, "shown": N, "truncated": bool}``. ``errors_only`` + keeps the calls that failed; ``limit`` caps how many are returned (the first ones).""" + lines: List[Dict[str, Any]] = [] + total = 0 + errors = 0 + for raw in data.decode("utf-8", "replace").splitlines(): + raw = raw.strip() + if not raw: + continue + try: + obj = json.loads(raw) + except ValueError: + continue + if not isinstance(obj, dict): + continue + total += 1 + failed = obj.get("ok") is False + errors += failed + if errors_only and not failed: + continue + if len(lines) < max(0, int(limit)): + lines.append({k: (_short(v) if k in ("req", "resp") else v) for k, v in obj.items()}) + shown_pool = errors if errors_only else total + return {"lines": lines, "total": total, "errors": errors, "shown": len(lines), + "truncated": len(lines) < shown_pool} + + +# -- module-level shortcuts -------------------------------------------------------------------------- + +def list_runs(**kwargs: Any) -> Dict[str, Any]: + """:meth:`RunsApi.list_runs` with the default credentials.""" + return RunsApi().list_runs(**kwargs) + + +def get_run(run_id: str) -> Dict[str, Any]: + """:meth:`RunsApi.get_run` with the default credentials.""" + return RunsApi().get_run(run_id) + + +def compare_runs(run_id: str, with_: str = "previous") -> Dict[str, Any]: + """:meth:`RunsApi.compare_runs` with the default credentials.""" + return RunsApi().compare_runs(run_id, with_) + + +def get_run_file(run_id: str, name: str) -> bytes: + """:meth:`RunsApi.get_run_file` with the default credentials.""" + return RunsApi().get_run_file(run_id, name) + + +def case_trace(run: Any, case_id: str, *, errors_only: bool = False, limit: int = 200) -> Dict[str, Any]: + """:meth:`RunsApi.case_trace` with the default credentials.""" + return RunsApi().case_trace(run, case_id, errors_only=errors_only, limit=limit) diff --git a/packages/embeddedci/tests/runs_api_fake.py b/packages/embeddedci/tests/runs_api_fake.py new file mode 100644 index 0000000..b17a1fd --- /dev/null +++ b/packages/embeddedci/tests/runs_api_fake.py @@ -0,0 +1,81 @@ +"""A fake embeddedci runs API for urllib.request.urlopen (shared by the SDK and MCP tests).""" + +from __future__ import annotations + +import io +import json +import urllib.error +import urllib.parse +from typing import Any, Dict, List + +TRACE = "\n".join(json.dumps(x) for x in [ + {"t": 1.0, "op": "command", "cmd": "can_open", "req": {"cmd": "can_open"}, "ms": 2.0, "ok": True, + "resp": {"blob": "z" * 2000}}, + {"t": 1.5, "op": "command", "cmd": "can_write", "req": {"cmd": "can_write"}, "ms": 3.0, "ok": False, + "error": "BP0412 no ack"}, + {"t": 2.0, "op": "command", "cmd": "can_write", "ms": 3.2, "ok": True}, +]) + "\n" + +RUN = { + "schema_version": 1, "id": "r-2", "run_id": "r-2", "url": "https://example.test/run/r-2", + "build_id": "b-1", "key": {"suite": "hil", "dut": "process-node"}, "trigger": "github", + "started": "2026-11-18T14:02:11.123Z", + "inputs": {"pod": {"name": "benchpod-abc123", "board": "analog-v3", "firmware": "3.10.12", + "gateware": 58, "connection": "lan"}, + "tests": {"repo": "o/r", "ref": "main", "sha": "abc", "dirty": False}}, + "health": {"valid": True, "reasons": []}, + "results": {"outcome": "failed", "duration_ms": 1000, + "counts": {"passed": 1, "failed": 1, "error": 0, "skipped": 0, "xfailed": 0, "xpassed": 0}, + "cases": [ + {"id": "tests/test_can.py::test_ack", "outcome": "failed", + "message": "ValueError: bad CRC: 6f72\n\n def test_ack(bus):", "trace": "trace/t-ack.ndjson", + "measurements": [{"name": "settle_ms", "value": 1200, "unit": "ms", "min": None, "max": 1000, + "outcome": "failed"}]}, + {"id": "tests/test_can.py::test_ok", "outcome": "passed", "trace": "", + "measurements": [{"name": "v", "value": 1, "unit": "V", "min": 0, "max": 2, + "outcome": "passed"}]}]}, + "files": [{"name": "junit.xml", "size_bytes": 10, "expired": False}, + {"name": "trace/t-ack.ndjson", "size_bytes": 300, "expired": False}], +} +COMPARE = {"found": True, "a": "r-1", "b": "r-2", "comparable": True, "reasons": [], "flags": [], + "inputs_changed": [{"field": "pod.firmware", "a": "3.10.11", "b": "3.10.12"}], + "cases": {"newly_failing": ["tests/test_can.py::test_ack"], "fixed": [], "still_failing": [], + "new": [], "removed": [], "newly_skipped": []}, + "measurements": [], "slower": [], "calls_slower": []} + + +class _Resp(io.BytesIO): + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +class FakeRunsServer: + def __init__(self) -> None: + self.requests: List[Dict[str, Any]] = [] + + def urlopen(self, req: Any, timeout: Any = None, **_k: Any) -> _Resp: + url = urllib.parse.urlsplit(req.full_url) + q = {k: v[0] for k, v in urllib.parse.parse_qs(url.query).items()} + self.requests.append({"method": req.get_method(), "path": url.path, "query": q, + "auth": req.get_header("Authorization"), "base": f"{url.scheme}://{url.netloc}"}) + if url.path == "/api/runs": + return _Resp(json.dumps({"runs": [{"id": "r-2", "suite": "hil", "outcome": "failed"}], + "next_before": "2026-11-18T14:02:11Z", "org_pods": 2}).encode()) + if url.path == "/api/runs/r-2": + return _Resp(json.dumps(RUN).encode()) + if url.path == "/api/runs/r-2/compare": + if q.get("with") == "baseline:none": + return _Resp(json.dumps({"found": False, "a": None, "b": "r-2", + "message": "no earlier valid run"}).encode()) + return _Resp(json.dumps(COMPARE).encode()) + if url.path == "/api/runs/r-2/files": + if q.get("name") == "trace/t-ack.ndjson": + return _Resp(TRACE.encode()) + if "name" not in q: + return _Resp(json.dumps({"files": RUN["files"]}).encode()) + raise urllib.error.HTTPError(req.full_url, 404, "no", {}, io.BytesIO(b'{"error":"run not found"}')) diff --git a/packages/embeddedci/tests/test_runs_api.py b/packages/embeddedci/tests/test_runs_api.py new file mode 100644 index 0000000..c0df434 --- /dev/null +++ b/packages/embeddedci/tests/test_runs_api.py @@ -0,0 +1,76 @@ +"""embeddedci.benchpod.runs.api against a fake runs API (urlopen faked, no network).""" + +from __future__ import annotations + +import urllib.request + +import pytest + +from embeddedci.benchpod.runs import api +from embeddedci.benchpod.server_api import ServerApi, ServerApiError + +from runs_api_fake import RUN, FakeRunsServer + + +@pytest.fixture +def fake(monkeypatch): + server = FakeRunsServer() + monkeypatch.setattr(urllib.request, "urlopen", server.urlopen) + monkeypatch.setenv("BENCHPOD_API_KEY", "eci_test") + monkeypatch.setenv("BENCHPOD_API_BASE", "https://staging.example") + return server + + +def test_list_runs_filters_and_auth(fake): + data = api.list_runs(suite="hil", dut="process-node", build="b-1", pod="benchpod-abc123", ref="main", + limit=500) + assert data["runs"][0]["id"] == "r-2" and data["next_before"] and data["org_pods"] == 2 + req = fake.requests[0] + assert req["base"] == "https://staging.example" and req["auth"] == "ApiKey eci_test" + assert req["query"] == {"suite": "hil", "dut": "process-node", "build": "b-1", + "device": "benchpod-abc123", "ref": "main", "limit": "200"} + + +def test_get_compare_and_files(fake): + assert api.get_run("r-2")["id"] == "r-2" + assert api.compare_runs("r-2")["cases"]["newly_failing"] == ["tests/test_can.py::test_ack"] + assert fake.requests[-1]["query"] == {"with": "previous"} + assert api.compare_runs("r-2", "baseline:none")["found"] is False + assert api.get_run_file("r-2", "trace/t-ack.ndjson").startswith(b"{") + assert [f["name"] for f in api.RunsApi().list_run_files("r-2")] == ["junit.xml", "trace/t-ack.ndjson"] + with pytest.raises(ServerApiError, match="404.*run not found"): + api.get_run("nope") + + +def test_summarize_run(): + s = api.summarize_run(RUN) + assert s["outcome"] == "failed" and s["build_id"] == "b-1" and s["pod"]["firmware"] == "3.10.12" + assert s["failed_cases"] == [{"id": "tests/test_can.py::test_ack", "outcome": "failed", + "message": "ValueError: bad CRC: 6f72", "trace": True}] + assert s["measurements_out_of_limits"] == [{"case": "tests/test_can.py::test_ack", "name": "settle_ms", + "value": 1200, "unit": "ms", "min": None, "max": 1000}] + + +def test_case_trace(fake): + t = api.case_trace("r-2", "test_ack") + assert t["case"] == "tests/test_can.py::test_ack" and t["total"] == 3 and t["errors"] == 1 + assert t["lines"][0]["resp"].startswith("<") and "bytes" in t["lines"][0]["resp"] # shortened + only = api.case_trace(RUN, "tests/test_can.py::test_ack", errors_only=True) + assert [ln["error"] for ln in only["lines"]] == ["BP0412 no ack"] and not only["truncated"] + capped = api.case_trace(RUN, "test_ack", limit=1) + assert capped["shown"] == 1 and capped["truncated"] + with pytest.raises(LookupError, match="no trace was uploaded"): + api.case_trace(RUN, "test_ok") + with pytest.raises(LookupError, match="no case"): + api.case_trace(RUN, "test_missing") + + +def test_find_case_ambiguous(): + run = {"results": {"cases": [{"id": "a::test_x"}, {"id": "b::test_x"}]}} + with pytest.raises(LookupError, match="matches 2 cases"): + api.find_case(run, "test_x") + + +def test_runs_api_takes_a_server(): + server = ServerApi(api_base="https://x", token_provider=lambda: "t") + assert api.RunsApi(server).server is server From 50b52b44b0ee61985c7474591a867bb76ab67c66 Mon Sep 17 00:00:00 2001 From: Edward Viaene Date: Thu, 19 Nov 2026 06:43:32 -0600 Subject: [PATCH 2/3] MCP: cloud_runs_list, cloud_run_get, cloud_run_compare, cloud_run_trace Read the test runs on embeddedci.com through the SDK's runs API, with the cloud_* tools' credentials: list runs, a run's summary (failed cases with the message's first line, measurements outside limits, files; full=true for the whole run.json), the diff against an earlier run (compare_with: previous, a run id, baseline:), and a failed case's pod calls. An SDK without the runs API gets a clear tool error. Tool snapshot, README table, guide and CHANGELOG updated. Co-Authored-By: Claude Opus 5.5 --- packages/embeddedci-mcp/CHANGELOG.md | 4 + packages/embeddedci-mcp/README.md | 9 +- .../src/embeddedci_mcp/guide.py | 2 +- .../src/embeddedci_mcp/models.py | 44 ++++ .../src/embeddedci_mcp/server.py | 101 +++++++++ .../embeddedci-mcp/tests/test_run_tools.py | 68 ++++++ packages/embeddedci-mcp/tests/test_tools.py | 4 + .../embeddedci-mcp/tests/tools_surface.json | 195 ++++++++++++++++++ 8 files changed, 425 insertions(+), 2 deletions(-) create mode 100644 packages/embeddedci-mcp/tests/test_run_tools.py diff --git a/packages/embeddedci-mcp/CHANGELOG.md b/packages/embeddedci-mcp/CHANGELOG.md index c753ace..f4a29c7 100644 --- a/packages/embeddedci-mcp/CHANGELOG.md +++ b/packages/embeddedci-mcp/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- New tools `cloud_runs_list`, `cloud_run_get`, `cloud_run_compare` and `cloud_run_trace`: list + the test runs on embeddedci.com, read one (the failed cases with their message's first line, the + measurements outside their limits), see what changed since an earlier run, and read a failed + case's pod calls. They need an SDK with `embeddedci.benchpod.runs.api` (newer than 3.0.3). - New tool `log_level`: read the pod's console log level, or set it with `level` (`error`, `warn`, `info`, `debug`; `info` is the default, `debug` prints every command trace on the pod's console). Firmware newer than 3.10.12; older firmware gets a clean unsupported-feature error. diff --git a/packages/embeddedci-mcp/README.md b/packages/embeddedci-mcp/README.md index b781913..09ef321 100644 --- a/packages/embeddedci-mcp/README.md +++ b/packages/embeddedci-mcp/README.md @@ -122,7 +122,10 @@ stored wiring profile. `BENCHPOD_API_KEY` wins when set, and inside GitHub Actions the SDK uses OIDC instead. `cloud_list_devices` lists the pods on your account and which are online, without connecting to -one; pass a result's `connection` to `connect`. The session token behind a cloud connection is +one; pass a result's `connection` to `connect`. `cloud_runs_list`, `cloud_run_get`, +`cloud_run_compare` and `cloud_run_trace` read the test runs uploaded to embeddedci.com, so an agent +can see why a run failed (the failed cases, the measurements outside their limits, the pod calls of +a failed test) and what changed since an earlier run, without the bench. The session token behind a cloud connection is renewed before it expires, so a long session keeps working. A pod is shared, so `connect` takes an exclusive lease, waiting up to `--lease-wait` seconds if @@ -192,6 +195,10 @@ description, ranges and enums, so an agent sees more than this table. | Tool | Purpose | Parameters | | --- | --- | --- | | `cloud_list_devices` | The pods on your embeddedci.com organization and whether each is online, without connecting. | none | +| `cloud_runs_list` | Test runs uploaded to embeddedci.com (`benchpod ci`, `pytest --embeddedci-upload`, `build_report`), newest first, with outcome and counts; `before` is the previous page's `next_before`. | `suite`, `dut`, `build`, `pod`, `ref`, `limit=20`, `before` | +| `cloud_run_get` | One test run: outcome, counts, health, pod and tests, the failed cases with the first line of their message, the measurements outside their limits and the uploaded files; `full=true` adds the whole `run.json`. | **`run_id`**, `full=false` | +| `cloud_run_compare` | What changed between an earlier run and this one: newly failing and fixed cases, measurements, slower tests and pod calls, and the inputs that differ (pod firmware, wiring, DUT firmware). `compare_with`: `previous`, a run id, or `baseline:`. | **`run_id`**, `compare_with="previous"` | +| `cloud_run_trace` | The pod calls of one test case from its uploaded trace (only failed and error cases have one): what was sent, how long it took, the answer or error; long requests and replies are shortened. `case_id` may be a unique end of the id. | **`run_id`**, **`case_id`**, `errors_only=false`, `limit=200` | ### Wiring profile diff --git a/packages/embeddedci-mcp/src/embeddedci_mcp/guide.py b/packages/embeddedci-mcp/src/embeddedci_mcp/guide.py index 1111e12..0930b97 100644 --- a/packages/embeddedci-mcp/src/embeddedci_mcp/guide.py +++ b/packages/embeddedci-mcp/src/embeddedci_mcp/guide.py @@ -12,7 +12,7 @@ 2. The I/O voltage (1.8 or 3.3, the DUT's) comes from the wiring profile's la_mv: connect sets it, so status normally reports one. When status warns that none is selected, the profile sets none: ask the user for the board's I/O voltage and set it in the profile (set_wiring with la_mv, then save_wiring), or call set_la_voltage for this session. The pod refuses flash, UART, LA capture, pull resistors and sensor emulation until it is set. 3. status shows firmware, capabilities, the LA voltage and open sessions. -Cloud pods: cloud_list_devices lists the pods registered on embeddedci.com (name, online) without connecting to any; connect to one with its `connection` ('embeddedci:'). Tools named cloud_* talk to embeddedci.com, not to a pod. Cloud access (cloud_* and connect to 'embeddedci:') authenticates with BENCHPOD_API_KEY, else the user's `benchpod login` session; if neither exists, ask the user to run `benchpod login`. +Cloud pods: cloud_list_devices lists the pods registered on embeddedci.com (name, online) without connecting to any; connect to one with its `connection` ('embeddedci:'). Tools named cloud_* talk to embeddedci.com, not to a pod. Test runs (benchpod ci, pytest --embeddedci-upload, build_report): cloud_runs_list finds them, cloud_run_get shows the failed cases and measurements outside their limits, cloud_run_trace the pod calls of a failed case, cloud_run_compare what changed since an earlier run. Cloud access (cloud_* and connect to 'embeddedci:') authenticates with BENCHPOD_API_KEY, else the user's `benchpod login` session; if neither exists, ask the user to run `benchpod login`. Wiring: the pod has 14 generic LA channels (LA1-LA14) and nothing is fixed. Call `wiring` first: it is the bench's profile, normally one of the named profiles stored on the pod itself (the same ones the web app shows; `profile_name`, the pod's default unless connect(wiring_profile=...) named another): which DUT signal is on which channel, the I/O voltage, the target-power rail, the UART baud, the SWD target. Every channel, baud, rail or SWD argument you omit comes from it, and channel arguments also accept its names ("READY", "uart_rx"). What the profile does not set is undefined: when a tool says a pin is missing, ask the user which LA channel it is on, never guess. Confirm the wiring with the user if it looks wrong. `set_wiring` changes it for this connection only: nothing is stored until save_wiring (or the server's --wiring-autosave). Once the wiring works on the board, ask the user whether to save it, and under which name when it is a different board (save_wiring(name='nrf52840')). `unsaved_changes` in the result lists what is not stored; it is lost on disconnect. load_wiring switches to another stored profile, set_default_wiring changes the pod's default (ask first). power_on applies the profile first (its I/O voltage, then the rail) and records it on the pod. Refusals carry a stable code (BP0009 ...) with the fix; follow it. Target power is internal 5 V (efuse 1) or external (efuse 2). The DUT's reset line goes to the pod's own reset pin: flash(nreset=true), reset_target. diff --git a/packages/embeddedci-mcp/src/embeddedci_mcp/models.py b/packages/embeddedci-mcp/src/embeddedci_mcp/models.py index ed082ce..b5286bd 100644 --- a/packages/embeddedci-mcp/src/embeddedci_mcp/models.py +++ b/packages/embeddedci-mcp/src/embeddedci_mcp/models.py @@ -801,6 +801,50 @@ class CloudDeviceList(BaseModel): devices: List[CloudDevice] +class CloudRunList(BaseModel): + auth: Literal["api_key", "benchpod_login"] + runs: List[Dict[str, Any]] = Field(description=( + "Newest first: id, url, suite, dut, outcome, valid, counts, started, duration_ms, build_id, " + "device_name, tests_ref, tests_sha, firmware, github_run_url.")) + next_before: Optional[str] = Field(None, description=( + "More runs exist: pass this as `before` for the next page.")) + org_pods: Optional[int] = Field(None, description="How many pods the organization has.") + + +class CloudRun(BaseModel): + auth: Literal["api_key", "benchpod_login"] + summary: Dict[str, Any] = Field(description=( + "id, url, suite, dut, outcome, counts, valid, health_reasons, pod, tests, build_id, " + "failed_cases (id, outcome, the message's first line, whether a trace was uploaded) and " + "measurements_out_of_limits.")) + files: List[Dict[str, Any]] = Field(default_factory=list, description=( + "The uploaded files (name, size_bytes, expired): junit.xml, the traces of failed cases.")) + run: Optional[Dict[str, Any]] = Field(None, description="The full run.json, with full=true.") + + +class CloudRunCompare(BaseModel): + auth: Literal["api_key", "benchpod_login"] + found: bool = Field(description="False when there is no earlier run to compare with.") + message: Optional[str] = None + diff: Dict[str, Any] = Field(default_factory=dict, description=( + "a (the earlier run), b (this one), comparable, reasons, flags, inputs_changed, cases " + "(newly_failing, fixed, still_failing, new, removed, newly_skipped), measurements, slower, " + "calls_slower, a_run (the earlier run's summary).")) + + +class CloudRunTrace(BaseModel): + auth: Literal["api_key", "benchpod_login"] + case: str + trace: str = Field(description="The trace file's name in the run.") + lines: List[Dict[str, Any]] = Field(description=( + "One object per pod call: t (s since the run started), op, cmd, req, ms, ok, resp or " + "error; long req/resp are shortened.")) + total: int + errors: int = Field(description="Calls that failed (ok false).") + shown: int + truncated: bool = Field(description="More lines exist than `limit` returned.") + + class RecordingResult(BaseModel): id: str name: str diff --git a/packages/embeddedci-mcp/src/embeddedci_mcp/server.py b/packages/embeddedci-mcp/src/embeddedci_mcp/server.py index 7285d47..19306ab 100644 --- a/packages/embeddedci-mcp/src/embeddedci_mcp/server.py +++ b/packages/embeddedci-mcp/src/embeddedci_mcp/server.py @@ -2611,6 +2611,107 @@ def op() -> m.CloudDeviceList: return await _call(op, lock=False) +def _runs_api() -> Tuple[Any, str]: + """The SDK's runs API over the same credentials as the other cloud_* tools.""" + try: + from embeddedci.benchpod.runs.api import RunsApi + except ImportError as exc: # an SDK older than the runs API + raise ToolError("reading test runs needs a newer embeddedci SDK (embeddedci.benchpod.runs.api); " + "upgrade the embeddedci package") from exc + server, auth = _cloud_api() + return RunsApi(server), auth + + +def _lookup(fn: Callable[[], T]) -> Callable[[], T]: + def op() -> T: + try: + return fn() + except LookupError as exc: + raise ToolError(str(exc)) from exc + return op + + +@mcp.tool(annotations=_ann("List test runs", read_only=True, cloud=True)) +async def cloud_runs_list( + suite: Annotated[Optional[str], Field(description="Only this suite (the run's key.suite).")] = None, + dut: Annotated[Optional[str], Field(description="Only this board under test ('none' for pod-only suites).")] = None, + build: Annotated[Optional[str], Field(description="Only the runs linked to this build id.")] = None, + pod: Annotated[Optional[str], Field(description="Only the runs on this pod (name or id).")] = None, + ref: Annotated[Optional[str], Field(description="Only the runs of this git branch or ref of the tests.")] = None, + limit: Annotated[int, Field(ge=1, le=200, description="How many runs, newest first.")] = 20, + before: Annotated[Optional[str], Field(description="The next_before of the previous page.")] = None, +) -> m.CloudRunList: + """Test runs uploaded to embeddedci.com (benchpod ci, pytest --embeddedci-upload, build_report), + newest first, with their outcome and counts. Needs no pod connection; authenticates like + cloud_list_devices.""" + def op() -> m.CloudRunList: + api, auth = _runs_api() + data = api.list_runs(suite=suite, dut=dut, build=build, pod=pod, ref=ref, limit=limit, before=before) + return m.CloudRunList(auth=auth, runs=list(data.get("runs") or []), + next_before=data.get("next_before"), org_pods=data.get("org_pods")) + + return await _call(op, lock=False) + + +@mcp.tool(annotations=_ann("Read a test run", read_only=True, cloud=True)) +async def cloud_run_get( + run_id: str, + full: Annotated[bool, Field(description="Also return the whole run.json (every case, inputs, calls).")] = False, +) -> m.CloudRun: + """One test run: outcome, counts, health, the pod and tests it ran with, the failed cases with + the first line of their message, and the measurements outside their limits. cloud_run_trace + reads a failed case's pod calls; cloud_run_compare shows what changed since an earlier run.""" + def op() -> m.CloudRun: + from embeddedci.benchpod.runs.api import summarize_run + + api, auth = _runs_api() + doc = api.get_run(run_id) + return m.CloudRun(auth=auth, summary=summarize_run(doc), files=list(doc.get("files") or []), + run=doc if full else None) + + return await _call(op, lock=False) + + +@mcp.tool(annotations=_ann("Compare test runs", read_only=True, cloud=True)) +async def cloud_run_compare( + run_id: str, + compare_with: Annotated[str, Field(description=( + "'previous' (the newest earlier valid run of the same suite and DUT), a run id, or " + "'baseline:' (the newest valid run on that branch)."))] = "previous", +) -> m.CloudRunCompare: + """What changed between an earlier run (a) and this one (b): newly failing and fixed cases, + measurements that moved or broke a limit, slower tests and pod calls, and the inputs that differ + (pod firmware, wiring, DUT firmware), so the likely cause sits next to the change.""" + def op() -> m.CloudRunCompare: + api, auth = _runs_api() + data = api.compare_runs(run_id, compare_with) + found = bool(data.get("found", True)) + return m.CloudRunCompare(auth=auth, found=found, message=data.get("message"), + diff={k: v for k, v in data.items() if k not in ("found", "message")}) + + return await _call(op, lock=False) + + +@mcp.tool(annotations=_ann("Read a test case's pod calls", read_only=True, cloud=True)) +async def cloud_run_trace( + run_id: str, + case_id: Annotated[str, Field(description=( + "The case id (pytest node id), or a unique end of it ('test_ack')."))], + errors_only: Annotated[bool, Field(description="Only the calls that failed.")] = False, + limit: Annotated[int, Field(ge=1, le=2000, description="How many lines, from the start.")] = 200, +) -> m.CloudRunTrace: + """Every call a test case made to the pod (what it sent, how long it took, what the pod + answered), from the trace uploaded with the run. Only failed and error cases have one.""" + def op() -> m.CloudRunTrace: + api, auth = _runs_api() + t = api.case_trace(run_id, case_id, errors_only=errors_only, limit=limit) + return m.CloudRunTrace(auth=auth, case=t["case"], trace=t["trace"], lines=t["lines"], + total=t["total"], errors=t["errors"], shown=t["shown"], + truncated=t["truncated"]) + + return await _call(_lookup(op), lock=False) + + @mcp.tool(annotations=_ann("Replay a library waveform", destructive=True, cloud=True)) async def replay_waveform( waveform_id: str, diff --git a/packages/embeddedci-mcp/tests/test_run_tools.py b/packages/embeddedci-mcp/tests/test_run_tools.py new file mode 100644 index 0000000..ec4fb89 --- /dev/null +++ b/packages/embeddedci-mcp/tests/test_run_tools.py @@ -0,0 +1,68 @@ +"""cloud_runs_list, cloud_run_get, cloud_run_compare and cloud_run_trace against a fake runs API +(the SDK tests' fake, urlopen patched: no network).""" + +from __future__ import annotations + +import importlib.util +import urllib.request +from pathlib import Path + +import pytest +from mcp.server.fastmcp.exceptions import ToolError + +from conftest import call + +pytest.importorskip("embeddedci.benchpod.runs.api", reason="an SDK without the runs API") + +_FAKE = Path(__file__).resolve().parents[2] / "embeddedci" / "tests" / "runs_api_fake.py" +_spec = importlib.util.spec_from_file_location("runs_api_fake_for_mcp", _FAKE) +fake_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(fake_mod) # type: ignore[union-attr] + + +@pytest.fixture +def server(monkeypatch): + fake = fake_mod.FakeRunsServer() + monkeypatch.setattr(urllib.request, "urlopen", fake.urlopen) + monkeypatch.setenv("BENCHPOD_API_KEY", "eci_test") + monkeypatch.delenv("BENCHPOD_API_BASE", raising=False) + return fake + + +def test_runs_list(server): + out = call("cloud_runs_list", suite="hil", pod="benchpod-abc123", limit=5) + assert out["auth"] == "api_key" and out["runs"][0]["id"] == "r-2" + assert out["next_before"] and out["org_pods"] == 2 + assert server.requests[0]["query"] == {"suite": "hil", "device": "benchpod-abc123", "limit": "5"} + assert server.requests[0]["auth"] == "ApiKey eci_test" + + +def test_run_get(server): + out = call("cloud_run_get", run_id="r-2") + s = out["summary"] + assert s["outcome"] == "failed" and s["failed_cases"][0]["message"] == "ValueError: bad CRC: 6f72" + assert s["measurements_out_of_limits"][0]["name"] == "settle_ms" + assert out["run"] is None and [f["name"] for f in out["files"]][0] == "junit.xml" + assert call("cloud_run_get", run_id="r-2", full=True)["run"]["results"]["cases"] + + +def test_run_compare(server): + out = call("cloud_run_compare", run_id="r-2") + assert out["found"] and out["diff"]["cases"]["newly_failing"] == ["tests/test_can.py::test_ack"] + assert server.requests[-1]["query"] == {"with": "previous"} + none = call("cloud_run_compare", run_id="r-2", compare_with="baseline:none") + assert none["found"] is False and "no earlier" in none["message"] + assert server.requests[-1]["query"] == {"with": "baseline:none"} + + +def test_run_trace(server): + out = call("cloud_run_trace", run_id="r-2", case_id="test_ack", errors_only=True) + assert out["case"] == "tests/test_can.py::test_ack" and out["total"] == 3 + assert [ln["cmd"] for ln in out["lines"]] == ["can_write"] and out["lines"][0]["ok"] is False + with pytest.raises(ToolError, match="no trace was uploaded"): + call("cloud_run_trace", run_id="r-2", case_id="test_ok") + + +def test_unknown_run(server): + with pytest.raises(ToolError, match="run not found"): + call("cloud_run_get", run_id="nope") diff --git a/packages/embeddedci-mcp/tests/test_tools.py b/packages/embeddedci-mcp/tests/test_tools.py index 93106c4..7845bba 100644 --- a/packages/embeddedci-mcp/tests/test_tools.py +++ b/packages/embeddedci-mcp/tests/test_tools.py @@ -39,6 +39,10 @@ "can_open", "can_write", "can_read", "can_respond", "can_status", "can_close", "la_step", "command", "cloud_list_devices", + "cloud_runs_list", + "cloud_run_get", + "cloud_run_compare", + "cloud_run_trace", "vision_cloud_boards", "vision_connect", "vision_disconnect", "vision_status", "vision_look", "vision_ocr", "vision_find", "vision_wait_for", "vision_compare", "vision_screen_health", "vision_calibrate", "vision_add_reference", "vision_press", "vision_tap", diff --git a/packages/embeddedci-mcp/tests/tools_surface.json b/packages/embeddedci-mcp/tests/tools_surface.json index 1faaab1..fb3dbda 100644 --- a/packages/embeddedci-mcp/tests/tools_surface.json +++ b/packages/embeddedci-mcp/tests/tools_surface.json @@ -1294,6 +1294,201 @@ "type": "object" } }, + "cloud_run_compare": { + "annotations": { + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": true, + "title": "Compare test runs" + }, + "input": { + "properties": { + "compare_with": { + "default": "previous", + "description": "'previous' (the newest earlier valid run of the same suite and DUT), a run id, or 'baseline:' (the newest valid run on that branch).", + "title": "Compare With", + "type": "string" + }, + "run_id": { + "title": "Run Id", + "type": "string" + } + }, + "required": [ + "run_id" + ], + "title": "cloud_run_compareArguments", + "type": "object" + } + }, + "cloud_run_get": { + "annotations": { + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": true, + "title": "Read a test run" + }, + "input": { + "properties": { + "full": { + "default": false, + "description": "Also return the whole run.json (every case, inputs, calls).", + "title": "Full", + "type": "boolean" + }, + "run_id": { + "title": "Run Id", + "type": "string" + } + }, + "required": [ + "run_id" + ], + "title": "cloud_run_getArguments", + "type": "object" + } + }, + "cloud_run_trace": { + "annotations": { + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": true, + "title": "Read a test case's pod calls" + }, + "input": { + "properties": { + "case_id": { + "description": "The case id (pytest node id), or a unique end of it ('test_ack').", + "title": "Case Id", + "type": "string" + }, + "errors_only": { + "default": false, + "description": "Only the calls that failed.", + "title": "Errors Only", + "type": "boolean" + }, + "limit": { + "default": 200, + "description": "How many lines, from the start.", + "maximum": 2000, + "minimum": 1, + "title": "Limit", + "type": "integer" + }, + "run_id": { + "title": "Run Id", + "type": "string" + } + }, + "required": [ + "run_id", + "case_id" + ], + "title": "cloud_run_traceArguments", + "type": "object" + } + }, + "cloud_runs_list": { + "annotations": { + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": true, + "title": "List test runs" + }, + "input": { + "properties": { + "before": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The next_before of the previous page.", + "title": "Before" + }, + "build": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Only the runs linked to this build id.", + "title": "Build" + }, + "dut": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Only this board under test ('none' for pod-only suites).", + "title": "Dut" + }, + "limit": { + "default": 20, + "description": "How many runs, newest first.", + "maximum": 200, + "minimum": 1, + "title": "Limit", + "type": "integer" + }, + "pod": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Only the runs on this pod (name or id).", + "title": "Pod" + }, + "ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Only the runs of this git branch or ref of the tests.", + "title": "Ref" + }, + "suite": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Only this suite (the run's key.suite).", + "title": "Suite" + } + }, + "title": "cloud_runs_listArguments", + "type": "object" + } + }, "command": { "annotations": { "destructiveHint": true, From af8bab336d3d5e03a4eea9ccc11e718c49b5d402 Mon Sep 17 00:00:00 2001 From: Edward Viaene Date: Thu, 19 Nov 2026 06:43:32 -0600 Subject: [PATCH 3/3] Docs: reading test runs back Co-Authored-By: Claude Opus 5.5 --- packages/embeddedci/CHANGELOG.md | 5 +++++ packages/embeddedci/README.md | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/packages/embeddedci/CHANGELOG.md b/packages/embeddedci/CHANGELOG.md index 45f7b73..87de884 100644 --- a/packages/embeddedci/CHANGELOG.md +++ b/packages/embeddedci/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- **Read test runs back:** `embeddedci.benchpod.runs.api` lists the runs on embeddedci.com + (`list_runs`, by suite, DUT, build, pod or ref), reads one (`get_run`, `summarize_run`: the + failed cases and the measurements outside their limits), compares two (`compare_runs` with + `previous`, a run id or `baseline:`) and fetches their files (`get_run_file`, + `case_trace`: a case's pod calls). `RunsApi` takes your own `ServerApi`. - **Console log level:** `bp.log_level()` reads how much the pod prints on its USB console and `bp.set_log_level(level)` sets it (`LOG_LEVELS`: `error`, `warn`, `info`, `debug`; default `info`, stored on the pod). Firmware newer than 3.10.12 (feature `log_level`); older firmware diff --git a/packages/embeddedci/README.md b/packages/embeddedci/README.md index 40cfbdf..6825b94 100644 --- a/packages/embeddedci/README.md +++ b/packages/embeddedci/README.md @@ -644,6 +644,26 @@ from embeddedci.benchpod.runs.upload import upload_run print(upload_run("embeddedci-runs/20261118T140211Z-hil")["url"]) ``` +**Reading runs back.** `embeddedci.benchpod.runs.api` lists, reads and compares the runs on +embeddedci.com, with the same credentials as the upload: + +```python +from embeddedci.benchpod.runs import api + +runs = api.list_runs(suite="process-node-hil", ref="main", limit=5)["runs"] # newest first +run = api.get_run(runs[0]["id"]) +print(api.summarize_run(run)["failed_cases"]) # id, outcome, the message's first line +diff = api.compare_runs(run["id"], "previous") # or a run id, or "baseline:main" +print(diff["cases"]["newly_failing"], diff["inputs_changed"]) +trace = api.case_trace(run, "test_ack", errors_only=True) # the failed case's pod calls +junit = api.get_run_file(run["id"], "junit.xml") +``` + +`list_runs` takes `suite`, `dut`, `build`, `pod` (name or id), `ref`, `limit` (up to 200) and +`before` (the previous page's `next_before`). `RunsApi(server)` does the same with a `ServerApi` +of your own. The MCP server's `cloud_runs_list`, `cloud_run_get`, `cloud_run_compare` and +`cloud_run_trace` tools are built on it. + **Inside `benchpod ci`.** The `benchpod ci` runner hands its run folder to the `pytest` steps it starts through the environment; the plugin then writes `parts/.json` (the cases, health counters, tool versions and pod) and its traces into that folder, and the runner merges them into