diff --git a/kits/mario/README.md b/kits/mario/README.md index 3fd904f..0b901aa 100644 --- a/kits/mario/README.md +++ b/kits/mario/README.md @@ -138,6 +138,18 @@ npm run dev # against a console at the same origin, or set a proxy for /a npm run build # what the image serves at /kits/mario ``` +### The package's configuration, objective and evidence + +`plugin/config.yaml` is the reflex's configuration, versioned with the package: the instructions, +the gate, the encoder, the tunables the rendering reads (the enemy and gap horizons, the tall-wall +height, the measured take-off windows) and the objective (pass on `cleared`, a failure per life +lost, `level_x` as the locus, `observations/` as the evidence). The environment archives every +frame it shows under `observations/` beside the live `frame.jpg`, and `plugin/tools/evidence.py` +cuts that archive into a contact sheet around each failure. `plugin/ledger.jsonl` records every +version with its evidence and verdict. An outer harness (the `plugins/calibrate` package in this +repository) reads all of it to improve the configuration one version at a time; the design is +`docs/dual-loop.md` in the System One Harness repository. + ## Credits - The game is **Full Screen Mario**, played at [supermarioplay.com](https://supermarioplay.com/game/mario.html?v=1.0.1). Mario and its characters belong to Nintendo; this kit plays the page as a person would and ships none of the game's files. diff --git a/kits/mario/plugin/config.yaml b/kits/mario/plugin/config.yaml new file mode 100644 index 0000000..5f64790 --- /dev/null +++ b/kits/mario/plugin/config.yaml @@ -0,0 +1,29 @@ +# The Super Mario reflex's configuration: everything that shapes its decisions except the model +# and the game. Versioned with the package; the outer loop (docs/dual-loop.md in the harness repo) +# changes it one thing at a time, with the evidence in ledger.jsonl beside it. +version: 1 +goal: "Play the level: keep running right, jump over enemies, gaps and pipes, hit the question blocks for coins, and get as far as you can." +instructions: | + You play a side-scrolling platform game as Mario, deciding several times a second. Each action sets the keys for the next moment and they stay set until you change them: run_right holds right and run; jump_right holds right, run and jump, and a jump held in the air goes again the moment Mario lands, so holding it hops him along; jump holds jump alone; walk_left holds left; wait lets every key go. Keep moving right. The state ends with a line beginning "Now:" that says what the measured facts call for at this moment; follow it. + + Facts, measured on this game: a decision lands a fraction of a second after the state you read, and at a full run Mario covers 2 to 3 tiles in that time. A jump held for one decision clears 3 tiles, for two decisions 4 tiles; jump_right pressed again on the ground is a new jump, and held through a landing it is a new jump at once. A running jump lands about 9 tiles on. An enemy is jumped at a run with the take-off 1.5 to 4 tiles before it; under a row of blocks an earlier jump hits the blocks and drops Mario onto it, so when that take-off cannot be hit, stop with walk_left and jump_right when it is 1 to 2.5 tiles away, which always works. An enemy walking at a standing Mario is jumped with jump_right when it is 1 to 2.5 tiles away. An enemy on a ledge above walks off its edge and drops onto whoever runs under it: wait for it to come down, then jump_right over it. A pipe 4 tiles tall is cleared only at a full run, leaving the ground 2 to 3 tiles before it with the jump held for three decisions; from standing or at a jog it is never cleared, so if Mario is stopped at one, walk_left for four decisions, then run_right, then jump_right; but if an enemy is walking at his back, walking left meets it: stand and take jump (the standing jump) when it is 2 tiles behind him, it passes under, and only then walk_left. A step one tile tall is hopped with jump_right; a stair is a hop per step. Where the ground drops away ahead, jump_right from the edge lands past the drop, and walking off the edge lands in a slot with no way out. A gap is crossed by a running jump from its edge, and a jump still in the air with a gap coming is kept held so the next hop goes the moment Mario lands; a standing jump falls in; enemies waiting where the jump would land walk to the gap and fall in if Mario stands at the edge. A question block pays a coin when hit from below by a jump started 1 to 1.5 tiles before it at a run. + + When Mario has just died, wait. Finish when the level is cleared. Escalate when Mario has no lives left. +gate: {read: 0.5, write: 0.7, destructive: 0.9} +encoder: {history_steps: 3} +tunables: + enemy_horizon_tiles: 16 # how far ahead an enemy is named + gap_horizon_tiles: 14 # how far ahead a gap is named and the jump timed + tall_wall_tiles: 4 # a wall this tall is cleared only at a full run + pipe_takeoff_tiles: [1.8, 3.4] # measured: where the jump over a 4-tile pipe leaves the ground + enemy_takeoff_tiles: [1.5, 6.5] # measured: where the jump over an enemy at a run may leave the ground, in the open + enemy_takeoff_under_blocks_tiles: [1.5, 4.0] # measured: under the block row, farther hits the blocks and drops onto it + archive_frames: true # keep every frame the page shows under observations/ +objective: + pass: "cleared == true" + failure: "lives decreased" + metrics: + - {field: cleared, better: true} + - {field: level_x, better: higher} + locus: [level_x] + evidence: observations/ diff --git a/kits/mario/plugin/ledger.jsonl b/kits/mario/plugin/ledger.jsonl new file mode 100644 index 0000000..50dd71c --- /dev/null +++ b/kits/mario/plugin/ledger.jsonl @@ -0,0 +1 @@ +{"version": 1, "at": "2026-09-20", "change": "the released environment and instructions (starter-kit 28ce645)", "channel": "told,shown", "evidence": ["the manual loop of 2026-09-20: recordings and contact sheets of attempts 22 to 42"], "runs": ["local attempts 34, 41, 42; one hosted run on hr-test"], "metrics_before": null, "metrics_after": {"pass": 1.0, "failures": 4.3, "runs": 3}, "verdict": "kept", "note": "pipes cleared only at a full run from a measured take-off; the ground read from Mario's level; a held jump goes again on landing; the nearest thing ahead decides the advice"} diff --git a/kits/mario/plugin/plugin.json b/kits/mario/plugin/plugin.json index 1f3864c..b081aea 100644 --- a/kits/mario/plugin/plugin.json +++ b/kits/mario/plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "mario-env", - "version": "0.3.2", + "version": "0.4.0", "description": "Full Screen Mario as an environment for the System One base: the game's state as literal text each step, the keys it listens to as actions, and the browser's frame for the page to show.", "homepage": "https://github.com/HarnessRouter/starter-kit", "license": "SEE LICENSE IN ../LICENSE.md" diff --git a/kits/mario/plugin/server/mario_env.py b/kits/mario/plugin/server/mario_env.py index c00bc96..15b6d21 100644 --- a/kits/mario/plugin/server/mario_env.py +++ b/kits/mario/plugin/server/mario_env.py @@ -17,6 +17,7 @@ import asyncio import base64 +import json import glob import os import pathlib @@ -38,7 +39,25 @@ "so holding it hops him along; jump holds jump alone; walk_left holds left; wait lets every key go. The state says where Mario is, what he is doing, what is ahead and behind with distances in " "tiles, and which keys are held. Keep moving right and jump over what is within one step. Finish when " "the level is cleared. Escalate when Mario has no lives left.") -TALL = 4 # a wall this tall is cleared only at a run +TALL = 4 # a wall this tall is cleared only at a run (the default; config.yaml tunables override) +TUNABLES = {"enemy_horizon_tiles": 16, "gap_horizon_tiles": 14, "tall_wall_tiles": TALL, "pipe_takeoff_tiles": [1.8, 3.4], + "enemy_takeoff_tiles": [1.5, 6.5], "enemy_takeoff_under_blocks_tiles": [1.5, 4.0], "archive_frames": True} + + +def load_tunables() -> dict: + """The package's config.yaml tunables over the defaults: the numbers the rendering reads, which + the outer loop changes one at a time (docs/dual-loop.md in the harness repo).""" + tun = dict(TUNABLES) + for cand in (os.environ.get("SYSTEMONE_CONFIG"), str(pathlib.Path(__file__).resolve().parents[1] / "config.yaml")): + if cand and os.path.exists(cand): + try: + import yaml + d = yaml.safe_load(open(cand)) or {} + tun.update({k: v for k, v in (d.get("tunables") or {}).items() if k in tun}) + except Exception: # noqa: BLE001 - a bad file leaves the defaults + pass + break + return tun # The frame the kit's page shows: Chrome's own screencast, a JPEG for every third frame the game # draws (about twenty a second), each written whole to frame.jpg. The page reads that file. SCREENCAST = {"format": "jpeg", "quality": 45, "maxWidth": 960, "maxHeight": 600, "everyNthFrame": 3} @@ -63,7 +82,8 @@ if (!c.alive || c === player || c.title === undefined) return; if (['Coin','Mushroom','FireFlower','Star','Vine','Text','Shell','Fireball'].indexOf(c.title) >= 0) return; var dx = (c.left - p.right) / T, dy = (p.bottom - c.bottom) / T, back = (p.left - c.right) / T; - if (dx >= 0 && dx <= 16) enemies.push({kind: c.title, dx: Math.round(dx * 10) / 10, dy: Math.round(dy * 10) / 10, dir: (c.xvel || 0) < 0 ? 'toward' : 'away'}); + var EH = (window.__s1tun && window.__s1tun.enemy_horizon_tiles) || 16; + if (dx >= 0 && dx <= EH) enemies.push({kind: c.title, dx: Math.round(dx * 10) / 10, dy: Math.round(dy * 10) / 10, dir: (c.xvel || 0) < 0 ? 'toward' : 'away'}); else if (back >= 0 && back <= 8) behind.push({kind: c.title, dx: Math.round(back * 10) / 10, dy: Math.round(dy * 10) / 10, dir: (c.xvel || 0) > 0 ? 'toward' : 'away'}); }); // the ground ahead as a profile from where Mario stands: each quarter tile, the highest surface @@ -165,6 +185,9 @@ def __init__(self): self._session = None self._cdp = None self.frame_path = workspace_root() / "frame.jpg" + self.tun = load_tunables() + self.archive = (workspace_root() / "observations") if self.tun.get("archive_frames") else None + self._archived = 0 self.started_at = time.time() self.steps = 0 self.frames = 0 @@ -210,10 +233,21 @@ def _on_frame(self, ev: dict, session_id=None) -> None: (a temp file renamed over it, so a reader never sees half a picture), and the frame is acknowledged, without which Chrome stops sending.""" try: + data = base64.b64decode(ev["data"]) tmp = self.frame_path.with_suffix(".jpg.tmp") - tmp.write_bytes(base64.b64decode(ev["data"])) + tmp.write_bytes(data) os.replace(tmp, self.frame_path) self.frames += 1 + if self.archive is not None and self._archived < 12000: + # the observation archive the outer loop reads: every frame the page showed, with + # its time, under observations/ (about 7 KB a frame, eighteen a second) + if self._archived == 0: + self.archive.mkdir(parents=True, exist_ok=True) + name = f"{self._archived:06d}.jpg" + (self.archive / name).write_bytes(data) + with open(self.archive / "frames.jsonl", "a") as f: + f.write(json.dumps({"t": round(time.time(), 3), "file": name}) + "\n") + self._archived += 1 except Exception: # noqa: BLE001 - a missed frame is a missed frame, never a failed step pass self._loop.create_task(self._cdp.cdp_client.send.Page.screencastFrameAck( @@ -320,7 +354,7 @@ async def go(): if isinstance(st, dict) and st.get("ready"): break await asyncio.sleep(0.25) - await self._eval("window.unpause && unpause(); 'ok'") + await self._eval(f"window.__s1tun = {json.dumps(self.tun)}; window.unpause && unpause(); 'ok'") try: await self._screencast() except Exception: # noqa: BLE001 - already running across the navigation @@ -445,7 +479,7 @@ def _now(self, st: dict, reach: float, en: list, gaps: list, walls: list, blocks things.append((walls[0]["dx"], "wall")) if drops and drops[0]["dx"] <= 12: things.append((drops[0]["dx"], "drop")) - if gaps and gaps[0]["dx"] <= 14: + if gaps and gaps[0]["dx"] <= self.tun["gap_horizon_tiles"]: things.append((gaps[0]["dx"], "gap")) if near is not None and near["dx"] <= 9: things.append((near["dx"], "enemy")) @@ -453,7 +487,7 @@ def _now(self, st: dict, reach: float, en: list, gaps: list, walls: list, blocks kind = things[0][1] if things else None if kind == "wall": w = walls[0]; w_next = w["dx"] - lag - if w["height"] >= TALL: + if w["height"] >= self.tun["tall_wall_tiles"]: # measured: the pipe is cleared at near full speed (4.9 and up) with the jump held # long; at a jog the apex is level with its top and the side stops him (recorded) w = walls[0]; w_next = w["dx"] - lag @@ -465,9 +499,10 @@ def _now(self, st: dict, reach: float, en: list, gaps: list, walls: list, blocks return "too slow for the pipe from here: walk_left for two decisions, then run_right to full speed and jump_right at 2 to 3 tiles." if not full: return "run_right to full speed; jump_right when the pipe is about 5 tiles ahead and Mario is running flat out." - if 1.8 <= w_next <= 3.4: + lo_t, hi_t = self.tun["pipe_takeoff_tiles"] + if lo_t <= w_next <= hi_t: return "jump_right now, and keep it held for three decisions: the pipe's take-off point is here." - if w_next < 1.8: + if w_next < lo_t: return "jump_right now and hold it three decisions." return "run_right toward the pipe; jump_right when it is about 5 tiles ahead at this speed." if w_next <= 1.5: @@ -486,7 +521,7 @@ def _now(self, st: dict, reach: float, en: list, gaps: list, walls: list, blocks # when the next state would already be past it; under the blocks a miss is a death, # so there the stop and the standing jump (8 of 8) take over instead under_blocks = any(o["dx"] <= near["dx"] + 1 for o in overhead) - lo, hi = (1.5, 4.0) if under_blocks else (1.5, 6.5) + lo, hi = self.tun["enemy_takeoff_under_blocks_tiles"] if under_blocks else self.tun["enemy_takeoff_tiles"] if lo <= takeoff <= hi: return "jump_right now over the enemy: this is the take-off." if takeoff > hi: @@ -582,7 +617,7 @@ def _describe(self, st: dict) -> dict: if walls: w = walls[0] need = ("a jump held for three decisions from a run, leaving the ground 2 to 3 tiles before it; from against it or from standing it is never cleared" - if w["height"] >= TALL else "a jump held for two decisions" if w["height"] >= 3 else "a hop (jump_right)" if w["height"] <= 1 else "a jump") + if w["height"] >= self.tun["tall_wall_tiles"] else "a jump held for two decisions" if w["height"] >= 3 else "a hop (jump_right)" if w["height"] <= 1 else "a jump") parts.append(f"Wall ahead: {w['kind']} {w['dx']} tiles ahead, {w['height']} tiles tall; it takes {need}.") else: parts.append("No pipe or wall within 8 tiles.") diff --git a/kits/mario/plugin/tools/evidence.py b/kits/mario/plugin/tools/evidence.py new file mode 100644 index 0000000..baf5d31 --- /dev/null +++ b/kits/mario/plugin/tools/evidence.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""The evidence renderer for this environment: a contact sheet around each failure. + + evidence.py + +Reads the run's trace and the archived frames (observations/NNNNNN.jpg with frames.jsonl of +timestamps), finds each failure (a life lost), and writes failure_N.jpg: nine frames from one +second before the last live decision to a second and a half after, captioned with the step, the +action and the state. With Pillow missing it writes failure_N.html instead, the same frames as +images. The outer loop reads these; nothing here is specific to the calibration method. +""" +import json +import os +import re +import sys + + +def _frames(obs_dir): + idx = os.path.join(obs_dir, "frames.jsonl") + rows = [] + if os.path.exists(idx): + for line in open(idx): + try: + d = json.loads(line) + rows.append((float(d["t"]), os.path.join(obs_dir, d["file"]))) + except (ValueError, KeyError): + continue + else: + for name in sorted(os.listdir(obs_dir)): + m = re.match(r"^(\d+(?:\.\d+)?)\.jpg$", name) + if m: + rows.append((float(m.group(1)), os.path.join(obs_dir, name))) + rows.sort() + return rows + + +def _failures(trace): + steps = trace.get("steps") or [] + prev = None + last_live = None + out = [] + for st in steps: + f = ((st.get("result") or {}).get("fields") or {}) + lives = f.get("lives") + if lives is not None and prev is not None and lives < prev: + src = last_live or st + out.append(src) + if lives is not None: + prev = lives + if f.get("lives") is not None and not f.get("restarting") and not f.get("dead"): + last_live = st + return out + + +def _nearest(frames, t): + best = None + for ft, path in frames: + if best is None or abs(ft - t) < abs(best[0] - t): + best = (ft, path) + return best + + +def main(argv): + trace_path, obs_dir, out_dir = argv[1], argv[2], argv[3] + trace = json.load(open(trace_path)) + frames = _frames(obs_dir) + os.makedirs(out_dir, exist_ok=True) + fails = _failures(trace) + try: + from PIL import Image, ImageDraw + except ImportError: + Image = None + written = [] + for n, st in enumerate(fails, 1): + t0 = float(st.get("started_at") or 0) + times = [t0 - 1.0 + i * 0.3125 for i in range(9)] + picks = [_nearest(frames, t) for t in times] if frames else [] + text = str((st.get("result") or {}).get("text") or "") + cap = f"failure {n} before step {st.get('index')}: {st.get('action')}: {text[:200]}" + if Image and picks: + tiles = [] + for ft, path in picks: + im = Image.open(path).convert("RGB") + im.thumbnail((400, 250)) + tiles.append((ft - t0, im)) + w, h = tiles[0][1].size + sheet = Image.new("RGB", (w * 3, (h + 22) * 3 + 24), "white") + d = ImageDraw.Draw(sheet) + d.text((6, 4), cap[:180], fill="black") + for i, (dt, im) in enumerate(tiles): + x, y = (i % 3) * w, 24 + (i // 3) * (h + 22) + sheet.paste(im, (x, y)) + d.text((x + 4, y + h + 4), f"t{dt:+.1f}s", fill="black") + out = os.path.join(out_dir, f"failure_{n}.jpg") + sheet.save(out, quality=80) + else: + out = os.path.join(out_dir, f"failure_{n}.html") + imgs = "".join(f'
t{ft - t0:+.1f}s
' for ft, p in picks) + open(out, "w").write(f"

{cap}

{imgs}
") + written.append({"failure": n, "step": st.get("index"), "action": st.get("action"), "file": out, "text": text[:300]}) + json.dump(written, open(os.path.join(out_dir, "failures.json"), "w"), indent=1) + print(f"{len(written)} failures rendered into {out_dir}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/plugins/calibrate/README.md b/plugins/calibrate/README.md new file mode 100644 index 0000000..5f552dc --- /dev/null +++ b/plugins/calibrate/README.md @@ -0,0 +1,22 @@ +# Calibrate + +The outer loop of the dual loop (the design is `docs/dual-loop.md` in the System One Harness +repository): a reasoning harness that improves another harness's configuration against the +objective that harness's package declares, one change per version, validated by that harness's +own model, with the evidence in a ledger. + +Install this package on a reasoning harness (a coding base) with platform access for one inner +harness: `HR_API_URL`, `HR_CALIBRATION_TOKEN` (a per-turn credential scoped to that harness) and +`HR_INNER_HARNESS` (its id). The Skill carries the method; the scripts do the platform work: + +| Script | What it does | +|---|---| +| `bench.py --runs 3 --package ` | K runs, one at a time; fetches each run's workspace; the objective's scoreboard and failure groups | +| `fetch.py --out package` | the inner harness's package (the one carrying the environment), never the harness's export | +| `probe.py "a,b,c"` | one run driven by a fixed action sequence, to measure the environment | +| `publish.py --package ` | uploads the package as the inner harness's plugin; its instructions follow `config.yaml` | +| `report.py --package traces...` | the report over traces already on disk | + +The inner harness's package must carry `config.yaml` (with an `objective`) and `ledger.jsonl`; +the harness must write `trace.json` into its session workspace, and its environment may archive +what it showed under `observations/`. The Super Mario kit is the first such package. diff --git a/plugins/calibrate/plugin.json b/plugins/calibrate/plugin.json new file mode 100644 index 0000000..6cea9a7 --- /dev/null +++ b/plugins/calibrate/plugin.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", + "name": "harnessrouter-calibrate", + "version": "0.1.3", + "description": "The outer loop: a reasoning harness calibrates another harness's configuration against its declared objective, one change at a time, with the evidence in a ledger.", + "author": { + "name": "HarnessRouter", + "url": "https://harnessrouter.ai" + }, + "homepage": "https://github.com/HarnessRouter/starter-kit/tree/main/plugins/calibrate", + "repository": "https://github.com/HarnessRouter/starter-kit", + "license": "LicenseRef-HarnessRouter-Starter-Kit-1.0.0", + "keywords": [ + "harnessrouter", + "calibration", + "system-one", + "system-two", + "dual-loop" + ] +} diff --git a/plugins/calibrate/skills/calibrate/SKILL.md b/plugins/calibrate/skills/calibrate/SKILL.md new file mode 100644 index 0000000..efb2edc --- /dev/null +++ b/plugins/calibrate/skills/calibrate/SKILL.md @@ -0,0 +1,55 @@ +--- +name: calibrate +description: Calibrate another harness's configuration against its declared objective, one change per version, validated by that harness's own model. Use when asked to improve a harness, raise its pass rate, or find out why it fails. +--- + +# Calibrate a harness + +You are the outer loop. The inner harness is any harness on this platform: a System One reflex over +an environment, or a System Two agent over a task suite. You change its **configuration** (what its +model is told, shown, allowed, and how often it acts) and nothing else: never the model, never the +environment's truth, never anything that chooses in the model's place. + +Everything you need is in the inner harness's package: `config.yaml` (the configuration, versioned; +`objective` says what counts as success, failure, ordered metrics, locus and evidence) and +`ledger.jsonl` (every version so far, with its evidence and verdict). Fetch it first with +`scripts/fetch.py --out package` (the package that carries the environment), then read both. +Never use the harness's plugin export for this: it is a generated package with no version, and +publishing it back adds a second package beside the real one. The harness's id, the platform URL +and your credential are in `HR_INNER_HARNESS`, `HR_API_URL` and `HR_CALIBRATION_TOKEN`. The +scripts under `scripts/` do the platform work; read `--help`. + +## The method + +1. **Baseline.** `scripts/bench.py --runs 3` starts three runs, one at a time (never two on one + machine: shared machines drop frames and fake regressions), fetches each run's `trace.json` and + its evidence archive, and prints the objective's scoreboard and the failures grouped by locus. + Write the scoreboard down. +2. **Read.** Take the largest failure group. Render its evidence (the package may ship a renderer + under `tools/`, for example `tools/evidence.py `; read the images it + writes). Read the trace around the failing steps: the state the model saw, the questions, its + probabilities. Say in one sentence what killed the run there. +3. **Measure.** No change without a measurement. Probe the environment at that locus with + `scripts/probe.py "action,action,..."` (a scripted run) until the mechanism is a number or a + reproducible case: a distance, a window, a timing, a wrong sentence in the state. +4. **Change one thing.** Edit `config.yaml`: one fact or rule in `instructions`, one tunable, one + gate threshold, or one encoder setting. One change. Bump `version`. Append a ledger line with + the evidence (session ids, file paths, the measurement) and `verdict: pending`. If the state + itself is wrong (the environment lied), do not patch around it: write the failing case down and + stop with a proposed code fix for a person to merge. +5. **Publish and validate.** `scripts/publish.py` uploads the package as the harness's plugin and + relaunches it. Run the bench again (same run count). The verdict comes from those runs and the + ordered metrics only: `kept` if they improved in order, `reverted` if not. On `reverted`, put + the previous version back and publish again. Record the verdict in the ledger line. +6. **Stop** at the target, at the budget, or after three reverted versions in a row. Report the + scoreboard before and after, the ledger lines you added, and which limit stopped you: the + world's truth, the pace, or the model. + +## What never changes + +- One change per version. Batching four changes cost a night of untangling. +- The verdict comes from the inner harness's own model runs. A stand-in that follows the rendered + advice deterministically went to zero failures while the model went from three passes in three + to none in six. +- Read failures from the evidence, not from the summary text. +- Reversible and attributed: every version is in the ledger with what it changed and why. diff --git a/plugins/calibrate/skills/calibrate/scripts/__pycache__/metrics.cpython-312.pyc b/plugins/calibrate/skills/calibrate/scripts/__pycache__/metrics.cpython-312.pyc new file mode 100644 index 0000000..6bb20df Binary files /dev/null and b/plugins/calibrate/skills/calibrate/scripts/__pycache__/metrics.cpython-312.pyc differ diff --git a/plugins/calibrate/skills/calibrate/scripts/bench.py b/plugins/calibrate/skills/calibrate/scripts/bench.py new file mode 100755 index 0000000..c49da60 --- /dev/null +++ b/plugins/calibrate/skills/calibrate/scripts/bench.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""K runs of the inner harness, one at a time, then the objective's scoreboard and failure groups. + + bench.py --runs 3 --package [--goal ...] [--out traces/] + +Each run's workspace (trace.json, observations/) is fetched into /run-N/. The report is +printed and written to /report.json. Never start two runs at once.""" +import argparse +import json +import os +import sys + +import yaml + +sys.path.insert(0, os.path.dirname(__file__)) +import hr # noqa: E402 +import metrics # noqa: E402 + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--runs", type=int, default=3) + ap.add_argument("--package", required=True, help="the inner harness's package directory (config.yaml, ledger.jsonl)") + ap.add_argument("--goal", default=None) + ap.add_argument("--model", default=None) + ap.add_argument("--out", default="traces") + a = ap.parse_args() + cfg = yaml.safe_load(open(os.path.join(a.package, "config.yaml"))) or {} + objective = cfg.get("objective") or {} + goal = a.goal or cfg.get("goal") or "Play." + traces = [] + sessions = [] + for i in range(1, a.runs + 1): + r = hr.start_run(goal, model=a.model) + rid = r["id"] + r = hr.wait_run(rid) + sid = hr.session_of(r) + print(f"run {i}: response {rid} status {r.get('status')} {((r.get('incomplete_details') or {}).get('reason') or '')} session {sid}", flush=True) + if not sid: + sys.exit("the response names no session; cannot fetch its workspace") + ws = hr.fetch_workspace(sid, os.path.join(a.out, f"run-{i}")) + if not ws["trace"]: + # a run without a record is still a run: it counts as a failure with no locus, and + # the bench goes on; the reason is in wait-log.jsonl for the platform + print(f"run {i}: no trace.json in the session's workspace ({ws['dir']}); counted as a failed run", flush=True) + t = {"status": r.get("status"), "reason": ((r.get("incomplete_details") or {}).get("reason") or (r.get("error") or {}).get("message") if isinstance(r.get("error"), dict) else r.get("error")), "steps": [], "started_at": None, "finished_at": None} + else: + t = json.load(open(ws["trace"])) + t["session_id"] = sid + t["observations"] = ws["observations"] + t["response_id"] = rid + traces.append(t) + sessions.append(sid) + rep = metrics.report(traces, objective) + rep["sessions"] = sessions + rep["config_version"] = cfg.get("version") + os.makedirs(a.out, exist_ok=True) + json.dump(rep, open(os.path.join(a.out, "report.json"), "w"), indent=1, default=str) + print("\nscoreboard:", json.dumps(rep["scoreboard"])) + for g in rep["failure_groups"][:8]: + ex = g["examples"][0] if g["examples"] else {} + print(f" {g['count']:3d} x at {g['locus']}: run {ex.get('run')} step {ex.get('step')} {ex.get('action')}: {str(ex.get('text'))[:160]}") + print(f"\nreport: {os.path.join(a.out, 'report.json')}; traces under {a.out}/run-N/") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/plugins/calibrate/skills/calibrate/scripts/fetch.py b/plugins/calibrate/skills/calibrate/scripts/fetch.py new file mode 100644 index 0000000..8f15a8f --- /dev/null +++ b/plugins/calibrate/skills/calibrate/scripts/fetch.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Fetch the inner harness's package (config.yaml, ledger.jsonl, the environment) into a directory. + + fetch.py [--name ] [--out package] + +Without --name, the package that carries the environment (the one with MCP servers) is taken. +Use this, not the harness's plugin export: the export is a generated package named after the +harness and has no version, and publishing it back adds a second package beside the real one.""" +import argparse +import base64 +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) +import hr # noqa: E402 + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--name", default=None) + ap.add_argument("--out", default="package") + a = ap.parse_args() + h = hr.get_harness() + plugins = h.get("plugins") or [] + if not plugins: + sys.exit("the inner harness carries no package") + chosen = None + if a.name: + chosen = next((p for p in plugins if p.get("name") == a.name), None) + else: + chosen = next((p for p in plugins if p.get("mcpServers")), None) or plugins[0] + if chosen is None: + sys.exit(f"no package named {a.name!r}; the harness carries {[p.get('name') for p in plugins]}") + listing = hr.call("GET", f"/v1/harnesses/{hr.HARNESS}/plugins/{chosen['name']}/files") + files = listing.get("files") if isinstance(listing, dict) else listing + if not files: + sys.exit(f"the package {chosen['name']!r} has no files to fetch") + os.makedirs(a.out, exist_ok=True) + n = 0 + for f in files: + rel = f.get("path") + if not rel or rel.startswith("/") or ".." in rel: + continue + p = os.path.join(a.out, rel) + os.makedirs(os.path.dirname(p) or ".", exist_ok=True) + if f.get("content") is not None: + open(p, "w").write(f["content"]) + elif f.get("content_b64") is not None: + open(p, "wb").write(base64.b64decode(f["content_b64"])) + else: + continue + n += 1 + print(f"fetched {chosen['name']} {(chosen.get('manifest') or {}).get('version')}: {n} files into {a.out}/") + for must in ("config.yaml", "ledger.jsonl", "plugin.json"): + if not os.path.exists(os.path.join(a.out, must)): + print(f" note: no {must} in the package") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/plugins/calibrate/skills/calibrate/scripts/hr.py b/plugins/calibrate/skills/calibrate/scripts/hr.py new file mode 100755 index 0000000..d1f24c9 --- /dev/null +++ b/plugins/calibrate/skills/calibrate/scripts/hr.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""The platform client the calibration scripts share: start a run on the inner harness, wait for +it, fetch its workspace, read and write the harness. Credentials: HR_API_URL and +HR_CALIBRATION_TOKEN (a per-turn credential scoped to the inner harness); for a self-hosted box +in development, HR_AUTH_USER and HR_AUTH_PASSWORD log in instead.""" +import io +import json +import os +import sys +import time +import urllib.request +import urllib.error +import http.cookiejar +import zipfile + +API = os.environ.get("HR_API_URL", "").rstrip("/") +TOKEN = os.environ.get("HR_CALIBRATION_TOKEN", "") +HARNESS = os.environ.get("HR_INNER_HARNESS", "") + +_jar = http.cookiejar.CookieJar() +_opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(_jar)) +_logged_in = False + + +def _url(path: str) -> str: + base = API + if not base: + sys.exit("HR_API_URL is not set") + if path.startswith("/v1/") and "/api/harness" not in base and not base.endswith("/v1") and os.environ.get("HR_AUTH_USER"): + base = base + "/api/harness" # the self-hosted console's mount of the API + return base + path + + +def _login() -> None: + global _logged_in + if _logged_in or TOKEN or not os.environ.get("HR_AUTH_USER"): + return + body = json.dumps({"username": os.environ["HR_AUTH_USER"], "password": os.environ["HR_AUTH_PASSWORD"]}).encode() + req = urllib.request.Request(API + "/api/selfhost/login", data=body, headers={"content-type": "application/json"}, method="POST") + _opener.open(req, timeout=60).read() + _logged_in = True + + +def call(method: str, path: str, body=None, raw: bool = False, timeout: int = 120): + _login() + headers = {"content-type": "application/json", "accept": "application/json" if not raw else "*/*"} + if TOKEN: + headers["authorization"] = f"Bearer {TOKEN}" + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(_url(path), data=data, headers=headers, method=method) + try: + with _opener.open(req, timeout=timeout) as r: + payload = r.read() + except urllib.error.HTTPError as e: + sys.exit(f"{method} {path} -> {e.code}: {e.read()[:300].decode(errors='replace')}") + return payload if raw else (json.loads(payload) if payload else {}) + + +def start_run(goal: str, harness: str | None = None, model: str | None = None, script: list[str] | None = None, + max_step: int | None = None, timeout_seconds: int | None = None) -> dict: + """Start one run in the background; returns the response object.""" + meta = {"harness_id": harness or HARNESS} + if script: + meta["systemone"] = {"script": list(script)} + body = {"input": goal, "metadata": meta, "background": True, "store": True} + if model: + body["model"] = model + if max_step: + body["max_step"] = max_step + if timeout_seconds: + body["timeout_seconds"] = timeout_seconds + return call("POST", "/v1/responses", body) + + +TERMINAL = ("completed", "failed", "incomplete", "cancelled") +LOG = os.environ.get("HR_CALIBRATION_LOG", "wait-log.jsonl") + + +def _log(kind: str, payload) -> None: + try: + with open(LOG, "a") as f: + f.write(json.dumps({"at": time.time(), "kind": kind, "payload": payload}, default=str)[:4000] + "\n") + except OSError: + pass + + +def session_running(session_id: str | None) -> bool: + if not session_id: + return False + try: + s = call("GET", f"/v1/sessions/{session_id}") + except SystemExit: + return False + return str(s.get("status") or "") in ("running", "in_progress", "queued") + + +def wait_run(response_id: str, poll: float = 5.0, limit: float = 3600.0) -> dict: + """Blocks until the run has ended: the response carries a terminal status AND its session is no + longer running. A server may say `running` or `queued` for work in progress, and a transient + `failed` was seen on a response whose turn was still running (2026-09-21), so a non-completed + terminal status is re-read a few times before it is believed. Every status read that is not + `completed` is logged to wait-log.jsonl for the platform to chase.""" + t0 = time.time() + confirmations = 0 + while True: + r = call("GET", f"/v1/responses/{response_id}") + status = r.get("status") + if status != "completed": + _log("status", {"response_id": response_id, "status": status, "error": r.get("error"), + "incomplete_details": r.get("incomplete_details"), "metadata": r.get("metadata")}) + if status in TERMINAL: + sid = session_of(r) + if session_running(sid): + confirmations = 0 + elif status == "completed": + return r + else: + confirmations += 1 + if confirmations >= 4: # about twenty seconds of the same terminal status with the session at rest + return r + if time.time() - t0 > limit: + sys.exit(f"run {response_id} still not finished after {limit:.0f}s") + time.sleep(poll) + + +def session_of(response: dict) -> str | None: + """The session the response ran in: named on the response, or else the inner harness's newest session.""" + meta = response.get("metadata") or {} + sid = meta.get("session_id") or response.get("session_id") or meta.get("harness_session_id") + if sid: + return sid + listing = call("GET", f"/v1/sessions?harness={HARNESS}&limit=1") + sessions = listing.get("sessions") or [] + return sessions[0].get("session_id") if sessions else None + + +def fetch_workspace(session_id: str, out_dir: str) -> dict: + """The session's files as one archive, unpacked; returns the paths of trace.json and observations/.""" + os.makedirs(out_dir, exist_ok=True) + blob = None + for attempt in range(12): + # the archive is built from the session's checkpoint, which lands once the turn has ended; + # a 404 right after the run means "not yet", so wait and ask again + try: + blob = call("GET", f"/v1/sessions/{session_id}/files/archive", raw=True, timeout=600) + break + except SystemExit as e: + _log("archive", {"session_id": session_id, "attempt": attempt, "error": str(e)[:300]}) + if "404" not in str(e) and "409" not in str(e): + raise + time.sleep(10) + if blob is None: + return {"trace": None, "observations": None, "dir": out_dir, "missing": True} + with zipfile.ZipFile(io.BytesIO(blob)) as z: + z.extractall(out_dir) + trace = None + obs = None + for root, dirs, files in os.walk(out_dir): + if "trace.json" in files and trace is None: + trace = os.path.join(root, "trace.json") + if os.path.basename(root) == "observations" and obs is None: + obs = root + return {"trace": trace, "observations": obs, "dir": out_dir} + + +def get_harness(harness: str | None = None) -> dict: + return call("GET", f"/v1/harnesses/{harness or HARNESS}") + + +def put_harness(body: dict, harness: str | None = None) -> dict: + return call("PUT", f"/v1/harnesses/{harness or HARNESS}", body) + + +if __name__ == "__main__": + print(json.dumps(get_harness(sys.argv[1] if len(sys.argv) > 1 else None), indent=1)[:2000]) diff --git a/plugins/calibrate/skills/calibrate/scripts/metrics.py b/plugins/calibrate/skills/calibrate/scripts/metrics.py new file mode 100755 index 0000000..2e93efe --- /dev/null +++ b/plugins/calibrate/skills/calibrate/scripts/metrics.py @@ -0,0 +1,190 @@ +"""The objective: what an environment declares as success, failure, ordered metrics and place. + +The outer loop reads only this declaration; it knows nothing about lives, levels or tests. The +predicates are small on purpose: + + pass: " == " | "" (truthy) | "terminal " + failure: " decreased" | " increased" | "terminal " | " == " + metrics: [{field: , better: true | false | higher | lower}, ...] ordered + locus: [, ...] where a failure happened + +`evaluate` reads a run record (the trace's dict) and returns the metrics and the failures; +`report` groups the failures of several runs by locus; `compare` says whether a run set improved. +""" +from __future__ import annotations + +import re +from statistics import mean + +ELAPSED = "elapsed" +FAILURES = "failures" + + +def _fields_of(step: dict) -> dict: + r = step.get("result") or {} + return dict(r.get("fields") or {}) if isinstance(r, dict) else {} + + +def _value(fields: dict, name: str): + return fields.get(name) + + +def _literal(text: str): + t = text.strip() + if t.lower() in ("true", "false"): + return t.lower() == "true" + if t.lower() in ("null", "none"): + return None + try: + return int(t) + except ValueError: + pass + try: + return float(t) + except ValueError: + return t.strip("'\"") + + +def _pass(run: dict, expr: str, final: dict) -> bool: + e = (expr or "").strip() + if not e: + return run.get("status") == "completed" + m = re.match(r"^terminal\s+(\S+)$", e) + if m: + return run.get("reason") == m.group(1) + m = re.match(r"^(\w+)\s*==\s*(.+)$", e) + if m: + return _value(final, m.group(1)) == _literal(m.group(2)) + return bool(_value(final, e)) + + +def _failure_steps(run: dict, expr: str) -> list[dict]: + """Each failure as {step, locus fields, last live text}: the step at which the failure signal fired.""" + e = (expr or "").strip() + steps = run.get("steps") or [] + out: list[dict] = [] + m = re.match(r"^(\w+)\s+(decreased|increased)$", e) + if m: + name, direction = m.group(1), m.group(2) + prev = None + last_live: dict | None = None + for st in steps: + f = _fields_of(st) + v = f.get(name) + if v is None: + continue + fired = prev is not None and ((direction == "decreased" and v < prev) or (direction == "increased" and v > prev)) + if fired: + src = last_live if last_live is not None else st + out.append({"step": int(src.get("index", 0)), "action": src.get("action"), "fields": _fields_of(src), + "text": str((src.get("result") or {}).get("text") or "")[:400]}) + prev = v + if not f.get("restarting") and not f.get("dead"): + last_live = st + return out + m = re.match(r"^terminal\s+(\S+)$", e) + if m: + if run.get("reason") == m.group(1) and steps: + st = steps[-1] + out.append({"step": int(st.get("index", 0)), "action": st.get("action"), "fields": _fields_of(st), + "text": str((st.get("result") or {}).get("text") or "")[:400]}) + return out + m = re.match(r"^(\w+)\s*==\s*(.+)$", e) + if m: + want = _literal(m.group(2)) + for st in steps: + if _fields_of(st).get(m.group(1)) == want: + out.append({"step": int(st.get("index", 0)), "action": st.get("action"), "fields": _fields_of(st), + "text": str((st.get("result") or {}).get("text") or "")[:400]}) + return out + return out + + +def evaluate(run: dict, objective: dict) -> dict: + """The metrics of one run under the objective, and its failures.""" + steps = run.get("steps") or [] + final: dict = {} + for st in steps: + f = _fields_of(st) + if f: + final = {**final, **f} + failures = _failure_steps(run, str(objective.get("failure") or "")) + started, finished = run.get("started_at"), run.get("finished_at") + elapsed = round(float(finished) - float(started), 1) if started is not None and finished is not None else None + metrics: dict = {"pass": _pass(run, str(objective.get("pass") or ""), final), FAILURES: len(failures), ELAPSED: elapsed} + for m in objective.get("metrics") or []: + name = m.get("field") + if name in (FAILURES, ELAPSED, "pass"): + continue + metrics[name] = final.get(name) + return {"metrics": metrics, "failures": failures, "final": final, "status": run.get("status"), "reason": run.get("reason"), + "steps": len(steps), "config_version": run.get("config_version")} + + +def _better(direction, a, b): + """True when a is better than b for the direction (true/false as target values, higher/lower).""" + if a is None or b is None: + return False + if isinstance(direction, bool) or str(direction).lower() in ("true", "false"): + want = direction if isinstance(direction, bool) else str(direction).lower() == "true" + return (a == want) and (b != want) + if str(direction).lower() == "higher": + return a > b + return a < b + + +def _aggregate(evals: list[dict], name: str): + vals = [e["metrics"].get(name) for e in evals] + vals = [v for v in vals if v is not None] + if not vals: + return None + if all(isinstance(v, bool) for v in vals): + return round(sum(1 for v in vals if v) / len(vals), 3) # a rate + return round(mean(float(v) for v in vals), 2) + + +def scoreboard(evals: list[dict], objective: dict) -> dict: + names = ["pass", FAILURES] + [m.get("field") for m in objective.get("metrics") or [] if m.get("field") not in ("pass", FAILURES, ELAPSED)] + [ELAPSED] + board = {n: _aggregate(evals, n) for n in names} + board["runs"] = len(evals) + return board + + +def compare(before: list[dict], after: list[dict], objective: dict) -> str: + """kept | reverted | same, by the ordered metrics; the first metric that differs decides.""" + b, a = scoreboard(before, objective), scoreboard(after, objective) + order = [("pass", True), (FAILURES, "lower")] + [(m.get("field"), m.get("better")) for m in objective.get("metrics") or [] + if m.get("field") not in ("pass", FAILURES)] + for name, direction in order: + x, y = a.get(name), b.get(name) + if x is None or y is None or x == y: + continue + if name == "pass": + return "kept" if x > y else "reverted" + if isinstance(direction, bool) or str(direction).lower() in ("true", "false"): + return "kept" if x > y else "reverted" + return "kept" if (str(direction).lower() == "higher" and x > y) or (str(direction).lower() == "lower" and x < y) else "reverted" + return "same" + + +def report(runs: list[dict], objective: dict) -> dict: + """The scoreboard over runs and the failures grouped by locus, largest group first.""" + evals = [evaluate(r, objective) for r in runs] + locus = list(objective.get("locus") or []) + groups: dict = {} + for i, e in enumerate(evals): + for f in e["failures"]: + key = tuple(_bucket(f["fields"].get(l)) for l in locus) if locus else ("*",) + g = groups.setdefault(key, {"locus": dict(zip(locus, key)) if locus else {}, "count": 0, "examples": []}) + g["count"] += 1 + if len(g["examples"]) < 3: + g["examples"].append({"run": i, "step": f["step"], "action": f["action"], "text": f["text"]}) + ordered = sorted(groups.values(), key=lambda g: -g["count"]) + return {"scoreboard": scoreboard(evals, objective), "runs": evals, "failure_groups": ordered} + + +def _bucket(v, width: float = 5.0): + """Numbers are grouped in bands so nearby failures fall together.""" + if isinstance(v, (int, float)) and not isinstance(v, bool): + return int(v // width) * width + return v diff --git a/plugins/calibrate/skills/calibrate/scripts/probe.py b/plugins/calibrate/skills/calibrate/scripts/probe.py new file mode 100755 index 0000000..5c9669f --- /dev/null +++ b/plugins/calibrate/skills/calibrate/scripts/probe.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""A probe: one run of the inner harness driven by a fixed action sequence instead of its model, +to measure the environment. Prints the trace's steps with their state text. + + probe.py "run_right,run_right,jump_right,jump_right" [--goal ...] [--out probe/]""" +import argparse +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) +import hr # noqa: E402 + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("script") + ap.add_argument("--goal", default="Probe.") + ap.add_argument("--out", default="probe") + a = ap.parse_args() + actions = [x.strip() for x in a.script.split(",") if x.strip()] + r = hr.wait_run(hr.start_run(a.goal, script=actions, max_step=len(actions) + 2)["id"]) + sid = hr.session_of(r) + ws = hr.fetch_workspace(sid, a.out) + if not ws["trace"]: + sys.exit("no trace.json came back") + t = json.load(open(ws["trace"])) + for s in t.get("steps") or []: + print(f"{s.get('index'):3d} {s.get('action'):12s} {str((s.get('result') or {}).get('text') or '')[:200]}") + print(f"\nsession {sid}; workspace under {a.out}/") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/plugins/calibrate/skills/calibrate/scripts/publish.py b/plugins/calibrate/skills/calibrate/scripts/publish.py new file mode 100755 index 0000000..395c799 --- /dev/null +++ b/plugins/calibrate/skills/calibrate/scripts/publish.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Publish the package as the inner harness's plugin: the new config.yaml and ledger.jsonl travel +with it, and the harness's instructions follow config.yaml. Every other harness setting is kept. + + publish.py --package """ +import argparse +import base64 +import json +import os +import sys + +import yaml + +sys.path.insert(0, os.path.dirname(__file__)) +import hr # noqa: E402 + +SKIP_DIRS = {"__pycache__", "node_modules", "observations", ".git"} +TEXT = {".py", ".md", ".json", ".yaml", ".yml", ".txt", ".sh", ".toml", ".cfg", ""} + + +def package_files(root: str) -> list[dict]: + out = [] + for dirpath, dirs, files in os.walk(root): + dirs[:] = [d for d in dirs if d not in SKIP_DIRS] + for name in files: + p = os.path.join(dirpath, name) + rel = os.path.relpath(p, root).replace(os.sep, "/") + data = open(p, "rb").read() + ext = os.path.splitext(name)[1].lower() + if ext in TEXT: + try: + out.append({"path": rel, "content": data.decode("utf-8")}) + continue + except UnicodeDecodeError: + pass + out.append({"path": rel, "content_b64": base64.b64encode(data).decode()}) + return out + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--package", required=True) + ap.add_argument("--new", action="store_true", help="add a package the harness does not carry yet") + a = ap.parse_args() + cfg = yaml.safe_load(open(os.path.join(a.package, "config.yaml"))) or {} + manifest = json.load(open(os.path.join(a.package, "plugin.json"))) + h = hr.get_harness() + files = package_files(a.package) + names = [p.get("name") for p in (h.get("plugins") or [])] + if not manifest.get("version"): + sys.exit(f"{a.package}/plugin.json has no version; fetch the real package with fetch.py, never the harness's export") + if manifest["name"] not in names and not a.new: + sys.exit(f"the harness carries {names}, not {manifest['name']!r}; a publish replaces the same-named package (use --new to add one)") + others = [p for p in (h.get("plugins") or []) if p.get("name") != manifest["name"]] + body = {"name": h["name"], "base": h["base"], "defaultModel": h.get("defaultModel"), + "mcpServers": h.get("mcpServers", []), "skills": h.get("skills", []), + "plugins": others + [{"name": manifest["name"], "files": files, "enabled": True}], + "disabledTools": h.get("disabledTools", []), "additionalHeaders": h.get("additionalHeaders", []), + "timeoutSeconds": h.get("timeoutSeconds"), + "system_prompt": cfg.get("instructions") or h.get("systemPrompt") or "", + "max_step": h.get("maxStep") or h.get("max_step")} + r = hr.put_harness(body) + pv = [(p["name"], (p.get("manifest") or {}).get("version")) for p in r.get("plugins") or []] + print(f"published config v{cfg.get('version')} of {manifest['name']} {manifest.get('version')} on {r.get('id')}: plugins {pv}, prompt {len(r.get('systemPrompt') or '')} chars") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/plugins/calibrate/skills/calibrate/scripts/report.py b/plugins/calibrate/skills/calibrate/scripts/report.py new file mode 100755 index 0000000..211ee2f --- /dev/null +++ b/plugins/calibrate/skills/calibrate/scripts/report.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""The objective's scoreboard and failure groups over traces already on disk. + + report.py --package traces/run-*/trace.json""" +import argparse +import json +import os +import sys + +import yaml + +sys.path.insert(0, os.path.dirname(__file__)) +import metrics # noqa: E402 + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--package", required=True) + ap.add_argument("traces", nargs="+") + a = ap.parse_args() + objective = (yaml.safe_load(open(os.path.join(a.package, "config.yaml"))) or {}).get("objective") or {} + rep = metrics.report([json.load(open(p)) for p in a.traces], objective) + print("scoreboard:", json.dumps(rep["scoreboard"])) + for g in rep["failure_groups"][:8]: + ex = g["examples"][0] if g["examples"] else {} + print(f" {g['count']:3d} x at {g['locus']}: run {ex.get('run')} step {ex.get('step')} {ex.get('action')}: {str(ex.get('text'))[:160]}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())