From e08fb7b320ecb953e5a00290400e5d77f558a4a6 Mon Sep 17 00:00:00 2001 From: Michael Avrukin Date: Sun, 8 Nov 2026 22:41:56 -0500 Subject: [PATCH 1/4] plugin: run the loop guard's judge on Apple's on-device model (local_model=fm) macOS 27 ships Apple's Foundation Models CLI at /usr/bin/fm: a ~3B model on the Neural Engine and GPU, 4,096-token window, no login or key. With local_model=fm the side calls local_roles names (default: judge) run on it instead of the agent's CLI, and are charged 0 tokens. - sidecall: fm_available, fm_call (prompt on stdin, greedy, optional JSON schema via `fm respond --schema`), local_runner, and a role argument to runner_for. A local call that fails (no fm, model not ready, prompt over the window, no reply) falls back to the agent's CLI, so the guard never loses a verdict; while that CLI is paused (signed out), the local judge still answers. - judge: the judge passes its verdict schema; under it the model returns the object every time, where in free text it often wraps or contradicts it. - local_judge_confidence (default 0.9): a stuck verdict from the local judge counts only at this floor. Its confidences come in coarse steps (0.65/0.72/0.85/0.92), most above judge_confidence (0.7). - Settings local_model, local_roles, local_judge_confidence (and TE_* vars); documented in plugin/README.md. Off by default. - tests/test_local_model.py; conftest fakes fm like the other side calls. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- plugin/README.md | 24 +++ tests/conftest.py | 3 + tests/test_local_model.py | 230 +++++++++++++++++++++++++++ tokeneconomizer/plugin/config.py | 36 +++++ tokeneconomizer/plugin/controller.py | 8 +- tokeneconomizer/plugin/fold.py | 8 +- tokeneconomizer/plugin/judge.py | 24 ++- tokeneconomizer/plugin/sidecall.py | 100 +++++++++++- 9 files changed, 424 insertions(+), 13 deletions(-) create mode 100644 tests/test_local_model.py diff --git a/README.md b/README.md index 449a089..bc73164 100644 --- a/README.md +++ b/README.md @@ -336,7 +336,9 @@ only when a turn shows signs of looping: the judge, and the redirect it writes i turn stuck. Micro-skills is off by default, because its proposer adds model calls and how much it saves depends on the repository and the work you do there. What each of these calls reads, how often it runs and which model it uses are listed under -[Model calls](plugin/README.md#model-calls-and-api-keys). Each mechanism has its own switch +[Model calls](plugin/README.md#model-calls-and-api-keys). On macOS 27, `local_model=fm` runs the +judge on Apple's on-device model instead ([On-device side calls](plugin/README.md#on-device-side-calls-macos)); +it is off by default. Each mechanism has its own switch (`microskills`, `toolprune`, `loopguard`) that takes `on` or `off`, and the italic letters above are these settings: diff --git a/plugin/README.md b/plugin/README.md index faabd0e..1cb0d8b 100644 --- a/plugin/README.md +++ b/plugin/README.md @@ -796,6 +796,9 @@ first session there ends. |---|---|---| | `enabled` | `true` | `false`: the whole plugin is off; its hooks return at once and record nothing | | `paid_model` | `""` | model for the plugin's own calls; empty: `haiku` on Claude Code, the agent's own default model on Codex and Pi, on Copilot `COPILOT_PROVIDER_MODEL_ID` or `COPILOT_MODEL` (unless it is `auto`), else a BYOK provider's Haiku, else Copilot's own default | +| `local_model` | `off` | `fm`: run the side calls `local_roles` names on Apple's on-device Foundation Models CLI (`/usr/bin/fm`, macOS 27 with Apple Intelligence) instead of the agent's CLI; see [On-device side calls](#on-device-side-calls-macos) | +| `local_roles` | `judge` | which side calls `local_model` takes: `judge`, `nudge` (the redirect writer), `proposer`, separated by commas | +| `local_judge_confidence` | `0.9` | how sure a stuck verdict from the on-device judge must be to count; below it the turn is spared | | `pi_extensions` | `""` | Pi only: extension files (comma-separated paths) the plugin's own calls load, for a model whose provider an extension registers | | `pi_cli` | `""` | Pi only: the `pi` the plugin's own calls run; empty: the first on PATH | | `shadow` | `false` | decide and log everything, act on nothing | @@ -903,6 +906,27 @@ at its backstop), the next session start says so once, and `config set microskil `loopguard=on`) tries again. `toolprune measure` on Claude Code makes its two calls the same way, and only when you run it. +## On-device side calls (macOS) + +With `local_model=fm`, the side calls `local_roles` names (by default the loop guard's judge) run +on the Foundation Models CLI that macOS 27 ships at `/usr/bin/fm` when Apple Intelligence is on: +a model on the Mac itself, with no login, key or network. The plugin sends it the same prompt it +would send the agent's CLI, on stdin, with greedy decoding; the judge's call also passes a JSON +schema for its verdict (`fm respond --schema`), which constrains what the model can return. Its +window is 4,096 tokens, enough for the judge's and the redirect writer's prompts. + +The ledger charges an on-device call 0 tokens. A call it cannot make (no `fm`, the model not +ready, a prompt over its window, no reply) leaves a line in `errors.log` and goes to the agent's +CLI as before, so turning this on never leaves the guard without its judge; while that CLI is +paused (signed out), the on-device judge still answers. A stuck verdict from the on-device judge +counts only at `local_judge_confidence` (0.9) or above: its confidences come in a few coarse +steps, most of them above the guard's own `judge_confidence` (0.7). + +```text +~/.tokeneconomizer/bin/te config set local_model=fm +~/.tokeneconomizer/bin/te config set local_roles=judge,nudge # also the redirect writer +``` + ## Troubleshooting | Symptom | Fix | diff --git a/tests/conftest.py b/tests/conftest.py index fe7119e..5640bd5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -60,3 +60,6 @@ def _no_model_calls(request, monkeypatch): for name in ("claude_call", "codex_call", "copilot_call", "grok_call", "hermes_call", "pi_call"): monkeypatch.setattr(sidecall, name, lambda *a, **k: ("", None)) + # Nor the on-device model: `fm` answers nothing, as if absent, unless a test fakes it. + monkeypatch.setattr(sidecall, "fm_call", lambda *a, **k: (None, 0)) + monkeypatch.setattr(sidecall, "_fm_ok", {}) diff --git a/tests/test_local_model.py b/tests/test_local_model.py new file mode 100644 index 0000000..b771db7 --- /dev/null +++ b/tests/test_local_model.py @@ -0,0 +1,230 @@ +"""The on-device model some side calls can run on (`local_model`, `local_roles`). + +`fm` is faked throughout: these tests pin the routing, the fallback to the agent's own CLI, the +zero charge, and the call itself (argv, schema, stdin, ANSI stripping), never the model. +""" +import json +import subprocess + +import pytest + +from tokeneconomizer.loopguard import judge as loop_judge +from tokeneconomizer.plugin import config as live_config +from tokeneconomizer.plugin import sidecall + + +def cfg(**kw): + c = live_config.LiveConfig() + for k, v in kw.items(): + assert c.set(k, v), c.warnings + return c + + +# ---- settings ------------------------------------------------------------------------------ +def test_local_model_is_off_by_default_and_names_the_judge(): + c = live_config.LiveConfig() + assert c.local_model == "off" and c.local_roles == ["judge"] + + +def test_local_settings_parse_and_reject_unknown_values(): + c = live_config.LiveConfig() + assert c.set("local_model", "fm") and c.local_model == "fm" + assert c.set("local_roles", "judge, nudge") and c.local_roles == ["judge", "nudge"] + assert not c.set("local_model", "ollama") and c.local_model == "fm" + assert not c.set("local_roles", "judge,summarizer") and c.local_roles == ["judge", "nudge"] + assert len(c.warnings) == 2 + + +def test_local_settings_come_from_the_environment(tmp_path): + c = live_config.load_config(str(tmp_path), env={"TE_LOCAL_MODEL": "fm", + "TE_LOCAL_ROLES": "judge,proposer"}) + assert (c.local_model, c.local_roles) == ("fm", ["judge", "proposer"]) + + +# ---- routing ------------------------------------------------------------------------------- +@pytest.fixture +def fm_ok(monkeypatch): + calls = [] + + def fake(prompt, schema=None, **k): + calls.append((prompt, schema)) + return '{"stuck": true, "confidence": 0.9, "reason": "same failure"}', 0 + + monkeypatch.setattr(sidecall, "fm_available", lambda cli="fm": True) + monkeypatch.setattr(sidecall, "fm_call", fake) + return calls + + +def test_no_local_runner_unless_turned_on_and_the_role_listed(fm_ok): + assert sidecall.local_runner("judge", cfg()) is None + assert sidecall.local_runner("nudge", cfg(local_model="fm")) is None + assert sidecall.local_runner("judge", cfg(local_model="fm")) is not None + + +def test_no_local_runner_without_fm(monkeypatch): + monkeypatch.setattr(sidecall, "fm_available", lambda cli="fm": False) + assert sidecall.local_runner("judge", cfg(local_model="fm")) is None + + +def test_runner_for_a_local_role_uses_fm_with_its_schema_and_charges_nothing(fm_ok, monkeypatch): + host = [] + monkeypatch.setattr(sidecall, "claude_call", lambda p, model="": host.append(p) or ("h", 9.0)) + run = sidecall.runner_for("claude", cfg(local_model="fm"), role="judge") + text, eff = run("digest", "agent") + assert json.loads(text)["stuck"] is True and eff == 0 and host == [] + assert fm_ok == [("digest", sidecall.JUDGE_SCHEMA)] + + +def test_runner_for_without_a_role_or_for_another_role_stays_on_the_host(fm_ok, monkeypatch): + monkeypatch.setattr(sidecall, "claude_call", lambda p, model="": ("h", 9.0)) + c = cfg(local_model="fm") + assert sidecall.runner_for("claude", c)("q") == ("h", 9.0) + assert sidecall.runner_for("claude", c, role="proposer")("q") == ("h", 9.0) + assert fm_ok == [] + + +def test_a_failed_local_call_falls_back_to_the_agents_cli(monkeypatch): + monkeypatch.setattr(sidecall, "fm_available", lambda cli="fm": True) + monkeypatch.setattr(sidecall, "fm_call", lambda *a, **k: (None, 0)) + monkeypatch.setattr(sidecall, "claude_call", lambda p, model="": ("host verdict", 9.0)) + run = sidecall.runner_for("claude", cfg(local_model="fm"), role="judge") + assert run("digest") == ("host verdict", 9.0) + + +def test_a_host_with_no_side_calls_gets_none_from_runner_for_and_an_empty_local_fallback( + fm_ok, monkeypatch): + monkeypatch.setattr(sidecall, "fm_call", lambda *a, **k: (None, 0)) + c = cfg(local_model="fm", local_roles="proposer") + assert sidecall.runner_for("antigravity", cfg()) is None + run = sidecall.runner_for("antigravity", c, role="proposer") + assert run is not None and run("q") == ("", 0) + + +# ---- the call ------------------------------------------------------------------------------ +@pytest.mark.real_sidecalls +def test_fm_call_runs_greedy_with_the_schema_file_and_the_prompt_on_stdin(monkeypatch): + seen = {} + + def fake_run(cmd, **k): + seen["cmd"], seen["input"] = cmd, k.get("input") + seen["schema"] = json.load(open(cmd[cmd.index("--schema") + 1])) + return subprocess.CompletedProcess(cmd, 0, stdout='\x1b[1m{"stuck": false}\x1b[0m\n', + stderr="") + + monkeypatch.setattr(sidecall.subprocess, "run", fake_run) + text, eff = sidecall.fm_call("PROMPT", schema=sidecall.JUDGE_SCHEMA) + assert (text, eff) == ('{"stuck": false}', 0) + assert seen["cmd"][:4] == ["fm", "respond", "--no-stream", "--greedy"] + assert seen["input"] == "PROMPT" and "PROMPT" not in seen["cmd"] + assert seen["schema"]["required"] == ["stuck", "confidence", "reason"] + + +@pytest.mark.real_sidecalls +@pytest.mark.parametrize("rc, out", [(1, ""), (0, ""), (0, "Error: The session's transcript " + "exceeded the model's context size.")]) +def test_fm_call_without_a_usable_reply_returns_none_and_logs(tmp_path, monkeypatch, rc, out): + monkeypatch.setenv("TOKENECONOMIZER_HOME", str(tmp_path)) + monkeypatch.setattr(sidecall.subprocess, "run", lambda cmd, **k: subprocess.CompletedProcess( + cmd, rc, stdout=out, stderr="")) + assert sidecall.fm_call("SECRET prompt") == (None, 0) + log = (tmp_path / "errors.log").read_text(encoding="utf-8") + assert "sidecall fm: no reply" in log and "SECRET" not in log + + +@pytest.mark.real_sidecalls +def test_fm_call_that_cannot_start_returns_none(tmp_path, monkeypatch): + monkeypatch.setenv("TOKENECONOMIZER_HOME", str(tmp_path)) + + def boom(cmd, **k): + raise subprocess.TimeoutExpired(cmd, 60) + + monkeypatch.setattr(sidecall.subprocess, "run", boom) + assert sidecall.fm_call("q") == (None, 0) + + +@pytest.mark.real_sidecalls +@pytest.mark.parametrize("which, rc, out, ok", [ + (None, 0, "System model available", False), + ("/usr/bin/fm", 0, "System model available", True), + ("/usr/bin/fm", 0, "System model unavailable: modelNotReady", False), + ("/usr/bin/fm", 1, "", False), +]) +def test_fm_available(monkeypatch, which, rc, out, ok): + monkeypatch.setattr(sidecall, "_fm_ok", {}) + monkeypatch.setattr(sidecall.shutil, "which", lambda name: which) + monkeypatch.setattr(sidecall.subprocess, "run", lambda cmd, **k: subprocess.CompletedProcess( + cmd, rc, stdout=out, stderr="")) + assert sidecall.fm_available() is ok + + +# ---- the judge reads the local verdict ----------------------------------------------------- +def test_the_stuck_judge_parses_a_schema_ordered_local_verdict_at_zero_cost(fm_ok): + run = sidecall.runner_for("claude", cfg(local_model="fm"), role="judge") + v = loop_judge.llm_stuck_judge("turns=20 ...", run) + assert (v["stuck"], v["confidence"], v["_tokens"]) == (True, 0.9, 0) + + +# ---- the judge job: the local floor -------------------------------------------------------- +class _Store: + def __init__(self, req): + self.files = {"judge/S.request.json": req} + self.appended = [] + + def read(self, k): + return self.files.get(k) + + def write(self, k, v): + self.files[k] = v + + def append(self, k, v): + self.appended.append((k, v)) + + +def _job(tmp_path, monkeypatch, confidence, **cfg_kw): + from tokeneconomizer.plugin import judge as plugin_judge + monkeypatch.setattr(sidecall, "fm_available", lambda cli="fm": True) + verdict = json.dumps({"stuck": True, "confidence": confidence, "reason": "same failure"}) + monkeypatch.setattr(sidecall, "fm_call", lambda prompt, schema=None, **k: ( + verdict if schema else "Re-read the failing test before editing again.", 0)) + store = _Store({"id": "J", "digest": "turns=20 ...", "threshold": 0.7}) + host = type("H", (), {"name": "claude"})() + ctx = type("C", (), {"host": host, "root": str(tmp_path), "store": store, + "config": cfg(local_model="fm", local_roles="judge,nudge", **cfg_kw)})() + v = plugin_judge.run_job(ctx, "S", "J") + return v, store + + +def test_a_local_stuck_verdict_below_the_floor_spares_the_turn(tmp_path, monkeypatch): + v, store = _job(tmp_path, monkeypatch, 0.85) + assert v["stuck"] is False and v["local"] == "fm" and "_nudge" not in v + assert v["reason"].startswith("below local_judge_confidence") + assert store.appended == [] # nothing spent, nothing charged + + +def test_a_local_stuck_verdict_at_the_floor_redirects_with_a_local_nudge(tmp_path, monkeypatch): + v, _ = _job(tmp_path, monkeypatch, 0.92) + assert v["stuck"] is True and v["_nudge"].startswith("Re-read") + + +def test_the_floor_is_a_setting(tmp_path, monkeypatch): + v, _ = _job(tmp_path, monkeypatch, 0.85, local_judge_confidence=0.8) + assert v["stuck"] is True + + +def test_the_floor_does_not_apply_when_the_host_answered(tmp_path, monkeypatch): + from tokeneconomizer.plugin import judge as plugin_judge + monkeypatch.setattr(sidecall, "fm_available", lambda cli="fm": True) + monkeypatch.setattr(sidecall, "fm_call", lambda *a, **k: (None, 0)) + monkeypatch.setattr(sidecall, "claude_call", lambda p, model="": ( + '{"stuck": true, "confidence": 0.75, "reason": "loop"}', 5.0)) + store = _Store({"id": "J", "digest": "turns=20 ...", "threshold": 0.7}) + ctx = type("C", (), {"host": type("H", (), {"name": "claude"})(), "root": str(tmp_path), + "store": store, "config": cfg(local_model="fm")})() + v = plugin_judge.run_job(ctx, "S", "J") + assert v["stuck"] is True and "local" not in v and store.appended # host spend charged + + +def test_local_judge_confidence_must_be_a_probability(): + c = live_config.LiveConfig() + assert c.set("local_judge_confidence", "0.8") and c.local_judge_confidence == 0.8 + assert not c.set("local_judge_confidence", 1.5) and not c.set("local_judge_confidence", True) diff --git a/tokeneconomizer/plugin/config.py b/tokeneconomizer/plugin/config.py index 3e58871..c139323 100644 --- a/tokeneconomizer/plugin/config.py +++ b/tokeneconomizer/plugin/config.py @@ -46,6 +46,19 @@ * loopguard. -- one guard threshold, pinned over the preset and over what the repository's history learned (LOOPGUARD_KEYS); * repomap, repomap_tokens, repomap_python, repomap_timeout -- the opt-in repository map; + * local_model, local_roles -- run the named side calls (`judge`, `nudge`, `proposer`; + default `judge`) on an on-device model instead of the + agent's CLI: `fm`, Apple's Foundation Models CLI on macOS 27. + Off by default. A call the local model cannot make -- no `fm`, + model unavailable, no reply -- falls back to the agent's CLI. + Local calls are charged 0 tokens; + * local_judge_confidence -- the confidence a stuck verdict from the local judge must reach + to count (default 0.9); below it the turn is spared. The + local model's confidences cluster (0.65/0.72/0.85/0.92), so + the guard's own `judge_confidence` (0.7) would let nearly + every local "stuck" through; 0.9 is where, on 302 replayed + SWE-agent judge points, it redirected fewer successful runs + than Haiku at 0.7 while keeping the same separation; * pi_extensions, pi_cli -- on Pi, the extensions the plugin's own model calls load (a custom provider's, whose model they could not reach otherwise; comma-separated paths), and the `pi` they run; @@ -78,6 +91,12 @@ from dataclasses import asdict, dataclass, field, fields MICROSKILLS = ("off", "envfacts", "envfacts+distiller", "distiller") +# An on-device model some side calls can run on instead of the agent's CLI (`sidecall.local_runner`): +# "fm" is Apple's Foundation Models CLI (`/usr/bin/fm`, macOS 27 with Apple Intelligence). +LOCAL_MODELS = ("off", "fm") +# The side calls `local_roles` may name. The judge is the default because it is the one the +# evaluation supports; the redirect writer and the proposer are opt-in. +LOCAL_ROLES = ("judge", "nudge", "proposer") TOOLPRUNE = ("off", "propose", "auto") LOOPGUARD = ("off", "nudge", "stop") REPOMAP = ("off", "on", "start") @@ -174,6 +193,9 @@ class LiveConfig: repomap_python: str = "" repomap_timeout: int = 8 paid_model: str = "" + local_model: str = "off" + local_roles: list = field(default_factory=lambda: ["judge"]) + local_judge_confidence: float = 0.9 pi_extensions: list = field(default_factory=list) pi_cli: str = "" hermes_cli: str = "" @@ -187,6 +209,7 @@ class LiveConfig: _CHOICES = {"mode": MODES, "microskills": MICROSKILLS, "toolprune": TOOLPRUNE, "loopguard": LOOPGUARD, "repomap": REPOMAP, "loopguard_aggr": AGGR, + "local_model": LOCAL_MODELS, "vocabulary": ("auto", "generic", "python-git", "node-web")} # The smallest value a number setting takes, where it is not 1. _MIN = {"toolprune_quiet_minutes": 0} @@ -226,6 +249,12 @@ def set(self, key, value, source="?"): if isinstance(cur, list): items = value if isinstance(value, list) else str(value).split(",") v = [str(x).strip() for x in items if str(x).strip()] + elif isinstance(cur, float): + if isinstance(value, bool): + raise ValueError("not a number") + v = float(value) + if not 0.0 <= v <= 1.0: + raise ValueError("must be between 0 and 1") elif isinstance(cur, bool): v = value if isinstance(value, bool) else str(value).strip().lower() in ( "1", "true", "yes", "on") @@ -243,6 +272,11 @@ def set(self, key, value, source="?"): except (TypeError, ValueError, OverflowError) as e: self.warnings.append(f"{source}: {key}={value!r} rejected ({e})") return False + if key == "local_roles": + bad = [r for r in v if r not in LOCAL_ROLES] + if bad: + self.warnings.append(f"{source}: local_roles {bad!r} not in {LOCAL_ROLES}") + return False choices = self._CHOICES.get(key) if choices and v not in choices: self.warnings.append(f"{source}: {key}={v!r} not one of {choices}") @@ -280,6 +314,8 @@ def _set_threshold(self, name, value, source): # Environment variables, old names first so that a new name set alongside its old one wins. _ENV = {"TE_MODE": "mode", "TE_SHADOW": "shadow", "TE_PAID_MODEL": "paid_model", "TE_VOCABULARY": "vocabulary", + "TE_LOCAL_MODEL": "local_model", "TE_LOCAL_ROLES": "local_roles", + "TE_LOCAL_JUDGE_CONFIDENCE": "local_judge_confidence", "TE_INJECT": "microskills", "TE_INJECT_MAX_FACTS": "microskills_max_facts", "TE_INJECT_MAX_TOKENS": "microskills_max_tokens", "TE_PROPOSE_EVERY": "microskills_propose_every", diff --git a/tokeneconomizer/plugin/controller.py b/tokeneconomizer/plugin/controller.py index cfef0b2..72e3d22 100644 --- a/tokeneconomizer/plugin/controller.py +++ b/tokeneconomizer/plugin/controller.py @@ -332,8 +332,12 @@ def _judge(self, thr, envelope): from .judge import AsyncJudge, NoVerdict if envelope is None: return None - if self.host.name in sidecall.NO_SIDE_CALLS or sidecall.paused(self.host.name, - self.ctx.root): + if self.host.name in sidecall.NO_SIDE_CALLS: + return NoVerdict() + # A local judge (`local_model`) needs no login, so it still answers while the host's + # CLI is paused (signed out). + if (sidecall.paused(self.host.name, self.ctx.root) + and sidecall.local_runner("judge", self.ctx.config) is None): return NoVerdict() return AsyncJudge(self.ctx, self.sid, thr.get("judge_confidence")) diff --git a/tokeneconomizer/plugin/fold.py b/tokeneconomizer/plugin/fold.py index 4c16825..a8685d6 100644 --- a/tokeneconomizer/plugin/fold.py +++ b/tokeneconomizer/plugin/fold.py @@ -77,9 +77,11 @@ def _fold_microskills(ctx, adapter, run_dir, session_id, injected=None, imported # Paused once the host's CLI said it is not signed in (`sidecall.paused`): the free layer # still learns. A session read from the agent's history never calls a model: the free # layer learns from it, and the proposer waits for sessions the plugin sees end. - runner = (sidecall.runner_for(ctx.host.name, ctx.config) - if icfg.paid and not imported and not sidecall.paused(ctx.host.name, ctx.root) - else None) + runner = None + if icfg.paid and not imported: + runner = (sidecall.runner_for(ctx.host.name, ctx.config, role="proposer") + if not sidecall.paused(ctx.host.name, ctx.root) + else sidecall.local_runner("proposer", ctx.config)) stage = PrefixStage(icfg, ctx.repo.label, ctx.mech_dir, False, runner=runner, optimizer_model="agent", adapter=adapter, agent=ctx.host.name) srow = {"resolved": None} diff --git a/tokeneconomizer/plugin/judge.py b/tokeneconomizer/plugin/judge.py index d4bd411..24b841d 100644 --- a/tokeneconomizer/plugin/judge.py +++ b/tokeneconomizer/plugin/judge.py @@ -67,19 +67,35 @@ def run_job(ctx, session_id, job_id): req = ctx.store.read(REQUEST.format(sid=session_id)) or {} if req.get("id") != job_id: # superseded by a newer request return None - runner = (None if sidecall.paused(ctx.host.name, ctx.root) - else sidecall.runner_for(ctx.host.name, ctx.config)) + paused = sidecall.paused(ctx.host.name, ctx.root) + # Each call asks for its own runner: `local_roles` may put the judge on-device and leave + # the redirect on the agent's CLI, or the other way round. A paused host's CLI is skipped, + # but a local runner (which needs no login) still answers. + runner = sidecall.runner_for(ctx.host.name, ctx.config, role="judge") if not paused \ + else sidecall.local_runner("judge", ctx.config) + writer = sidecall.runner_for(ctx.host.name, ctx.config, role="nudge") if not paused \ + else sidecall.local_runner("nudge", ctx.config) verdict, written = None, None if runner is not None: try: verdict = loop_judge.llm_stuck_judge(req.get("digest") or "", runner, model="agent") except Exception: verdict = None + if verdict and getattr(runner, "answered_locally", False): + # The local model's confidences are coarse and high: a stuck verdict counts only at + # its own calibrated floor (`local_judge_confidence`); below it the turn is spared. + verdict["local"] = getattr(runner, "local", "local") + floor = float(getattr(ctx.config, "local_judge_confidence", 0.9) or 0) + if verdict.get("stuck") and float(verdict.get("confidence") or 0) < floor: + verdict["stuck"] = False + verdict["reason"] = ("below local_judge_confidence: " + + str(verdict.get("reason") or ""))[:160] conf = float((verdict or {}).get("confidence") or 0) if verdict and verdict.get("stuck") and conf >= float(req.get("threshold") or 0): try: - written = loop_judge.llm_nudge_writer(req.get("digest") or "", runner, - model="agent") + written = (loop_judge.llm_nudge_writer(req.get("digest") or "", writer, + model="agent") + if writer is not None else None) except Exception: written = None if written and (written.get("text") or "").strip(): diff --git a/tokeneconomizer/plugin/sidecall.py b/tokeneconomizer/plugin/sidecall.py index f30c114..bf32bcf 100644 --- a/tokeneconomizer/plugin/sidecall.py +++ b/tokeneconomizer/plugin/sidecall.py @@ -742,6 +742,91 @@ def copilot_call(prompt, model="", cli="copilot", timeout=TIMEOUT_S): # Hosts the plugin makes no model call for: their CLI cannot run a prompt with every tool off. NO_SIDE_CALLS = frozenset({"antigravity"}) + +# ---- an on-device model for some side calls (`local_model`, `local_roles`) ------------------- +# Apple's Foundation Models CLI, `/usr/bin/fm` on macOS 27 with Apple Intelligence on. It runs a +# ~3B model on the Neural Engine and GPU with a 4,096-token window, needs no login or key, and +# takes the prompt on stdin. `--schema` constrains its decoding to a JSON schema, which is what +# makes the judge usable: in free text it often wraps or contradicts its verdict, under a schema +# it returns the object every time. A call it cannot make returns None, and the caller falls back +# to the agent's own CLI, so turning this on never costs a verdict. + +FM_CLI = "fm" +FM_TIMEOUT_S = 60 +# The judge's verdict, as `loopguard.judge.llm_stuck_judge` parses it. +JUDGE_SCHEMA = { + "title": "Verdict", "type": "object", "additionalProperties": False, + "x-order": ["stuck", "confidence", "reason"], "required": ["stuck", "confidence", "reason"], + "properties": { + "stuck": {"type": "boolean"}, + "confidence": {"type": "number", "description": "0.0 to 1.0"}, + "reason": {"type": "string", "description": "short leakage-safe justification"}}} +ROLE_SCHEMAS = {"judge": JUDGE_SCHEMA} +_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") +_fm_ok = {} + + +def fm_available(cli=FM_CLI): + """Whether `fm` is on PATH and reports its on-device model available (once per process).""" + if cli not in _fm_ok: + ok = False + if shutil.which(cli): + try: + out = subprocess.run([cli, "available"], capture_output=True, encoding="utf-8", + errors="replace", timeout=30, env=_env(), + stdin=subprocess.DEVNULL) + ok = out.returncode == 0 and "unavailable" not in (out.stdout or "").lower() + except (OSError, subprocess.SubprocessError): + ok = False + _fm_ok[cli] = ok + return _fm_ok[cli] + + +def fm_call(prompt, schema=None, cli=FM_CLI, timeout=FM_TIMEOUT_S): + """(text, 0) from the on-device model, or (None, 0) when it gave no usable reply. Greedy + decoding, so the same digest gets the same verdict; `schema` (a JSON schema dict) constrains + the output. A prompt over the model's window fails here and falls back.""" + with tempfile.TemporaryDirectory(prefix="te-fm-", ignore_cleanup_errors=True) as tmp: + cmd = [cli, "respond", "--no-stream", "--greedy"] + if schema: + path = os.path.join(tmp, "schema.json") + with open(path, "w", encoding="utf-8") as fh: + json.dump(schema, fh) + cmd += ["--schema", path] + try: + out = subprocess.run(cmd, input=prompt, capture_output=True, encoding="utf-8", + errors="replace", timeout=timeout, env=_env(), cwd=tmp) + except (OSError, subprocess.SubprocessError) as e: + _log(f"sidecall fm: no reply ({type(e).__name__})\n") + return None, 0 + text = _ANSI_RE.sub("", out.stdout or "").strip() + if out.returncode != 0 or not text or text.startswith("Error:"): + err = _ANSI_RE.sub("", (out.stderr or out.stdout or "").strip())[:200] + _log(f"sidecall fm: no reply (exit {out.returncode}): {err}\n") + return None, 0 + return text, 0 + + +def local_runner(role, config, fallback=None): + """The side-call runner for `role` when `local_model` routes it on-device, else None. On a + failed local call the runner tries `fallback` (the agent's own runner), so a turned-on local + model never loses a verdict; with no fallback (a host in `NO_SIDE_CALLS`) it returns ''.""" + if getattr(config, "local_model", "off") != "fm" or role not in getattr(config, "local_roles", ()): + return None + if not fm_available(): + return None + schema = ROLE_SCHEMAS.get(role) + + def run(prompt, model="agent"): + text, eff = fm_call(prompt, schema=schema) + run.answered_locally = text is not None + if text is not None: + return text, eff + return fallback(prompt, model) if fallback is not None else ("", 0) + run.local = "fm" + run.answered_locally = False + return run + # The command each agent's side calls run, as the notices name it; the rest run ` -p`. _CALLS = {"codex": "codex exec", "opencode": "opencode run", "cursor": "cursor-agent -p", "hermes": "hermes chat"} @@ -752,13 +837,22 @@ def call_name(host_name): return _CALLS.get(host_name, f"{host_name} -p") -def runner_for(host_name, config): +def runner_for(host_name, config, role=None): """The optimizer callable for this host. Each host's calls go through that host's own CLI, never another agent's. The caller decides whether a call is allowed: micro-skills' proposer when micro-skills is on, the loop guard's judge when the guard is. None for a host in - `NO_SIDE_CALLS`. A call that raises is noted in errors.log (`_noted`).""" + `NO_SIDE_CALLS`. A call that raises is noted in errors.log (`_noted`). + + `role` (`judge`, `nudge`, `proposer`) lets `local_model` run that call on-device instead + (`local_runner`), falling back to the host's CLI when the local call fails; on a host in + `NO_SIDE_CALLS` the local model is the only runner.""" call = _host_call(host_name, config) - return None if call is None else _noted(host_name, call) + host = None if call is None else _noted(host_name, call) + if role is not None: + local = local_runner(role, config, fallback=host) + if local is not None: + return local + return host def _host_call(host_name, config): From 14fbe5ccd91fc1666f9dacf18cd0b4aa091e5029 Mon Sep 17 00:00:00 2001 From: Michael Avrukin Date: Sun, 8 Nov 2026 22:42:32 -0500 Subject: [PATCH 2/4] plugin: media=audio, on-device transcripts of the audio files a prompt names An agent cannot read audio: Claude Code's Read rejects an .m4a or .aiff as binary before any hook sees the call, so a prompt like "implement what the PM asks in notes/memo.m4a" sends the agent hunting for a transcriber. With media=audio, the prompt hook finds the audio paths the prompt names, transcribes each on the Mac with Apple's Speech framework (SpeechAnalyzer), and adds the transcripts to that prompt's context, as the repository map is added. No model call, nothing leaves the machine; transcripts are cached by content hash. - plugin/media.py: path detection (up to 3 files, 50 MB each), the cache, and the helper's build. - plugin/scripts/te-transcribe.swift: the transcriber, compiled once with swiftc into /bin by `te media setup`, or in the background by the first prompt that needs it (that prompt goes without). Without macOS 26+ and swiftc the mechanism does nothing and says so once. - controller.user_prompt adds the block on hosts with prompt context; shadow logs it without injecting. `te media [status|setup]`. - Setting media (off|audio), TE_MEDIA; documented in plugin/README.md. Off by default. tests/test_media.py. Co-Authored-By: Claude Opus 5.5 --- README.md | 5 +- plugin/README.md | 18 ++++ plugin/scripts/te-transcribe.swift | 38 +++++++ tests/test_media.py | 144 +++++++++++++++++++++++++++ tokeneconomizer/plugin/cli.py | 15 +++ tokeneconomizer/plugin/config.py | 10 +- tokeneconomizer/plugin/controller.py | 37 ++++++- tokeneconomizer/plugin/media.py | 138 +++++++++++++++++++++++++ 8 files changed, 399 insertions(+), 6 deletions(-) create mode 100644 plugin/scripts/te-transcribe.swift create mode 100644 tests/test_media.py create mode 100644 tokeneconomizer/plugin/media.py diff --git a/README.md b/README.md index bc73164..02e482e 100644 --- a/README.md +++ b/README.md @@ -337,8 +337,9 @@ turn stuck. Micro-skills is off by default, because its proposer adds model call saves depends on the repository and the work you do there. What each of these calls reads, how often it runs and which model it uses are listed under [Model calls](plugin/README.md#model-calls-and-api-keys). On macOS 27, `local_model=fm` runs the -judge on Apple's on-device model instead ([On-device side calls](plugin/README.md#on-device-side-calls-macos)); -it is off by default. Each mechanism has its own switch +judge on Apple's on-device model instead ([On-device side calls](plugin/README.md#on-device-side-calls-macos)), +and `media=audio` puts on-device transcripts of audio files a prompt names into its context +([Audio transcripts](plugin/README.md#audio-transcripts-macos)); both are off by default. Each mechanism has its own switch (`microskills`, `toolprune`, `loopguard`) that takes `on` or `off`, and the italic letters above are these settings: diff --git a/plugin/README.md b/plugin/README.md index 1cb0d8b..dc07226 100644 --- a/plugin/README.md +++ b/plugin/README.md @@ -799,6 +799,7 @@ first session there ends. | `local_model` | `off` | `fm`: run the side calls `local_roles` names on Apple's on-device Foundation Models CLI (`/usr/bin/fm`, macOS 27 with Apple Intelligence) instead of the agent's CLI; see [On-device side calls](#on-device-side-calls-macos) | | `local_roles` | `judge` | which side calls `local_model` takes: `judge`, `nudge` (the redirect writer), `proposer`, separated by commas | | `local_judge_confidence` | `0.9` | how sure a stuck verdict from the on-device judge must be to count; below it the turn is spared | +| `media` | `off` | `audio`: add on-device transcripts of the audio files a prompt names to that prompt's context; see [Audio transcripts](#audio-transcripts-macos) | | `pi_extensions` | `""` | Pi only: extension files (comma-separated paths) the plugin's own calls load, for a model whose provider an extension registers | | `pi_cli` | `""` | Pi only: the `pi` the plugin's own calls run; empty: the first on PATH | | `shadow` | `false` | decide and log everything, act on nothing | @@ -927,6 +928,23 @@ steps, most of them above the guard's own `judge_confidence` (0.7). ~/.tokeneconomizer/bin/te config set local_roles=judge,nudge # also the redirect writer ``` +## Audio transcripts (macOS) + +An agent cannot read audio: Claude Code's Read refuses an `.m4a` or `.aiff` as binary before any +hook sees the call. With `media=audio`, when a prompt names an audio file that exists (relative to +the session's directory; `aiff`, `aif`, `wav`, `m4a`, `mp3`, `caf`, `aac`, `flac`; up to three, +50 MB each; paths without spaces), the prompt hook transcribes it on the Mac with Apple's Speech +framework and adds the transcript to that prompt's context, as the repository map is added. No +model call is made and nothing leaves the machine. Transcripts are cached by the file's content. + +The transcriber is a small Swift program (`plugin/scripts/te-transcribe.swift`) that is compiled +once into `~/.tokeneconomizer/bin/`: run `te media setup`, or the first prompt that names an audio +file starts the build in the background and goes without a transcript. It needs macOS 26 or later +and `swiftc` (Xcode or the Command Line Tools); without them `media=audio` does nothing and says +so once. `te media` reports whether it is built. Only hosts whose prompt hook can add context get +transcripts (Claude Code, the Copilot CLI, Pi, opencode, Hermes Agent); it was verified on Claude +Code only. + ## Troubleshooting | Symptom | Fix | diff --git a/plugin/scripts/te-transcribe.swift b/plugin/scripts/te-transcribe.swift new file mode 100644 index 0000000..9ebd37d --- /dev/null +++ b/plugin/scripts/te-transcribe.swift @@ -0,0 +1,38 @@ +// te-transcribe: on-device speech-to-text for TokenEconomizer's `media=audio` (macOS 26+). +// usage: te-transcribe [locale] -> the transcript on stdout, exit 0 +// Apple's Speech framework (SpeechAnalyzer + SpeechTranscriber): no network, no account. Built +// once by `te media setup` (or the first prompt that names an audio file) with `swiftc`. +import AVFoundation +import Foundation +import Speech + +@main struct TETranscribe { + static func main() async { + let args = CommandLine.arguments + guard args.count > 1 else { + FileHandle.standardError.write("usage: te-transcribe [locale]\n".data(using: .utf8)!) + exit(2) + } + let url = URL(fileURLWithPath: args[1]) + let locale = Locale(identifier: args.count > 2 ? args[2] : Locale.current.identifier) + do { + let transcriber = SpeechTranscriber(locale: locale, preset: .transcription) + if let req = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { + try await req.downloadAndInstall() + } + let collect = Task { () -> String in + var out = "" + for try await r in transcriber.results { out += String(r.text.characters) } + return out + } + let file = try AVAudioFile(forReading: url) + let analyzer = try await SpeechAnalyzer(inputAudioFile: file, modules: [transcriber], + finishAfterFile: true) + _ = analyzer + print(try await collect.value.trimmingCharacters(in: .whitespacesAndNewlines)) + } catch { + FileHandle.standardError.write("te-transcribe: \(error)\n".data(using: .utf8)!) + exit(1) + } + } +} diff --git a/tests/test_media.py b/tests/test_media.py new file mode 100644 index 0000000..dc383a0 --- /dev/null +++ b/tests/test_media.py @@ -0,0 +1,144 @@ +"""`media=audio`: transcripts of the audio files a prompt names, from an on-device helper.""" +import os +import stat +import subprocess + +import pytest + +from tokeneconomizer.plugin import config as live_config +from tokeneconomizer.plugin import media + + +@pytest.fixture +def repo(tmp_path): + (tmp_path / "notes").mkdir() + (tmp_path / "notes" / "memo.aiff").write_bytes(b"FORM....AIFF") + (tmp_path / "talk.m4a").write_bytes(b"....ftypM4A ") + return tmp_path + + +@pytest.mark.parametrize("prompt, want", [ + ("Implement what notes/memo.aiff asks.", ["notes/memo.aiff"]), # a sentence's period + ("see `talk.m4a`, then notes/memo.aiff", ["talk.m4a", "notes/memo.aiff"]), + ("notes/memo.aiff and notes/memo.aiff again", ["notes/memo.aiff"]), + ("https://example.com/talk.m4a", []), # a URL is not a file + ("notes/missing.aiff", []), # nor a missing one + ("notes/memo.aiff.bak", []), +]) +def test_audio_paths(repo, prompt, want): + assert media.audio_paths(prompt, str(repo)) == [str(repo / w) for w in want] + + +def _fake_helper(tmp_path, monkeypatch, body): + home = tmp_path / "home" + monkeypatch.setenv("TOKENECONOMIZER_HOME", str(home)) + h = home / "bin" / "te-transcribe" + h.parent.mkdir(parents=True) + h.write_text("#!/bin/sh\n" + body) + h.chmod(h.stat().st_mode | stat.S_IEXEC) + return home + + +def test_for_prompt_transcribes_and_caches(repo, tmp_path, monkeypatch): + home = _fake_helper(tmp_path, monkeypatch, 'echo "round half up"; echo x >> "$0.calls"\n') + block, info = media.for_prompt("do what notes/memo.aiff says", str(repo)) + assert "On-device transcript of notes/memo.aiff" in block and "round half up" in block + assert info == {"files": 1, "transcribed": 1, "needs_helper": False} + media.for_prompt("again: notes/memo.aiff", str(repo)) + assert (home / "bin" / "te-transcribe.calls").read_text().count("x") == 1 # cached + + +def test_for_prompt_without_the_helper_says_it_needs_one(repo, tmp_path, monkeypatch): + monkeypatch.setenv("TOKENECONOMIZER_HOME", str(tmp_path / "empty")) + block, info = media.for_prompt("notes/memo.aiff", str(repo)) + assert block == "" and info["needs_helper"] is True + + +def test_a_failing_helper_adds_nothing(repo, tmp_path, monkeypatch): + _fake_helper(tmp_path, monkeypatch, "echo boom >&2; exit 1\n") + assert media.for_prompt("notes/memo.aiff", str(repo))[0] == "" + + +def test_build_helper_needs_swiftc(monkeypatch): + monkeypatch.setattr(media, "supported", lambda: False) + ok, msg = media.build_helper() + assert not ok and "swiftc" in msg + + +def test_build_helper_compiles_the_bundled_source(tmp_path, monkeypatch): + monkeypatch.setenv("TOKENECONOMIZER_HOME", str(tmp_path)) + monkeypatch.setattr(media, "supported", lambda: True) + seen = {} + + def fake_run(cmd, **k): + seen["cmd"] = cmd + open(cmd[cmd.index("-o") + 1], "w").close() + return subprocess.CompletedProcess(cmd, 0, "", "") + + monkeypatch.setattr(media.subprocess, "run", fake_run) + ok, out = media.build_helper() + assert ok and out == media.helper_path() and os.path.isfile(out) + assert seen["cmd"][0] == "swiftc" and seen["cmd"][3].endswith("te-transcribe.swift") + + +def test_media_setting(): + c = live_config.LiveConfig() + assert c.media == "off" and c.set("media", "audio") and not c.set("media", "video") + + +# ---- through the hooks --------------------------------------------------------------------- +from tests.test_live_hooks import call, env, repo as hook_repo # noqa: E402,F401 + + +def _ctx_text(out): + return (out.get("hookSpecificOutput") or {}).get("additionalContext") or out.get( + "additionalContext") or "" + + +def _prompt(e, cwd, sid, prompt): + base = {"session_id": sid, "transcript_path": "", "cwd": cwd, "prompt_id": "p"} + call(e, dict(base, hook_event_name="SessionStart", source="startup")) + return call(e, dict(base, hook_event_name="UserPromptSubmit", prompt=prompt)) + + +@pytest.fixture +def memo(hook_repo): + os.makedirs(os.path.join(hook_repo, "notes"), exist_ok=True) + with open(os.path.join(hook_repo, "notes", "memo.aiff"), "wb") as fh: + fh.write(b"FORM....AIFF") + return hook_repo + + +def _helper(e): + if os.name == "nt": + pytest.skip("the stand-in helper is an sh script") + h = os.path.join(e["TOKENECONOMIZER_HOME"], "bin", "te-transcribe") + os.makedirs(os.path.dirname(h), exist_ok=True) + with open(h, "w") as fh: + fh.write("#!/bin/sh\necho 'please round half up'\n") + os.chmod(h, 0o755) + + +def test_a_prompt_naming_audio_gets_its_transcript_in_context(env, memo): + e = dict(env, TE_MEDIA="audio") + _helper(e) + out = _prompt(e, memo, "m1", "Implement what notes/memo.aiff asks.") + text = _ctx_text(out) + assert "On-device transcript of notes/memo.aiff" in text and "round half up" in text + + +def test_media_off_or_shadow_adds_nothing(env, memo): + _helper(env) + assert _ctx_text(_prompt(env, memo, "m2", "Implement notes/memo.aiff")) == "" + e = dict(env, TE_MEDIA="audio", TE_SHADOW="1") + assert _ctx_text(_prompt(e, memo, "m3", "Implement notes/memo.aiff")) == "" + + +def test_without_the_helper_the_first_prompt_starts_a_build_and_says_so(env, memo, monkeypatch): + from tokeneconomizer.plugin import media as m + monkeypatch.setattr(m, "supported", lambda: True) + spawned = [] + from tokeneconomizer.plugin import context + monkeypatch.setattr(context.LiveContext, "spawn", lambda self, argv, log: spawned.append(argv)) + out = _prompt(dict(env, TE_MEDIA="audio"), memo, "m4", "Implement notes/memo.aiff") + assert spawned == [["media", "setup"]] and "audio transcriber" in out.get("systemMessage", "") diff --git a/tokeneconomizer/plugin/cli.py b/tokeneconomizer/plugin/cli.py index 77a9b67..696bdca 100644 --- a/tokeneconomizer/plugin/cli.py +++ b/tokeneconomizer/plugin/cli.py @@ -698,6 +698,18 @@ def cmd_history(args): return 0 +def cmd_media(args): + """`media=audio`'s on-device transcriber: build it (`setup`) or say whether it is ready.""" + from . import media + if args.action == "setup": + ok, msg = media.build_helper() + print(("built " if ok else "not built: ") + msg) + return 0 if ok else 1 + print(f"media helper: {'ready at ' + media.helper_path() if media.helper_ready() else 'not built'}" + f"{'' if media.supported() else ' (needs macOS with swiftc)'}") + return 0 + + def cmd_judge(args): """The detached half of the stuck judge (see plugin/judge.py); not for people.""" from . import judge @@ -953,6 +965,9 @@ def add(name, *aliases): p = add("repomap") p.add_argument("action", nargs="?", default="show", choices=("show", "refresh")) p.set_defaults(fn=cmd_repomap) + p = add("media") + p.add_argument("action", nargs="?", default="status", choices=("status", "setup")) + p.set_defaults(fn=cmd_media) p = add("probe") p.add_argument("--sessions", type=int, default=6) p.add_argument("--model", default=None) diff --git a/tokeneconomizer/plugin/config.py b/tokeneconomizer/plugin/config.py index c139323..d59f104 100644 --- a/tokeneconomizer/plugin/config.py +++ b/tokeneconomizer/plugin/config.py @@ -46,6 +46,10 @@ * loopguard. -- one guard threshold, pinned over the preset and over what the repository's history learned (LOOPGUARD_KEYS); * repomap, repomap_tokens, repomap_python, repomap_timeout -- the opt-in repository map; + * media -- `audio`: add on-device transcripts of the audio files a prompt + names (an agent cannot read audio) to that prompt's context, + on hosts that take prompt context; macOS with swiftc. Off by + default (`plugin/media.py`); * local_model, local_roles -- run the named side calls (`judge`, `nudge`, `proposer`; default `judge`) on an on-device model instead of the agent's CLI: `fm`, Apple's Foundation Models CLI on macOS 27. @@ -100,6 +104,7 @@ TOOLPRUNE = ("off", "propose", "auto") LOOPGUARD = ("off", "nudge", "stop") REPOMAP = ("off", "on", "start") +MEDIA = ("off", "audio") MODES = ("auto", "interactive", "headless") AGGR = ("conservative", "default", "aggressive") @@ -193,6 +198,7 @@ class LiveConfig: repomap_python: str = "" repomap_timeout: int = 8 paid_model: str = "" + media: str = "off" local_model: str = "off" local_roles: list = field(default_factory=lambda: ["judge"]) local_judge_confidence: float = 0.9 @@ -209,7 +215,7 @@ class LiveConfig: _CHOICES = {"mode": MODES, "microskills": MICROSKILLS, "toolprune": TOOLPRUNE, "loopguard": LOOPGUARD, "repomap": REPOMAP, "loopguard_aggr": AGGR, - "local_model": LOCAL_MODELS, + "local_model": LOCAL_MODELS, "media": MEDIA, "vocabulary": ("auto", "generic", "python-git", "node-web")} # The smallest value a number setting takes, where it is not 1. _MIN = {"toolprune_quiet_minutes": 0} @@ -314,7 +320,7 @@ def _set_threshold(self, name, value, source): # Environment variables, old names first so that a new name set alongside its old one wins. _ENV = {"TE_MODE": "mode", "TE_SHADOW": "shadow", "TE_PAID_MODEL": "paid_model", "TE_VOCABULARY": "vocabulary", - "TE_LOCAL_MODEL": "local_model", "TE_LOCAL_ROLES": "local_roles", + "TE_LOCAL_MODEL": "local_model", "TE_LOCAL_ROLES": "local_roles", "TE_MEDIA": "media", "TE_LOCAL_JUDGE_CONFIDENCE": "local_judge_confidence", "TE_INJECT": "microskills", "TE_INJECT_MAX_FACTS": "microskills_max_facts", "TE_INJECT_MAX_TOKENS": "microskills_max_tokens", diff --git a/tokeneconomizer/plugin/controller.py b/tokeneconomizer/plugin/controller.py index 72e3d22..aa45de7 100644 --- a/tokeneconomizer/plugin/controller.py +++ b/tokeneconomizer/plugin/controller.py @@ -268,6 +268,36 @@ def _inject_map(self, s, seen, block, event, how): seen["repomap"] = True return out + def _media(self, prompt): + """`media=audio`: the transcripts of the audio files this prompt names, as context. The + first prompt that needs the helper starts building it in the background and goes + without; a missing toolchain is said once.""" + from . import media + try: + block, info = media.for_prompt(prompt, self.p.get("cwd") or self.ctx.repo.root, + env=self.ctx.env) + except Exception: + return [] + if not info["files"]: + return [] + self._log("media", **info, chars=len(block)) + out = [] + seen = self.store.read("notices.json") or {} + if info["needs_helper"] and not seen.get("media_building"): + seen["media_building"] = True + if media.supported(): + self.ctx.spawn(["media", "setup"], "media.log") + out.append(self.host.notice( + "TokenEconomizer: building the on-device audio transcriber (once); audio " + "named from the next prompt on is transcribed into context.")) + else: + out.append(self.host.notice( + "TokenEconomizer: media=audio needs macOS with swiftc; doing nothing.")) + self.store.write("notices.json", seen) + if block and not self.cfg.shadow: + out.append(self.host.context("UserPromptSubmit", block)) + return out + # ---- prompt-turns ------------------------------------------------------------------- def _close_turn(self, s, stopped_normally): data = s.get("turn") @@ -287,11 +317,14 @@ def _close_turn(self, s, stopped_normally): return summary def user_prompt(self): - if self.cfg.loopguard == "off" and not (self.cfg.repomap == "on" - and self.host.prompt_context): + media_on = self.cfg.media == "audio" and self.host.prompt_context + if self.cfg.loopguard == "off" and not media_on and not (self.cfg.repomap == "on" + and self.host.prompt_context): return {} s = self._session() outputs = [] + if media_on: + outputs += self._media(self.p.get("prompt") or "") if self.cfg.repomap == "on" and self.host.prompt_context and not s.get("repomap_done"): # Once per session, on its first prompt: a map ranked toward what the prompt names. s["repomap_done"] = True diff --git a/tokeneconomizer/plugin/media.py b/tokeneconomizer/plugin/media.py new file mode 100644 index 0000000..a13cf76 --- /dev/null +++ b/tokeneconomizer/plugin/media.py @@ -0,0 +1,138 @@ +"""`media=audio`: transcripts of the audio files a prompt names, transcribed on the machine. + +An agent cannot read audio: Claude Code's Read rejects an `.m4a` or `.aiff` as binary before any +hook sees the call, so a prompt such as "implement what the PM asks in notes/memo.m4a" sends the +agent hunting for a transcriber -- turns, tokens and usually a failed task. With `media=audio` +the prompt's hook finds the audio paths it names, transcribes each with Apple's Speech framework +(`plugin/scripts/te-transcribe.swift`, macOS 26+; no network, no account, no model call billed), +and adds the transcripts to the prompt's context, as the repository map is added. + +The helper is a Swift program built once, with `swiftc`, into `/bin/`: by +`te media setup`, or in the background by the first prompt that names an audio file (that prompt +goes without a transcript). Without `swiftc` or on another OS the mechanism does nothing. +Transcripts are cached by the file's content hash, so naming the same memo again costs nothing. +""" +import hashlib +import os +import re +import shutil +import subprocess +import sys + +from . import paths + +AUDIO_EXT = ("aiff", "aif", "wav", "m4a", "mp3", "caf", "aac", "flac") +# A path the prompt names: a word of path characters ending in an audio extension, which may be +# quoted or backticked; not part of a URL. +AUDIO_RE = re.compile(r"(?= MAX_FILES: + break + return out + + +def source_path(): + """The helper's Swift source in the plugin directory (the checkout or the bundle).""" + plugin = os.environ.get("TE_PLUGIN_ROOT") + candidates = [os.path.join(plugin, "scripts", "te-transcribe.swift")] if plugin else [] + core = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + candidates += [os.path.join(core, "plugin", "scripts", "te-transcribe.swift"), + os.path.join(os.path.dirname(core), "scripts", "te-transcribe.swift")] + return next((c for c in candidates if os.path.isfile(c)), None) + + +def helper_path(env=None): + return os.path.join(paths.state_root(env), "bin", "te-transcribe") + + +def helper_ready(env=None): + h = helper_path(env) + return os.path.isfile(h) and os.access(h, os.X_OK) + + +def supported(): + return sys.platform == "darwin" and shutil.which("swiftc") is not None + + +def build_helper(env=None, timeout=300): + """Compile the helper; (ok, message).""" + if not supported(): + return False, "needs macOS with swiftc (Xcode or the Command Line Tools)" + src = source_path() + if not src: + return False, "te-transcribe.swift not found in the plugin" + out = helper_path(env) + os.makedirs(os.path.dirname(out), exist_ok=True) + tmp = "%s.%d.tmp" % (out, os.getpid()) + try: + r = subprocess.run(["swiftc", "-O", "-parse-as-library", src, "-o", tmp], + capture_output=True, encoding="utf-8", errors="replace", + timeout=timeout, stdin=subprocess.DEVNULL) + except (OSError, subprocess.SubprocessError) as e: + return False, f"swiftc failed to run: {type(e).__name__}" + if r.returncode != 0 or not os.path.isfile(tmp): + return False, "swiftc: " + (r.stderr or r.stdout or "").strip()[-300:] + os.replace(tmp, out) + return True, out + + +def transcript(path, env=None): + """The transcript of `path` ("" when the helper is missing or fails), cached by content.""" + try: + with open(path, "rb") as fh: + key = hashlib.sha256(fh.read()).hexdigest()[:24] + except OSError: + return "" + cache = os.path.join(paths.state_root(env), "media", key + ".txt") + try: + with open(cache, encoding="utf-8") as fh: + return fh.read() + except OSError: + pass + if not helper_ready(env): + return "" + try: + r = subprocess.run([helper_path(env), path], capture_output=True, encoding="utf-8", + errors="replace", timeout=TRANSCRIBE_TIMEOUT_S, + stdin=subprocess.DEVNULL) + except (OSError, subprocess.SubprocessError): + return "" + text = (r.stdout or "").strip()[:MAX_CHARS] + if r.returncode != 0 or not text: + return "" + os.makedirs(os.path.dirname(cache), exist_ok=True) + with open(cache, "w", encoding="utf-8") as fh: + fh.write(text) + return text + + +def for_prompt(prompt, cwd, env=None): + """(block, info) for the prompt's context: the transcripts of the audio it names. `info` + says what happened, for the plugin's log: files found, transcribed, and whether the helper + still has to be built.""" + files = audio_paths(prompt, cwd) + info = {"files": len(files), "transcribed": 0, "needs_helper": bool(files) and not + helper_ready(env)} + parts = [] + for p in files: + text = transcript(p, env) + if text: + info["transcribed"] += 1 + rel = os.path.relpath(p, cwd) if p.startswith(os.path.abspath(cwd)) else p + parts.append(f"[On-device transcript of {rel} (Apple Speech framework; it may " + f"contain recognition errors)]\n{text}") + return "\n\n".join(parts), info From 23052c2c675fca9be3ce71974d62aa97ba3613e2 Mon Sep 17 00:00:00 2001 From: Michael Avrukin Date: Tue, 10 Nov 2026 17:59:47 -0500 Subject: [PATCH 3/4] plugin: drop media=audio Per review on #185: transcribing files at prompt time is outside what the plugin does (it reads finished sessions to change what later sessions cost), and it would be the first compiled helper, which CI cannot build or test. It may come back as a separate plugin. Removes plugin/media.py, plugin/scripts/te-transcribe.swift, `te media`, the `media` setting and its docs, and tests/test_media.py. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UporVpn64UvJqEnurKVwza --- README.md | 5 +- plugin/README.md | 18 ---- plugin/scripts/te-transcribe.swift | 38 ------- tests/test_media.py | 144 --------------------------- tokeneconomizer/plugin/cli.py | 15 --- tokeneconomizer/plugin/config.py | 10 +- tokeneconomizer/plugin/controller.py | 37 +------ tokeneconomizer/plugin/media.py | 138 ------------------------- 8 files changed, 6 insertions(+), 399 deletions(-) delete mode 100644 plugin/scripts/te-transcribe.swift delete mode 100644 tests/test_media.py delete mode 100644 tokeneconomizer/plugin/media.py diff --git a/README.md b/README.md index 02e482e..bc73164 100644 --- a/README.md +++ b/README.md @@ -337,9 +337,8 @@ turn stuck. Micro-skills is off by default, because its proposer adds model call saves depends on the repository and the work you do there. What each of these calls reads, how often it runs and which model it uses are listed under [Model calls](plugin/README.md#model-calls-and-api-keys). On macOS 27, `local_model=fm` runs the -judge on Apple's on-device model instead ([On-device side calls](plugin/README.md#on-device-side-calls-macos)), -and `media=audio` puts on-device transcripts of audio files a prompt names into its context -([Audio transcripts](plugin/README.md#audio-transcripts-macos)); both are off by default. Each mechanism has its own switch +judge on Apple's on-device model instead ([On-device side calls](plugin/README.md#on-device-side-calls-macos)); +it is off by default. Each mechanism has its own switch (`microskills`, `toolprune`, `loopguard`) that takes `on` or `off`, and the italic letters above are these settings: diff --git a/plugin/README.md b/plugin/README.md index f326802..3bb26e1 100644 --- a/plugin/README.md +++ b/plugin/README.md @@ -801,7 +801,6 @@ first session there ends. | `local_model` | `off` | `fm`: run the side calls `local_roles` names on Apple's on-device Foundation Models CLI (`/usr/bin/fm`, macOS 27 with Apple Intelligence) instead of the agent's CLI; see [On-device side calls](#on-device-side-calls-macos) | | `local_roles` | `judge` | which side calls `local_model` takes: `judge`, `nudge` (the redirect writer), `proposer`, separated by commas | | `local_judge_confidence` | `0.9` | how sure a stuck verdict from the on-device judge must be to count; below it the turn is spared | -| `media` | `off` | `audio`: add on-device transcripts of the audio files a prompt names to that prompt's context; see [Audio transcripts](#audio-transcripts-macos) | | `pi_extensions` | `""` | Pi only: extension files (comma-separated paths) the plugin's own calls load, for a model whose provider an extension registers | | `pi_cli` | `""` | Pi only: the `pi` the plugin's own calls run; empty: the first on PATH | | `shadow` | `false` | decide and log everything, act on nothing | @@ -930,23 +929,6 @@ steps, most of them above the guard's own `judge_confidence` (0.7). ~/.tokeneconomizer/bin/te config set local_roles=judge,nudge # also the redirect writer ``` -## Audio transcripts (macOS) - -An agent cannot read audio: Claude Code's Read refuses an `.m4a` or `.aiff` as binary before any -hook sees the call. With `media=audio`, when a prompt names an audio file that exists (relative to -the session's directory; `aiff`, `aif`, `wav`, `m4a`, `mp3`, `caf`, `aac`, `flac`; up to three, -50 MB each; paths without spaces), the prompt hook transcribes it on the Mac with Apple's Speech -framework and adds the transcript to that prompt's context, as the repository map is added. No -model call is made and nothing leaves the machine. Transcripts are cached by the file's content. - -The transcriber is a small Swift program (`plugin/scripts/te-transcribe.swift`) that is compiled -once into `~/.tokeneconomizer/bin/`: run `te media setup`, or the first prompt that names an audio -file starts the build in the background and goes without a transcript. It needs macOS 26 or later -and `swiftc` (Xcode or the Command Line Tools); without them `media=audio` does nothing and says -so once. `te media` reports whether it is built. Only hosts whose prompt hook can add context get -transcripts (Claude Code, the Copilot CLI, Pi, opencode, Hermes Agent); it was verified on Claude -Code only. - ## Troubleshooting | Symptom | Fix | diff --git a/plugin/scripts/te-transcribe.swift b/plugin/scripts/te-transcribe.swift deleted file mode 100644 index 9ebd37d..0000000 --- a/plugin/scripts/te-transcribe.swift +++ /dev/null @@ -1,38 +0,0 @@ -// te-transcribe: on-device speech-to-text for TokenEconomizer's `media=audio` (macOS 26+). -// usage: te-transcribe [locale] -> the transcript on stdout, exit 0 -// Apple's Speech framework (SpeechAnalyzer + SpeechTranscriber): no network, no account. Built -// once by `te media setup` (or the first prompt that names an audio file) with `swiftc`. -import AVFoundation -import Foundation -import Speech - -@main struct TETranscribe { - static func main() async { - let args = CommandLine.arguments - guard args.count > 1 else { - FileHandle.standardError.write("usage: te-transcribe [locale]\n".data(using: .utf8)!) - exit(2) - } - let url = URL(fileURLWithPath: args[1]) - let locale = Locale(identifier: args.count > 2 ? args[2] : Locale.current.identifier) - do { - let transcriber = SpeechTranscriber(locale: locale, preset: .transcription) - if let req = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { - try await req.downloadAndInstall() - } - let collect = Task { () -> String in - var out = "" - for try await r in transcriber.results { out += String(r.text.characters) } - return out - } - let file = try AVAudioFile(forReading: url) - let analyzer = try await SpeechAnalyzer(inputAudioFile: file, modules: [transcriber], - finishAfterFile: true) - _ = analyzer - print(try await collect.value.trimmingCharacters(in: .whitespacesAndNewlines)) - } catch { - FileHandle.standardError.write("te-transcribe: \(error)\n".data(using: .utf8)!) - exit(1) - } - } -} diff --git a/tests/test_media.py b/tests/test_media.py deleted file mode 100644 index dc383a0..0000000 --- a/tests/test_media.py +++ /dev/null @@ -1,144 +0,0 @@ -"""`media=audio`: transcripts of the audio files a prompt names, from an on-device helper.""" -import os -import stat -import subprocess - -import pytest - -from tokeneconomizer.plugin import config as live_config -from tokeneconomizer.plugin import media - - -@pytest.fixture -def repo(tmp_path): - (tmp_path / "notes").mkdir() - (tmp_path / "notes" / "memo.aiff").write_bytes(b"FORM....AIFF") - (tmp_path / "talk.m4a").write_bytes(b"....ftypM4A ") - return tmp_path - - -@pytest.mark.parametrize("prompt, want", [ - ("Implement what notes/memo.aiff asks.", ["notes/memo.aiff"]), # a sentence's period - ("see `talk.m4a`, then notes/memo.aiff", ["talk.m4a", "notes/memo.aiff"]), - ("notes/memo.aiff and notes/memo.aiff again", ["notes/memo.aiff"]), - ("https://example.com/talk.m4a", []), # a URL is not a file - ("notes/missing.aiff", []), # nor a missing one - ("notes/memo.aiff.bak", []), -]) -def test_audio_paths(repo, prompt, want): - assert media.audio_paths(prompt, str(repo)) == [str(repo / w) for w in want] - - -def _fake_helper(tmp_path, monkeypatch, body): - home = tmp_path / "home" - monkeypatch.setenv("TOKENECONOMIZER_HOME", str(home)) - h = home / "bin" / "te-transcribe" - h.parent.mkdir(parents=True) - h.write_text("#!/bin/sh\n" + body) - h.chmod(h.stat().st_mode | stat.S_IEXEC) - return home - - -def test_for_prompt_transcribes_and_caches(repo, tmp_path, monkeypatch): - home = _fake_helper(tmp_path, monkeypatch, 'echo "round half up"; echo x >> "$0.calls"\n') - block, info = media.for_prompt("do what notes/memo.aiff says", str(repo)) - assert "On-device transcript of notes/memo.aiff" in block and "round half up" in block - assert info == {"files": 1, "transcribed": 1, "needs_helper": False} - media.for_prompt("again: notes/memo.aiff", str(repo)) - assert (home / "bin" / "te-transcribe.calls").read_text().count("x") == 1 # cached - - -def test_for_prompt_without_the_helper_says_it_needs_one(repo, tmp_path, monkeypatch): - monkeypatch.setenv("TOKENECONOMIZER_HOME", str(tmp_path / "empty")) - block, info = media.for_prompt("notes/memo.aiff", str(repo)) - assert block == "" and info["needs_helper"] is True - - -def test_a_failing_helper_adds_nothing(repo, tmp_path, monkeypatch): - _fake_helper(tmp_path, monkeypatch, "echo boom >&2; exit 1\n") - assert media.for_prompt("notes/memo.aiff", str(repo))[0] == "" - - -def test_build_helper_needs_swiftc(monkeypatch): - monkeypatch.setattr(media, "supported", lambda: False) - ok, msg = media.build_helper() - assert not ok and "swiftc" in msg - - -def test_build_helper_compiles_the_bundled_source(tmp_path, monkeypatch): - monkeypatch.setenv("TOKENECONOMIZER_HOME", str(tmp_path)) - monkeypatch.setattr(media, "supported", lambda: True) - seen = {} - - def fake_run(cmd, **k): - seen["cmd"] = cmd - open(cmd[cmd.index("-o") + 1], "w").close() - return subprocess.CompletedProcess(cmd, 0, "", "") - - monkeypatch.setattr(media.subprocess, "run", fake_run) - ok, out = media.build_helper() - assert ok and out == media.helper_path() and os.path.isfile(out) - assert seen["cmd"][0] == "swiftc" and seen["cmd"][3].endswith("te-transcribe.swift") - - -def test_media_setting(): - c = live_config.LiveConfig() - assert c.media == "off" and c.set("media", "audio") and not c.set("media", "video") - - -# ---- through the hooks --------------------------------------------------------------------- -from tests.test_live_hooks import call, env, repo as hook_repo # noqa: E402,F401 - - -def _ctx_text(out): - return (out.get("hookSpecificOutput") or {}).get("additionalContext") or out.get( - "additionalContext") or "" - - -def _prompt(e, cwd, sid, prompt): - base = {"session_id": sid, "transcript_path": "", "cwd": cwd, "prompt_id": "p"} - call(e, dict(base, hook_event_name="SessionStart", source="startup")) - return call(e, dict(base, hook_event_name="UserPromptSubmit", prompt=prompt)) - - -@pytest.fixture -def memo(hook_repo): - os.makedirs(os.path.join(hook_repo, "notes"), exist_ok=True) - with open(os.path.join(hook_repo, "notes", "memo.aiff"), "wb") as fh: - fh.write(b"FORM....AIFF") - return hook_repo - - -def _helper(e): - if os.name == "nt": - pytest.skip("the stand-in helper is an sh script") - h = os.path.join(e["TOKENECONOMIZER_HOME"], "bin", "te-transcribe") - os.makedirs(os.path.dirname(h), exist_ok=True) - with open(h, "w") as fh: - fh.write("#!/bin/sh\necho 'please round half up'\n") - os.chmod(h, 0o755) - - -def test_a_prompt_naming_audio_gets_its_transcript_in_context(env, memo): - e = dict(env, TE_MEDIA="audio") - _helper(e) - out = _prompt(e, memo, "m1", "Implement what notes/memo.aiff asks.") - text = _ctx_text(out) - assert "On-device transcript of notes/memo.aiff" in text and "round half up" in text - - -def test_media_off_or_shadow_adds_nothing(env, memo): - _helper(env) - assert _ctx_text(_prompt(env, memo, "m2", "Implement notes/memo.aiff")) == "" - e = dict(env, TE_MEDIA="audio", TE_SHADOW="1") - assert _ctx_text(_prompt(e, memo, "m3", "Implement notes/memo.aiff")) == "" - - -def test_without_the_helper_the_first_prompt_starts_a_build_and_says_so(env, memo, monkeypatch): - from tokeneconomizer.plugin import media as m - monkeypatch.setattr(m, "supported", lambda: True) - spawned = [] - from tokeneconomizer.plugin import context - monkeypatch.setattr(context.LiveContext, "spawn", lambda self, argv, log: spawned.append(argv)) - out = _prompt(dict(env, TE_MEDIA="audio"), memo, "m4", "Implement notes/memo.aiff") - assert spawned == [["media", "setup"]] and "audio transcriber" in out.get("systemMessage", "") diff --git a/tokeneconomizer/plugin/cli.py b/tokeneconomizer/plugin/cli.py index 6a8b450..5838624 100644 --- a/tokeneconomizer/plugin/cli.py +++ b/tokeneconomizer/plugin/cli.py @@ -698,18 +698,6 @@ def cmd_history(args): return 0 -def cmd_media(args): - """`media=audio`'s on-device transcriber: build it (`setup`) or say whether it is ready.""" - from . import media - if args.action == "setup": - ok, msg = media.build_helper() - print(("built " if ok else "not built: ") + msg) - return 0 if ok else 1 - print(f"media helper: {'ready at ' + media.helper_path() if media.helper_ready() else 'not built'}" - f"{'' if media.supported() else ' (needs macOS with swiftc)'}") - return 0 - - def cmd_judge(args): """The detached half of the stuck judge (see plugin/judge.py); not for people.""" from . import judge @@ -965,9 +953,6 @@ def add(name, *aliases): p = add("repomap") p.add_argument("action", nargs="?", default="show", choices=("show", "refresh")) p.set_defaults(fn=cmd_repomap) - p = add("media") - p.add_argument("action", nargs="?", default="status", choices=("status", "setup")) - p.set_defaults(fn=cmd_media) p = add("probe") p.add_argument("--sessions", type=int, default=6) p.add_argument("--model", default=None) diff --git a/tokeneconomizer/plugin/config.py b/tokeneconomizer/plugin/config.py index 09c61d4..c7eadac 100644 --- a/tokeneconomizer/plugin/config.py +++ b/tokeneconomizer/plugin/config.py @@ -50,10 +50,6 @@ `paid_model` (claude, codex, copilot, pi, opencode, cursor, grok, hermes); * claude_paid_effort, codex_paid_effort -- their reasoning effort on those two agents; - * media -- `audio`: add on-device transcripts of the audio files a prompt - names (an agent cannot read audio) to that prompt's context, - on hosts that take prompt context; macOS with swiftc. Off by - default (`plugin/media.py`); * local_model, local_roles -- run the named side calls (`judge`, `nudge`, `proposer`; default `judge`) on an on-device model instead of the agent's CLI: `fm`, Apple's Foundation Models CLI on macOS 27. @@ -108,7 +104,6 @@ TOOLPRUNE = ("off", "propose", "auto") LOOPGUARD = ("off", "nudge", "stop") REPOMAP = ("off", "on", "start") -MEDIA = ("off", "audio") MODES = ("auto", "interactive", "headless") AGGR = ("conservative", "default", "aggressive") @@ -212,7 +207,6 @@ class LiveConfig: hermes_paid_model: str = "" claude_paid_effort: str = "" codex_paid_effort: str = "" - media: str = "off" local_model: str = "off" local_roles: list = field(default_factory=lambda: ["judge"]) local_judge_confidence: float = 0.9 @@ -229,7 +223,7 @@ class LiveConfig: _CHOICES = {"mode": MODES, "microskills": MICROSKILLS, "toolprune": TOOLPRUNE, "loopguard": LOOPGUARD, "repomap": REPOMAP, "loopguard_aggr": AGGR, - "local_model": LOCAL_MODELS, "media": MEDIA, + "local_model": LOCAL_MODELS, "vocabulary": ("auto", "generic", "python-git", "node-web"), # Each CLI's own words (Claude Code 2.1.319, Codex 0.167.0); whether a model # takes one is the CLI's to say. @@ -342,7 +336,7 @@ def _set_threshold(self, name, value, source): **{f"TE_{a.upper()}_PAID_MODEL": f"{a}_paid_model" for a in ( "claude", "codex", "copilot", "pi", "opencode", "cursor", "grok", "hermes")}, "TE_CLAUDE_PAID_EFFORT": "claude_paid_effort", "TE_CODEX_PAID_EFFORT": "codex_paid_effort", - "TE_LOCAL_MODEL": "local_model", "TE_LOCAL_ROLES": "local_roles", "TE_MEDIA": "media", + "TE_LOCAL_MODEL": "local_model", "TE_LOCAL_ROLES": "local_roles", "TE_LOCAL_JUDGE_CONFIDENCE": "local_judge_confidence", "TE_INJECT": "microskills", "TE_INJECT_MAX_FACTS": "microskills_max_facts", "TE_INJECT_MAX_TOKENS": "microskills_max_tokens", diff --git a/tokeneconomizer/plugin/controller.py b/tokeneconomizer/plugin/controller.py index aa45de7..72e3d22 100644 --- a/tokeneconomizer/plugin/controller.py +++ b/tokeneconomizer/plugin/controller.py @@ -268,36 +268,6 @@ def _inject_map(self, s, seen, block, event, how): seen["repomap"] = True return out - def _media(self, prompt): - """`media=audio`: the transcripts of the audio files this prompt names, as context. The - first prompt that needs the helper starts building it in the background and goes - without; a missing toolchain is said once.""" - from . import media - try: - block, info = media.for_prompt(prompt, self.p.get("cwd") or self.ctx.repo.root, - env=self.ctx.env) - except Exception: - return [] - if not info["files"]: - return [] - self._log("media", **info, chars=len(block)) - out = [] - seen = self.store.read("notices.json") or {} - if info["needs_helper"] and not seen.get("media_building"): - seen["media_building"] = True - if media.supported(): - self.ctx.spawn(["media", "setup"], "media.log") - out.append(self.host.notice( - "TokenEconomizer: building the on-device audio transcriber (once); audio " - "named from the next prompt on is transcribed into context.")) - else: - out.append(self.host.notice( - "TokenEconomizer: media=audio needs macOS with swiftc; doing nothing.")) - self.store.write("notices.json", seen) - if block and not self.cfg.shadow: - out.append(self.host.context("UserPromptSubmit", block)) - return out - # ---- prompt-turns ------------------------------------------------------------------- def _close_turn(self, s, stopped_normally): data = s.get("turn") @@ -317,14 +287,11 @@ def _close_turn(self, s, stopped_normally): return summary def user_prompt(self): - media_on = self.cfg.media == "audio" and self.host.prompt_context - if self.cfg.loopguard == "off" and not media_on and not (self.cfg.repomap == "on" - and self.host.prompt_context): + if self.cfg.loopguard == "off" and not (self.cfg.repomap == "on" + and self.host.prompt_context): return {} s = self._session() outputs = [] - if media_on: - outputs += self._media(self.p.get("prompt") or "") if self.cfg.repomap == "on" and self.host.prompt_context and not s.get("repomap_done"): # Once per session, on its first prompt: a map ranked toward what the prompt names. s["repomap_done"] = True diff --git a/tokeneconomizer/plugin/media.py b/tokeneconomizer/plugin/media.py deleted file mode 100644 index a13cf76..0000000 --- a/tokeneconomizer/plugin/media.py +++ /dev/null @@ -1,138 +0,0 @@ -"""`media=audio`: transcripts of the audio files a prompt names, transcribed on the machine. - -An agent cannot read audio: Claude Code's Read rejects an `.m4a` or `.aiff` as binary before any -hook sees the call, so a prompt such as "implement what the PM asks in notes/memo.m4a" sends the -agent hunting for a transcriber -- turns, tokens and usually a failed task. With `media=audio` -the prompt's hook finds the audio paths it names, transcribes each with Apple's Speech framework -(`plugin/scripts/te-transcribe.swift`, macOS 26+; no network, no account, no model call billed), -and adds the transcripts to the prompt's context, as the repository map is added. - -The helper is a Swift program built once, with `swiftc`, into `/bin/`: by -`te media setup`, or in the background by the first prompt that names an audio file (that prompt -goes without a transcript). Without `swiftc` or on another OS the mechanism does nothing. -Transcripts are cached by the file's content hash, so naming the same memo again costs nothing. -""" -import hashlib -import os -import re -import shutil -import subprocess -import sys - -from . import paths - -AUDIO_EXT = ("aiff", "aif", "wav", "m4a", "mp3", "caf", "aac", "flac") -# A path the prompt names: a word of path characters ending in an audio extension, which may be -# quoted or backticked; not part of a URL. -AUDIO_RE = re.compile(r"(?= MAX_FILES: - break - return out - - -def source_path(): - """The helper's Swift source in the plugin directory (the checkout or the bundle).""" - plugin = os.environ.get("TE_PLUGIN_ROOT") - candidates = [os.path.join(plugin, "scripts", "te-transcribe.swift")] if plugin else [] - core = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - candidates += [os.path.join(core, "plugin", "scripts", "te-transcribe.swift"), - os.path.join(os.path.dirname(core), "scripts", "te-transcribe.swift")] - return next((c for c in candidates if os.path.isfile(c)), None) - - -def helper_path(env=None): - return os.path.join(paths.state_root(env), "bin", "te-transcribe") - - -def helper_ready(env=None): - h = helper_path(env) - return os.path.isfile(h) and os.access(h, os.X_OK) - - -def supported(): - return sys.platform == "darwin" and shutil.which("swiftc") is not None - - -def build_helper(env=None, timeout=300): - """Compile the helper; (ok, message).""" - if not supported(): - return False, "needs macOS with swiftc (Xcode or the Command Line Tools)" - src = source_path() - if not src: - return False, "te-transcribe.swift not found in the plugin" - out = helper_path(env) - os.makedirs(os.path.dirname(out), exist_ok=True) - tmp = "%s.%d.tmp" % (out, os.getpid()) - try: - r = subprocess.run(["swiftc", "-O", "-parse-as-library", src, "-o", tmp], - capture_output=True, encoding="utf-8", errors="replace", - timeout=timeout, stdin=subprocess.DEVNULL) - except (OSError, subprocess.SubprocessError) as e: - return False, f"swiftc failed to run: {type(e).__name__}" - if r.returncode != 0 or not os.path.isfile(tmp): - return False, "swiftc: " + (r.stderr or r.stdout or "").strip()[-300:] - os.replace(tmp, out) - return True, out - - -def transcript(path, env=None): - """The transcript of `path` ("" when the helper is missing or fails), cached by content.""" - try: - with open(path, "rb") as fh: - key = hashlib.sha256(fh.read()).hexdigest()[:24] - except OSError: - return "" - cache = os.path.join(paths.state_root(env), "media", key + ".txt") - try: - with open(cache, encoding="utf-8") as fh: - return fh.read() - except OSError: - pass - if not helper_ready(env): - return "" - try: - r = subprocess.run([helper_path(env), path], capture_output=True, encoding="utf-8", - errors="replace", timeout=TRANSCRIBE_TIMEOUT_S, - stdin=subprocess.DEVNULL) - except (OSError, subprocess.SubprocessError): - return "" - text = (r.stdout or "").strip()[:MAX_CHARS] - if r.returncode != 0 or not text: - return "" - os.makedirs(os.path.dirname(cache), exist_ok=True) - with open(cache, "w", encoding="utf-8") as fh: - fh.write(text) - return text - - -def for_prompt(prompt, cwd, env=None): - """(block, info) for the prompt's context: the transcripts of the audio it names. `info` - says what happened, for the plugin's log: files found, transcribed, and whether the helper - still has to be built.""" - files = audio_paths(prompt, cwd) - info = {"files": len(files), "transcribed": 0, "needs_helper": bool(files) and not - helper_ready(env)} - parts = [] - for p in files: - text = transcript(p, env) - if text: - info["transcribed"] += 1 - rel = os.path.relpath(p, cwd) if p.startswith(os.path.abspath(cwd)) else p - parts.append(f"[On-device transcript of {rel} (Apple Speech framework; it may " - f"contain recognition errors)]\n{text}") - return "\n\n".join(parts), info From e0b5175546f49eac55f1f0a123c596ecd6b8d7e2 Mon Sep 17 00:00:00 2001 From: Michael Avrukin Date: Tue, 10 Nov 2026 18:01:03 -0500 Subject: [PATCH 4/4] plugin: local judge review fixes (no fm probe in a hook, /usr/bin/fm on macOS only) Per review on #185: - controller: while the agent's CLI is paused, PostToolUse picks the judge from the settings alone (sidecall.local_routes: local_model, local_roles and the platform; no subprocess). Each hook is a new process, so the old `fm available` probe ran on every tool call. The judge's own job checks that the model answers, and with none spares the turn. - sidecall: FM_CLI is /usr/bin/fm, run by its full path (os.access), never looked up on PATH, and only on macOS. - config: the local_judge_confidence docstring points at #186 instead of quoting the comparison with Haiku. - plugin/README.md: without fm the call goes to the agent's CLI without an errors.log line; only a call fm cannot answer is logged. The setting does nothing off macOS. - tests: local_routes runs no subprocess; a paused host's _judge never probes fm (fails on the old code); fm_available runs /usr/bin/fm on macOS only and never searches PATH. A `_macos` seam lets them run on Linux CI. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UporVpn64UvJqEnurKVwza --- plugin/README.md | 13 +++--- tests/test_local_model.py | 63 +++++++++++++++++++++++----- tokeneconomizer/plugin/config.py | 10 ++--- tokeneconomizer/plugin/controller.py | 5 ++- tokeneconomizer/plugin/sidecall.py | 26 +++++++++--- 5 files changed, 87 insertions(+), 30 deletions(-) diff --git a/plugin/README.md b/plugin/README.md index 3bb26e1..e6a11e4 100644 --- a/plugin/README.md +++ b/plugin/README.md @@ -917,12 +917,13 @@ would send the agent's CLI, on stdin, with greedy decoding; the judge's call als schema for its verdict (`fm respond --schema`), which constrains what the model can return. Its window is 4,096 tokens, enough for the judge's and the redirect writer's prompts. -The ledger charges an on-device call 0 tokens. A call it cannot make (no `fm`, the model not -ready, a prompt over its window, no reply) leaves a line in `errors.log` and goes to the agent's -CLI as before, so turning this on never leaves the guard without its judge; while that CLI is -paused (signed out), the on-device judge still answers. A stuck verdict from the on-device judge -counts only at `local_judge_confidence` (0.9) or above: its confidences come in a few coarse -steps, most of them above the guard's own `judge_confidence` (0.7). +The ledger charges an on-device call 0 tokens. Without `/usr/bin/fm`, or with its model not ready, +the call goes to the agent's CLI as before; a call `fm` cannot answer (a prompt over its window, no +reply) leaves a line in `errors.log` and goes there too, so turning this on never leaves the guard +without its judge. While that CLI is paused (signed out), the on-device judge still answers. On any +system but macOS the setting does nothing. A stuck verdict from the on-device judge counts only at +`local_judge_confidence` (0.9) or above: its confidences come in a few coarse steps, most of them +above the guard's own `judge_confidence` (0.7). ```text ~/.tokeneconomizer/bin/te config set local_model=fm diff --git a/tests/test_local_model.py b/tests/test_local_model.py index b771db7..8bad8b4 100644 --- a/tests/test_local_model.py +++ b/tests/test_local_model.py @@ -41,6 +41,12 @@ def test_local_settings_come_from_the_environment(tmp_path): assert (c.local_model, c.local_roles) == ("fm", ["judge", "proposer"]) +@pytest.fixture(autouse=True) +def _on_macos(monkeypatch): + """`fm` runs only on macOS; these tests run everywhere, as if there.""" + monkeypatch.setattr(sidecall, "_macos", lambda: True) + + # ---- routing ------------------------------------------------------------------------------- @pytest.fixture def fm_ok(monkeypatch): @@ -61,6 +67,38 @@ def test_no_local_runner_unless_turned_on_and_the_role_listed(fm_ok): assert sidecall.local_runner("judge", cfg(local_model="fm")) is not None +def test_local_routes_reads_only_the_settings_and_the_platform(monkeypatch): + def probe(*a, **k): + raise AssertionError("local_routes ran a subprocess") + monkeypatch.setattr(sidecall, "fm_available", probe) + monkeypatch.setattr(sidecall.subprocess, "run", probe) + assert sidecall.local_routes("judge", cfg(local_model="fm")) + assert not sidecall.local_routes("nudge", cfg(local_model="fm")) + assert not sidecall.local_routes("judge", cfg()) + monkeypatch.setattr(sidecall, "_macos", lambda: False) + assert not sidecall.local_routes("judge", cfg(local_model="fm")) + + +@pytest.mark.parametrize("local_model, expect", [("fm", "AsyncJudge"), ("off", "NoVerdict")]) +def test_a_paused_hosts_hook_picks_its_judge_from_the_settings_without_probing_fm( + monkeypatch, local_model, expect): + # PostToolUse runs `_judge` in a new process on every tool call: it must not run `fm`. + from tokeneconomizer.plugin import controller + monkeypatch.setattr(sidecall, "paused", lambda host, root=None: {"at": 1.0}) + monkeypatch.setattr(sidecall, "fm_available", lambda *a, **k: pytest.fail("probed fm")) + monkeypatch.setattr(sidecall.subprocess, "run", lambda *a, **k: pytest.fail("ran a process")) + ctx = type("Ctx", (), {"config": cfg(local_model=local_model), "root": None})() + host = type("Host", (), {"name": "claude"})() + ctl = type("C", (), {"host": host, "ctx": ctx, "sid": "S"})() + verdict = controller.LiveController._judge(ctl, {"judge_confidence": 0.7}, envelope={}) + assert type(verdict).__name__ == expect + + +def test_no_local_runner_off_macos(fm_ok, monkeypatch): + monkeypatch.setattr(sidecall, "_macos", lambda: False) + assert sidecall.local_runner("judge", cfg(local_model="fm")) is None + + def test_no_local_runner_without_fm(monkeypatch): monkeypatch.setattr(sidecall, "fm_available", lambda cli="fm": False) assert sidecall.local_runner("judge", cfg(local_model="fm")) is None @@ -114,7 +152,7 @@ def fake_run(cmd, **k): monkeypatch.setattr(sidecall.subprocess, "run", fake_run) text, eff = sidecall.fm_call("PROMPT", schema=sidecall.JUDGE_SCHEMA) assert (text, eff) == ('{"stuck": false}', 0) - assert seen["cmd"][:4] == ["fm", "respond", "--no-stream", "--greedy"] + assert seen["cmd"][:4] == ["/usr/bin/fm", "respond", "--no-stream", "--greedy"] assert seen["input"] == "PROMPT" and "PROMPT" not in seen["cmd"] assert seen["schema"]["required"] == ["stuck", "confidence", "reason"] @@ -143,18 +181,23 @@ def boom(cmd, **k): @pytest.mark.real_sidecalls -@pytest.mark.parametrize("which, rc, out, ok", [ - (None, 0, "System model available", False), - ("/usr/bin/fm", 0, "System model available", True), - ("/usr/bin/fm", 0, "System model unavailable: modelNotReady", False), - ("/usr/bin/fm", 1, "", False), +@pytest.mark.parametrize("macos, installed, rc, out, ok", [ + (True, False, 0, "System model available", False), + (False, True, 0, "System model available", False), + (True, True, 0, "System model available", True), + (True, True, 0, "System model unavailable: modelNotReady", False), + (True, True, 1, "", False), ]) -def test_fm_available(monkeypatch, which, rc, out, ok): +def test_fm_available_runs_usr_bin_fm_on_macos_only(monkeypatch, macos, installed, rc, out, ok): + ran = [] monkeypatch.setattr(sidecall, "_fm_ok", {}) - monkeypatch.setattr(sidecall.shutil, "which", lambda name: which) - monkeypatch.setattr(sidecall.subprocess, "run", lambda cmd, **k: subprocess.CompletedProcess( - cmd, rc, stdout=out, stderr="")) + monkeypatch.setattr(sidecall, "_macos", lambda: macos) + monkeypatch.setattr(sidecall.os, "access", lambda p, m: installed and p == "/usr/bin/fm") + monkeypatch.setattr(sidecall.shutil, "which", lambda *a, **k: pytest.fail("looked up PATH")) + monkeypatch.setattr(sidecall.subprocess, "run", lambda cmd, **k: ran.append(cmd) or + subprocess.CompletedProcess(cmd, rc, stdout=out, stderr="")) assert sidecall.fm_available() is ok + assert ran == ([["/usr/bin/fm", "available"]] if macos and installed else []) # ---- the judge reads the local verdict ----------------------------------------------------- diff --git a/tokeneconomizer/plugin/config.py b/tokeneconomizer/plugin/config.py index c7eadac..ceec38c 100644 --- a/tokeneconomizer/plugin/config.py +++ b/tokeneconomizer/plugin/config.py @@ -57,12 +57,10 @@ model unavailable, no reply -- falls back to the agent's CLI. Local calls are charged 0 tokens; * local_judge_confidence -- the confidence a stuck verdict from the local judge must reach - to count (default 0.9); below it the turn is spared. The - local model's confidences cluster (0.65/0.72/0.85/0.92), so - the guard's own `judge_confidence` (0.7) would let nearly - every local "stuck" through; 0.9 is where, on 302 replayed - SWE-agent judge points, it redirected fewer successful runs - than Haiku at 0.7 while keeping the same separation; + to count (default 0.9); below it the turn is spared. The local + model's confidences come in coarse steps, so the guard's own + `judge_confidence` would let nearly every local "stuck" + through; how 0.9 was chosen: microsoft/TokenEconomizer#186; * pi_extensions, pi_cli -- on Pi, the extensions the plugin's own model calls load (a custom provider's, whose model they could not reach otherwise; comma-separated paths), and the `pi` they run; diff --git a/tokeneconomizer/plugin/controller.py b/tokeneconomizer/plugin/controller.py index 72e3d22..d1ce084 100644 --- a/tokeneconomizer/plugin/controller.py +++ b/tokeneconomizer/plugin/controller.py @@ -335,9 +335,10 @@ def _judge(self, thr, envelope): if self.host.name in sidecall.NO_SIDE_CALLS: return NoVerdict() # A local judge (`local_model`) needs no login, so it still answers while the host's - # CLI is paused (signed out). + # CLI is paused (signed out). Only the settings are read here, as this runs on every + # tool call; the job checks that the model answers, and with none spares the turn. if (sidecall.paused(self.host.name, self.ctx.root) - and sidecall.local_runner("judge", self.ctx.config) is None): + and not sidecall.local_routes("judge", self.ctx.config)): return NoVerdict() return AsyncJudge(self.ctx, self.sid, thr.get("judge_confidence")) diff --git a/tokeneconomizer/plugin/sidecall.py b/tokeneconomizer/plugin/sidecall.py index cf5a21b..dacbce9 100644 --- a/tokeneconomizer/plugin/sidecall.py +++ b/tokeneconomizer/plugin/sidecall.py @@ -58,6 +58,7 @@ import re import shutil import subprocess +import sys import tempfile from .. import trace @@ -754,7 +755,9 @@ def copilot_call(prompt, model="", cli="copilot", timeout=TIMEOUT_S): # it returns the object every time. A call it cannot make returns None, and the caller falls back # to the agent's own CLI, so turning this on never costs a verdict. -FM_CLI = "fm" +# Apple's Foundation Models CLI: macOS 27 ships it here, so it is run by this path, never looked +# up on PATH, and only on macOS. +FM_CLI = "/usr/bin/fm" FM_TIMEOUT_S = 60 # The judge's verdict, as `loopguard.judge.llm_stuck_judge` parses it. JUDGE_SCHEMA = { @@ -769,11 +772,24 @@ def copilot_call(prompt, model="", cli="copilot", timeout=TIMEOUT_S): _fm_ok = {} +def _macos(): + return sys.platform == "darwin" + + +def local_routes(role, config): + """Whether `local_model` sends `role`'s side call on-device: the settings and the platform + alone, no subprocess, so a hook can ask on every tool call. Whether the model answers is + `fm_available`'s to say, in the judge's own job.""" + return (_macos() and getattr(config, "local_model", "off") == "fm" + and role in (getattr(config, "local_roles", None) or ())) + + def fm_available(cli=FM_CLI): - """Whether `fm` is on PATH and reports its on-device model available (once per process).""" + """Whether `fm` is installed (macOS only) and reports its on-device model available (once + per process).""" if cli not in _fm_ok: ok = False - if shutil.which(cli): + if _macos() and os.access(cli, os.X_OK): try: out = subprocess.run([cli, "available"], capture_output=True, encoding="utf-8", errors="replace", timeout=30, env=_env(), @@ -814,9 +830,7 @@ def local_runner(role, config, fallback=None): """The side-call runner for `role` when `local_model` routes it on-device, else None. On a failed local call the runner tries `fallback` (the agent's own runner), so a turned-on local model never loses a verdict; with no fallback (a host in `NO_SIDE_CALLS`) it returns ''.""" - if getattr(config, "local_model", "off") != "fm" or role not in getattr(config, "local_roles", ()): - return None - if not fm_available(): + if not local_routes(role, config) or not fm_available(): return None schema = ROLE_SCHEMAS.get(role)