diff --git a/README.md b/README.md index 449a089..bc73164 100644 --- a/README.md +++ b/README.md @@ -336,7 +336,9 @@ only when a turn shows signs of looping: the judge, and the redirect it writes i turn stuck. Micro-skills is off by default, because its proposer adds model calls and how much it saves depends on the repository and the work you do there. What each of these calls reads, how often it runs and which model it uses are listed under -[Model calls](plugin/README.md#model-calls-and-api-keys). Each mechanism has its own switch +[Model calls](plugin/README.md#model-calls-and-api-keys). On macOS 27, `local_model=fm` runs the +judge on Apple's on-device model instead ([On-device side calls](plugin/README.md#on-device-side-calls-macos)); +it is off by default. Each mechanism has its own switch (`microskills`, `toolprune`, `loopguard`) that takes `on` or `off`, and the italic letters above are these settings: diff --git a/plugin/README.md b/plugin/README.md index 1483d7f..e6a11e4 100644 --- a/plugin/README.md +++ b/plugin/README.md @@ -798,6 +798,9 @@ first session there ends. | `paid_model` | `""` | model for the plugin's own calls on every agent; empty: `haiku` on Claude Code, the agent's own default model on Codex and Pi, on Copilot `COPILOT_PROVIDER_MODEL_ID` or `COPILOT_MODEL` (unless it is `auto`), else a BYOK provider's Haiku, else Copilot's own default | | `_paid_model` | `""` | the same for one agent (`claude`, `codex`, `copilot`, `pi`, `opencode`, `cursor`, `grok`, `hermes`), over `paid_model` | | `claude_paid_effort`, `codex_paid_effort` | `""` | the reasoning effort of those calls: Claude Code `low` to `max`, Codex `none` to `max`; which values a model takes is the CLI's to say; empty: the CLI's default | +| `local_model` | `off` | `fm`: run the side calls `local_roles` names on Apple's on-device Foundation Models CLI (`/usr/bin/fm`, macOS 27 with Apple Intelligence) instead of the agent's CLI; see [On-device side calls](#on-device-side-calls-macos) | +| `local_roles` | `judge` | which side calls `local_model` takes: `judge`, `nudge` (the redirect writer), `proposer`, separated by commas | +| `local_judge_confidence` | `0.9` | how sure a stuck verdict from the on-device judge must be to count; below it the turn is spared | | `pi_extensions` | `""` | Pi only: extension files (comma-separated paths) the plugin's own calls load, for a model whose provider an extension registers | | `pi_cli` | `""` | Pi only: the `pi` the plugin's own calls run; empty: the first on PATH | | `shadow` | `false` | decide and log everything, act on nothing | @@ -905,6 +908,28 @@ at its backstop), the next session start says so once, and `config set microskil `loopguard=on`) tries again. `toolprune measure` on Claude Code makes its two calls the same way, and only when you run it. +## On-device side calls (macOS) + +With `local_model=fm`, the side calls `local_roles` names (by default the loop guard's judge) run +on the Foundation Models CLI that macOS 27 ships at `/usr/bin/fm` when Apple Intelligence is on: +a model on the Mac itself, with no login, key or network. The plugin sends it the same prompt it +would send the agent's CLI, on stdin, with greedy decoding; the judge's call also passes a JSON +schema for its verdict (`fm respond --schema`), which constrains what the model can return. Its +window is 4,096 tokens, enough for the judge's and the redirect writer's prompts. + +The ledger charges an on-device call 0 tokens. Without `/usr/bin/fm`, or with its model not ready, +the call goes to the agent's CLI as before; a call `fm` cannot answer (a prompt over its window, no +reply) leaves a line in `errors.log` and goes there too, so turning this on never leaves the guard +without its judge. While that CLI is paused (signed out), the on-device judge still answers. On any +system but macOS the setting does nothing. A stuck verdict from the on-device judge counts only at +`local_judge_confidence` (0.9) or above: its confidences come in a few coarse steps, most of them +above the guard's own `judge_confidence` (0.7). + +```text +~/.tokeneconomizer/bin/te config set local_model=fm +~/.tokeneconomizer/bin/te config set local_roles=judge,nudge # also the redirect writer +``` + ## Troubleshooting | Symptom | Fix | diff --git a/tests/conftest.py b/tests/conftest.py index fe7119e..5640bd5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -60,3 +60,6 @@ def _no_model_calls(request, monkeypatch): for name in ("claude_call", "codex_call", "copilot_call", "grok_call", "hermes_call", "pi_call"): monkeypatch.setattr(sidecall, name, lambda *a, **k: ("", None)) + # Nor the on-device model: `fm` answers nothing, as if absent, unless a test fakes it. + monkeypatch.setattr(sidecall, "fm_call", lambda *a, **k: (None, 0)) + monkeypatch.setattr(sidecall, "_fm_ok", {}) diff --git a/tests/test_local_model.py b/tests/test_local_model.py new file mode 100644 index 0000000..8bad8b4 --- /dev/null +++ b/tests/test_local_model.py @@ -0,0 +1,273 @@ +"""The on-device model some side calls can run on (`local_model`, `local_roles`). + +`fm` is faked throughout: these tests pin the routing, the fallback to the agent's own CLI, the +zero charge, and the call itself (argv, schema, stdin, ANSI stripping), never the model. +""" +import json +import subprocess + +import pytest + +from tokeneconomizer.loopguard import judge as loop_judge +from tokeneconomizer.plugin import config as live_config +from tokeneconomizer.plugin import sidecall + + +def cfg(**kw): + c = live_config.LiveConfig() + for k, v in kw.items(): + assert c.set(k, v), c.warnings + return c + + +# ---- settings ------------------------------------------------------------------------------ +def test_local_model_is_off_by_default_and_names_the_judge(): + c = live_config.LiveConfig() + assert c.local_model == "off" and c.local_roles == ["judge"] + + +def test_local_settings_parse_and_reject_unknown_values(): + c = live_config.LiveConfig() + assert c.set("local_model", "fm") and c.local_model == "fm" + assert c.set("local_roles", "judge, nudge") and c.local_roles == ["judge", "nudge"] + assert not c.set("local_model", "ollama") and c.local_model == "fm" + assert not c.set("local_roles", "judge,summarizer") and c.local_roles == ["judge", "nudge"] + assert len(c.warnings) == 2 + + +def test_local_settings_come_from_the_environment(tmp_path): + c = live_config.load_config(str(tmp_path), env={"TE_LOCAL_MODEL": "fm", + "TE_LOCAL_ROLES": "judge,proposer"}) + assert (c.local_model, c.local_roles) == ("fm", ["judge", "proposer"]) + + +@pytest.fixture(autouse=True) +def _on_macos(monkeypatch): + """`fm` runs only on macOS; these tests run everywhere, as if there.""" + monkeypatch.setattr(sidecall, "_macos", lambda: True) + + +# ---- routing ------------------------------------------------------------------------------- +@pytest.fixture +def fm_ok(monkeypatch): + calls = [] + + def fake(prompt, schema=None, **k): + calls.append((prompt, schema)) + return '{"stuck": true, "confidence": 0.9, "reason": "same failure"}', 0 + + monkeypatch.setattr(sidecall, "fm_available", lambda cli="fm": True) + monkeypatch.setattr(sidecall, "fm_call", fake) + return calls + + +def test_no_local_runner_unless_turned_on_and_the_role_listed(fm_ok): + assert sidecall.local_runner("judge", cfg()) is None + assert sidecall.local_runner("nudge", cfg(local_model="fm")) is None + assert sidecall.local_runner("judge", cfg(local_model="fm")) is not None + + +def test_local_routes_reads_only_the_settings_and_the_platform(monkeypatch): + def probe(*a, **k): + raise AssertionError("local_routes ran a subprocess") + monkeypatch.setattr(sidecall, "fm_available", probe) + monkeypatch.setattr(sidecall.subprocess, "run", probe) + assert sidecall.local_routes("judge", cfg(local_model="fm")) + assert not sidecall.local_routes("nudge", cfg(local_model="fm")) + assert not sidecall.local_routes("judge", cfg()) + monkeypatch.setattr(sidecall, "_macos", lambda: False) + assert not sidecall.local_routes("judge", cfg(local_model="fm")) + + +@pytest.mark.parametrize("local_model, expect", [("fm", "AsyncJudge"), ("off", "NoVerdict")]) +def test_a_paused_hosts_hook_picks_its_judge_from_the_settings_without_probing_fm( + monkeypatch, local_model, expect): + # PostToolUse runs `_judge` in a new process on every tool call: it must not run `fm`. + from tokeneconomizer.plugin import controller + monkeypatch.setattr(sidecall, "paused", lambda host, root=None: {"at": 1.0}) + monkeypatch.setattr(sidecall, "fm_available", lambda *a, **k: pytest.fail("probed fm")) + monkeypatch.setattr(sidecall.subprocess, "run", lambda *a, **k: pytest.fail("ran a process")) + ctx = type("Ctx", (), {"config": cfg(local_model=local_model), "root": None})() + host = type("Host", (), {"name": "claude"})() + ctl = type("C", (), {"host": host, "ctx": ctx, "sid": "S"})() + verdict = controller.LiveController._judge(ctl, {"judge_confidence": 0.7}, envelope={}) + assert type(verdict).__name__ == expect + + +def test_no_local_runner_off_macos(fm_ok, monkeypatch): + monkeypatch.setattr(sidecall, "_macos", lambda: False) + assert sidecall.local_runner("judge", cfg(local_model="fm")) is None + + +def test_no_local_runner_without_fm(monkeypatch): + monkeypatch.setattr(sidecall, "fm_available", lambda cli="fm": False) + assert sidecall.local_runner("judge", cfg(local_model="fm")) is None + + +def test_runner_for_a_local_role_uses_fm_with_its_schema_and_charges_nothing(fm_ok, monkeypatch): + host = [] + monkeypatch.setattr(sidecall, "claude_call", lambda p, model="": host.append(p) or ("h", 9.0)) + run = sidecall.runner_for("claude", cfg(local_model="fm"), role="judge") + text, eff = run("digest", "agent") + assert json.loads(text)["stuck"] is True and eff == 0 and host == [] + assert fm_ok == [("digest", sidecall.JUDGE_SCHEMA)] + + +def test_runner_for_without_a_role_or_for_another_role_stays_on_the_host(fm_ok, monkeypatch): + monkeypatch.setattr(sidecall, "claude_call", lambda p, model="": ("h", 9.0)) + c = cfg(local_model="fm") + assert sidecall.runner_for("claude", c)("q") == ("h", 9.0) + assert sidecall.runner_for("claude", c, role="proposer")("q") == ("h", 9.0) + assert fm_ok == [] + + +def test_a_failed_local_call_falls_back_to_the_agents_cli(monkeypatch): + monkeypatch.setattr(sidecall, "fm_available", lambda cli="fm": True) + monkeypatch.setattr(sidecall, "fm_call", lambda *a, **k: (None, 0)) + monkeypatch.setattr(sidecall, "claude_call", lambda p, model="": ("host verdict", 9.0)) + run = sidecall.runner_for("claude", cfg(local_model="fm"), role="judge") + assert run("digest") == ("host verdict", 9.0) + + +def test_a_host_with_no_side_calls_gets_none_from_runner_for_and_an_empty_local_fallback( + fm_ok, monkeypatch): + monkeypatch.setattr(sidecall, "fm_call", lambda *a, **k: (None, 0)) + c = cfg(local_model="fm", local_roles="proposer") + assert sidecall.runner_for("antigravity", cfg()) is None + run = sidecall.runner_for("antigravity", c, role="proposer") + assert run is not None and run("q") == ("", 0) + + +# ---- the call ------------------------------------------------------------------------------ +@pytest.mark.real_sidecalls +def test_fm_call_runs_greedy_with_the_schema_file_and_the_prompt_on_stdin(monkeypatch): + seen = {} + + def fake_run(cmd, **k): + seen["cmd"], seen["input"] = cmd, k.get("input") + seen["schema"] = json.load(open(cmd[cmd.index("--schema") + 1])) + return subprocess.CompletedProcess(cmd, 0, stdout='\x1b[1m{"stuck": false}\x1b[0m\n', + stderr="") + + monkeypatch.setattr(sidecall.subprocess, "run", fake_run) + text, eff = sidecall.fm_call("PROMPT", schema=sidecall.JUDGE_SCHEMA) + assert (text, eff) == ('{"stuck": false}', 0) + assert seen["cmd"][:4] == ["/usr/bin/fm", "respond", "--no-stream", "--greedy"] + assert seen["input"] == "PROMPT" and "PROMPT" not in seen["cmd"] + assert seen["schema"]["required"] == ["stuck", "confidence", "reason"] + + +@pytest.mark.real_sidecalls +@pytest.mark.parametrize("rc, out", [(1, ""), (0, ""), (0, "Error: The session's transcript " + "exceeded the model's context size.")]) +def test_fm_call_without_a_usable_reply_returns_none_and_logs(tmp_path, monkeypatch, rc, out): + monkeypatch.setenv("TOKENECONOMIZER_HOME", str(tmp_path)) + monkeypatch.setattr(sidecall.subprocess, "run", lambda cmd, **k: subprocess.CompletedProcess( + cmd, rc, stdout=out, stderr="")) + assert sidecall.fm_call("SECRET prompt") == (None, 0) + log = (tmp_path / "errors.log").read_text(encoding="utf-8") + assert "sidecall fm: no reply" in log and "SECRET" not in log + + +@pytest.mark.real_sidecalls +def test_fm_call_that_cannot_start_returns_none(tmp_path, monkeypatch): + monkeypatch.setenv("TOKENECONOMIZER_HOME", str(tmp_path)) + + def boom(cmd, **k): + raise subprocess.TimeoutExpired(cmd, 60) + + monkeypatch.setattr(sidecall.subprocess, "run", boom) + assert sidecall.fm_call("q") == (None, 0) + + +@pytest.mark.real_sidecalls +@pytest.mark.parametrize("macos, installed, rc, out, ok", [ + (True, False, 0, "System model available", False), + (False, True, 0, "System model available", False), + (True, True, 0, "System model available", True), + (True, True, 0, "System model unavailable: modelNotReady", False), + (True, True, 1, "", False), +]) +def test_fm_available_runs_usr_bin_fm_on_macos_only(monkeypatch, macos, installed, rc, out, ok): + ran = [] + monkeypatch.setattr(sidecall, "_fm_ok", {}) + monkeypatch.setattr(sidecall, "_macos", lambda: macos) + monkeypatch.setattr(sidecall.os, "access", lambda p, m: installed and p == "/usr/bin/fm") + monkeypatch.setattr(sidecall.shutil, "which", lambda *a, **k: pytest.fail("looked up PATH")) + monkeypatch.setattr(sidecall.subprocess, "run", lambda cmd, **k: ran.append(cmd) or + subprocess.CompletedProcess(cmd, rc, stdout=out, stderr="")) + assert sidecall.fm_available() is ok + assert ran == ([["/usr/bin/fm", "available"]] if macos and installed else []) + + +# ---- the judge reads the local verdict ----------------------------------------------------- +def test_the_stuck_judge_parses_a_schema_ordered_local_verdict_at_zero_cost(fm_ok): + run = sidecall.runner_for("claude", cfg(local_model="fm"), role="judge") + v = loop_judge.llm_stuck_judge("turns=20 ...", run) + assert (v["stuck"], v["confidence"], v["_tokens"]) == (True, 0.9, 0) + + +# ---- the judge job: the local floor -------------------------------------------------------- +class _Store: + def __init__(self, req): + self.files = {"judge/S.request.json": req} + self.appended = [] + + def read(self, k): + return self.files.get(k) + + def write(self, k, v): + self.files[k] = v + + def append(self, k, v): + self.appended.append((k, v)) + + +def _job(tmp_path, monkeypatch, confidence, **cfg_kw): + from tokeneconomizer.plugin import judge as plugin_judge + monkeypatch.setattr(sidecall, "fm_available", lambda cli="fm": True) + verdict = json.dumps({"stuck": True, "confidence": confidence, "reason": "same failure"}) + monkeypatch.setattr(sidecall, "fm_call", lambda prompt, schema=None, **k: ( + verdict if schema else "Re-read the failing test before editing again.", 0)) + store = _Store({"id": "J", "digest": "turns=20 ...", "threshold": 0.7}) + host = type("H", (), {"name": "claude"})() + ctx = type("C", (), {"host": host, "root": str(tmp_path), "store": store, + "config": cfg(local_model="fm", local_roles="judge,nudge", **cfg_kw)})() + v = plugin_judge.run_job(ctx, "S", "J") + return v, store + + +def test_a_local_stuck_verdict_below_the_floor_spares_the_turn(tmp_path, monkeypatch): + v, store = _job(tmp_path, monkeypatch, 0.85) + assert v["stuck"] is False and v["local"] == "fm" and "_nudge" not in v + assert v["reason"].startswith("below local_judge_confidence") + assert store.appended == [] # nothing spent, nothing charged + + +def test_a_local_stuck_verdict_at_the_floor_redirects_with_a_local_nudge(tmp_path, monkeypatch): + v, _ = _job(tmp_path, monkeypatch, 0.92) + assert v["stuck"] is True and v["_nudge"].startswith("Re-read") + + +def test_the_floor_is_a_setting(tmp_path, monkeypatch): + v, _ = _job(tmp_path, monkeypatch, 0.85, local_judge_confidence=0.8) + assert v["stuck"] is True + + +def test_the_floor_does_not_apply_when_the_host_answered(tmp_path, monkeypatch): + from tokeneconomizer.plugin import judge as plugin_judge + monkeypatch.setattr(sidecall, "fm_available", lambda cli="fm": True) + monkeypatch.setattr(sidecall, "fm_call", lambda *a, **k: (None, 0)) + monkeypatch.setattr(sidecall, "claude_call", lambda p, model="": ( + '{"stuck": true, "confidence": 0.75, "reason": "loop"}', 5.0)) + store = _Store({"id": "J", "digest": "turns=20 ...", "threshold": 0.7}) + ctx = type("C", (), {"host": type("H", (), {"name": "claude"})(), "root": str(tmp_path), + "store": store, "config": cfg(local_model="fm")})() + v = plugin_judge.run_job(ctx, "S", "J") + assert v["stuck"] is True and "local" not in v and store.appended # host spend charged + + +def test_local_judge_confidence_must_be_a_probability(): + c = live_config.LiveConfig() + assert c.set("local_judge_confidence", "0.8") and c.local_judge_confidence == 0.8 + assert not c.set("local_judge_confidence", 1.5) and not c.set("local_judge_confidence", True) diff --git a/tokeneconomizer/plugin/config.py b/tokeneconomizer/plugin/config.py index c71538b..ceec38c 100644 --- a/tokeneconomizer/plugin/config.py +++ b/tokeneconomizer/plugin/config.py @@ -50,6 +50,17 @@ `paid_model` (claude, codex, copilot, pi, opencode, cursor, grok, hermes); * claude_paid_effort, codex_paid_effort -- their reasoning effort on those two agents; + * local_model, local_roles -- run the named side calls (`judge`, `nudge`, `proposer`; + default `judge`) on an on-device model instead of the + agent's CLI: `fm`, Apple's Foundation Models CLI on macOS 27. + Off by default. A call the local model cannot make -- no `fm`, + model unavailable, no reply -- falls back to the agent's CLI. + Local calls are charged 0 tokens; + * local_judge_confidence -- the confidence a stuck verdict from the local judge must reach + to count (default 0.9); below it the turn is spared. The local + model's confidences come in coarse steps, so the guard's own + `judge_confidence` would let nearly every local "stuck" + through; how 0.9 was chosen: microsoft/TokenEconomizer#186; * pi_extensions, pi_cli -- on Pi, the extensions the plugin's own model calls load (a custom provider's, whose model they could not reach otherwise; comma-separated paths), and the `pi` they run; @@ -82,6 +93,12 @@ from dataclasses import asdict, dataclass, field, fields MICROSKILLS = ("off", "envfacts", "envfacts+distiller", "distiller") +# An on-device model some side calls can run on instead of the agent's CLI (`sidecall.local_runner`): +# "fm" is Apple's Foundation Models CLI (`/usr/bin/fm`, macOS 27 with Apple Intelligence). +LOCAL_MODELS = ("off", "fm") +# The side calls `local_roles` may name. The judge is the default because it is the one the +# evaluation supports; the redirect writer and the proposer are opt-in. +LOCAL_ROLES = ("judge", "nudge", "proposer") TOOLPRUNE = ("off", "propose", "auto") LOOPGUARD = ("off", "nudge", "stop") REPOMAP = ("off", "on", "start") @@ -188,6 +205,9 @@ class LiveConfig: hermes_paid_model: str = "" claude_paid_effort: str = "" codex_paid_effort: str = "" + local_model: str = "off" + local_roles: list = field(default_factory=lambda: ["judge"]) + local_judge_confidence: float = 0.9 pi_extensions: list = field(default_factory=list) pi_cli: str = "" hermes_cli: str = "" @@ -201,6 +221,7 @@ class LiveConfig: _CHOICES = {"mode": MODES, "microskills": MICROSKILLS, "toolprune": TOOLPRUNE, "loopguard": LOOPGUARD, "repomap": REPOMAP, "loopguard_aggr": AGGR, + "local_model": LOCAL_MODELS, "vocabulary": ("auto", "generic", "python-git", "node-web"), # Each CLI's own words (Claude Code 2.1.319, Codex 0.167.0); whether a model # takes one is the CLI's to say. @@ -245,6 +266,12 @@ def set(self, key, value, source="?"): if isinstance(cur, list): items = value if isinstance(value, list) else str(value).split(",") v = [str(x).strip() for x in items if str(x).strip()] + elif isinstance(cur, float): + if isinstance(value, bool): + raise ValueError("not a number") + v = float(value) + if not 0.0 <= v <= 1.0: + raise ValueError("must be between 0 and 1") elif isinstance(cur, bool): v = value if isinstance(value, bool) else str(value).strip().lower() in ( "1", "true", "yes", "on") @@ -262,6 +289,11 @@ def set(self, key, value, source="?"): except (TypeError, ValueError, OverflowError) as e: self.warnings.append(f"{source}: {key}={value!r} rejected ({e})") return False + if key == "local_roles": + bad = [r for r in v if r not in LOCAL_ROLES] + if bad: + self.warnings.append(f"{source}: local_roles {bad!r} not in {LOCAL_ROLES}") + return False choices = self._CHOICES.get(key) if choices and v not in choices: self.warnings.append(f"{source}: {key}={v!r} not one of {choices}") @@ -302,6 +334,8 @@ def _set_threshold(self, name, value, source): **{f"TE_{a.upper()}_PAID_MODEL": f"{a}_paid_model" for a in ( "claude", "codex", "copilot", "pi", "opencode", "cursor", "grok", "hermes")}, "TE_CLAUDE_PAID_EFFORT": "claude_paid_effort", "TE_CODEX_PAID_EFFORT": "codex_paid_effort", + "TE_LOCAL_MODEL": "local_model", "TE_LOCAL_ROLES": "local_roles", + "TE_LOCAL_JUDGE_CONFIDENCE": "local_judge_confidence", "TE_INJECT": "microskills", "TE_INJECT_MAX_FACTS": "microskills_max_facts", "TE_INJECT_MAX_TOKENS": "microskills_max_tokens", "TE_PROPOSE_EVERY": "microskills_propose_every", diff --git a/tokeneconomizer/plugin/controller.py b/tokeneconomizer/plugin/controller.py index cfef0b2..d1ce084 100644 --- a/tokeneconomizer/plugin/controller.py +++ b/tokeneconomizer/plugin/controller.py @@ -332,8 +332,13 @@ def _judge(self, thr, envelope): from .judge import AsyncJudge, NoVerdict if envelope is None: return None - if self.host.name in sidecall.NO_SIDE_CALLS or sidecall.paused(self.host.name, - self.ctx.root): + if self.host.name in sidecall.NO_SIDE_CALLS: + return NoVerdict() + # A local judge (`local_model`) needs no login, so it still answers while the host's + # CLI is paused (signed out). Only the settings are read here, as this runs on every + # tool call; the job checks that the model answers, and with none spares the turn. + if (sidecall.paused(self.host.name, self.ctx.root) + and not sidecall.local_routes("judge", self.ctx.config)): return NoVerdict() return AsyncJudge(self.ctx, self.sid, thr.get("judge_confidence")) diff --git a/tokeneconomizer/plugin/fold.py b/tokeneconomizer/plugin/fold.py index a99e0d5..9c565f2 100644 --- a/tokeneconomizer/plugin/fold.py +++ b/tokeneconomizer/plugin/fold.py @@ -77,9 +77,11 @@ def _fold_microskills(ctx, adapter, run_dir, session_id, injected=None, imported # Paused once the host's CLI said it is not signed in (`sidecall.paused`): the free layer # still learns. A session read from the agent's history never calls a model: the free # layer learns from it, and the proposer waits for sessions the plugin sees end. - runner = (sidecall.runner_for(ctx.host.name, ctx.config) - if icfg.paid and not imported and not sidecall.paused(ctx.host.name, ctx.root) - else None) + runner = None + if icfg.paid and not imported: + runner = (sidecall.runner_for(ctx.host.name, ctx.config, role="proposer") + if not sidecall.paused(ctx.host.name, ctx.root) + else sidecall.local_runner("proposer", ctx.config)) stage = PrefixStage(icfg, ctx.repo.label, ctx.mech_dir, False, runner=runner, optimizer_model="agent", adapter=adapter, agent=ctx.host.name) srow = {"resolved": None} diff --git a/tokeneconomizer/plugin/judge.py b/tokeneconomizer/plugin/judge.py index d4bd411..24b841d 100644 --- a/tokeneconomizer/plugin/judge.py +++ b/tokeneconomizer/plugin/judge.py @@ -67,19 +67,35 @@ def run_job(ctx, session_id, job_id): req = ctx.store.read(REQUEST.format(sid=session_id)) or {} if req.get("id") != job_id: # superseded by a newer request return None - runner = (None if sidecall.paused(ctx.host.name, ctx.root) - else sidecall.runner_for(ctx.host.name, ctx.config)) + paused = sidecall.paused(ctx.host.name, ctx.root) + # Each call asks for its own runner: `local_roles` may put the judge on-device and leave + # the redirect on the agent's CLI, or the other way round. A paused host's CLI is skipped, + # but a local runner (which needs no login) still answers. + runner = sidecall.runner_for(ctx.host.name, ctx.config, role="judge") if not paused \ + else sidecall.local_runner("judge", ctx.config) + writer = sidecall.runner_for(ctx.host.name, ctx.config, role="nudge") if not paused \ + else sidecall.local_runner("nudge", ctx.config) verdict, written = None, None if runner is not None: try: verdict = loop_judge.llm_stuck_judge(req.get("digest") or "", runner, model="agent") except Exception: verdict = None + if verdict and getattr(runner, "answered_locally", False): + # The local model's confidences are coarse and high: a stuck verdict counts only at + # its own calibrated floor (`local_judge_confidence`); below it the turn is spared. + verdict["local"] = getattr(runner, "local", "local") + floor = float(getattr(ctx.config, "local_judge_confidence", 0.9) or 0) + if verdict.get("stuck") and float(verdict.get("confidence") or 0) < floor: + verdict["stuck"] = False + verdict["reason"] = ("below local_judge_confidence: " + + str(verdict.get("reason") or ""))[:160] conf = float((verdict or {}).get("confidence") or 0) if verdict and verdict.get("stuck") and conf >= float(req.get("threshold") or 0): try: - written = loop_judge.llm_nudge_writer(req.get("digest") or "", runner, - model="agent") + written = (loop_judge.llm_nudge_writer(req.get("digest") or "", writer, + model="agent") + if writer is not None else None) except Exception: written = None if written and (written.get("text") or "").strip(): diff --git a/tokeneconomizer/plugin/sidecall.py b/tokeneconomizer/plugin/sidecall.py index dc617a2..dacbce9 100644 --- a/tokeneconomizer/plugin/sidecall.py +++ b/tokeneconomizer/plugin/sidecall.py @@ -58,6 +58,7 @@ import re import shutil import subprocess +import sys import tempfile from .. import trace @@ -745,6 +746,104 @@ def copilot_call(prompt, model="", cli="copilot", timeout=TIMEOUT_S): # Hosts the plugin makes no model call for: their CLI cannot run a prompt with every tool off. NO_SIDE_CALLS = frozenset({"antigravity"}) + +# ---- an on-device model for some side calls (`local_model`, `local_roles`) ------------------- +# Apple's Foundation Models CLI, `/usr/bin/fm` on macOS 27 with Apple Intelligence on. It runs a +# ~3B model on the Neural Engine and GPU with a 4,096-token window, needs no login or key, and +# takes the prompt on stdin. `--schema` constrains its decoding to a JSON schema, which is what +# makes the judge usable: in free text it often wraps or contradicts its verdict, under a schema +# it returns the object every time. A call it cannot make returns None, and the caller falls back +# to the agent's own CLI, so turning this on never costs a verdict. + +# Apple's Foundation Models CLI: macOS 27 ships it here, so it is run by this path, never looked +# up on PATH, and only on macOS. +FM_CLI = "/usr/bin/fm" +FM_TIMEOUT_S = 60 +# The judge's verdict, as `loopguard.judge.llm_stuck_judge` parses it. +JUDGE_SCHEMA = { + "title": "Verdict", "type": "object", "additionalProperties": False, + "x-order": ["stuck", "confidence", "reason"], "required": ["stuck", "confidence", "reason"], + "properties": { + "stuck": {"type": "boolean"}, + "confidence": {"type": "number", "description": "0.0 to 1.0"}, + "reason": {"type": "string", "description": "short leakage-safe justification"}}} +ROLE_SCHEMAS = {"judge": JUDGE_SCHEMA} +_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") +_fm_ok = {} + + +def _macos(): + return sys.platform == "darwin" + + +def local_routes(role, config): + """Whether `local_model` sends `role`'s side call on-device: the settings and the platform + alone, no subprocess, so a hook can ask on every tool call. Whether the model answers is + `fm_available`'s to say, in the judge's own job.""" + return (_macos() and getattr(config, "local_model", "off") == "fm" + and role in (getattr(config, "local_roles", None) or ())) + + +def fm_available(cli=FM_CLI): + """Whether `fm` is installed (macOS only) and reports its on-device model available (once + per process).""" + if cli not in _fm_ok: + ok = False + if _macos() and os.access(cli, os.X_OK): + try: + out = subprocess.run([cli, "available"], capture_output=True, encoding="utf-8", + errors="replace", timeout=30, env=_env(), + stdin=subprocess.DEVNULL) + ok = out.returncode == 0 and "unavailable" not in (out.stdout or "").lower() + except (OSError, subprocess.SubprocessError): + ok = False + _fm_ok[cli] = ok + return _fm_ok[cli] + + +def fm_call(prompt, schema=None, cli=FM_CLI, timeout=FM_TIMEOUT_S): + """(text, 0) from the on-device model, or (None, 0) when it gave no usable reply. Greedy + decoding, so the same digest gets the same verdict; `schema` (a JSON schema dict) constrains + the output. A prompt over the model's window fails here and falls back.""" + with tempfile.TemporaryDirectory(prefix="te-fm-", ignore_cleanup_errors=True) as tmp: + cmd = [cli, "respond", "--no-stream", "--greedy"] + if schema: + path = os.path.join(tmp, "schema.json") + with open(path, "w", encoding="utf-8") as fh: + json.dump(schema, fh) + cmd += ["--schema", path] + try: + out = subprocess.run(cmd, input=prompt, capture_output=True, encoding="utf-8", + errors="replace", timeout=timeout, env=_env(), cwd=tmp) + except (OSError, subprocess.SubprocessError) as e: + _log(f"sidecall fm: no reply ({type(e).__name__})\n") + return None, 0 + text = _ANSI_RE.sub("", out.stdout or "").strip() + if out.returncode != 0 or not text or text.startswith("Error:"): + err = _ANSI_RE.sub("", (out.stderr or out.stdout or "").strip())[:200] + _log(f"sidecall fm: no reply (exit {out.returncode}): {err}\n") + return None, 0 + return text, 0 + + +def local_runner(role, config, fallback=None): + """The side-call runner for `role` when `local_model` routes it on-device, else None. On a + failed local call the runner tries `fallback` (the agent's own runner), so a turned-on local + model never loses a verdict; with no fallback (a host in `NO_SIDE_CALLS`) it returns ''.""" + if not local_routes(role, config) or not fm_available(): + return None + schema = ROLE_SCHEMAS.get(role) + + def run(prompt, model="agent"): + text, eff = fm_call(prompt, schema=schema) + run.answered_locally = text is not None + if text is not None: + return text, eff + return fallback(prompt, model) if fallback is not None else ("", 0) + run.local = "fm" + run.answered_locally = False + return run + # The command each agent's side calls run, as the notices name it; the rest run ` -p`. _CALLS = {"codex": "codex exec", "opencode": "opencode run", "cursor": "cursor-agent -p", "hermes": "hermes chat"} @@ -755,13 +854,22 @@ def call_name(host_name): return _CALLS.get(host_name, f"{host_name} -p") -def runner_for(host_name, config): +def runner_for(host_name, config, role=None): """The optimizer callable for this host. Each host's calls go through that host's own CLI, never another agent's. The caller decides whether a call is allowed: micro-skills' proposer when micro-skills is on, the loop guard's judge when the guard is. None for a host in - `NO_SIDE_CALLS`. A call that raises is noted in errors.log (`_noted`).""" + `NO_SIDE_CALLS`. A call that raises is noted in errors.log (`_noted`). + + `role` (`judge`, `nudge`, `proposer`) lets `local_model` run that call on-device instead + (`local_runner`), falling back to the host's CLI when the local call fails; on a host in + `NO_SIDE_CALLS` the local model is the only runner.""" call = _host_call(host_name, config) - return None if call is None else _noted(host_name, call) + host = None if call is None else _noted(host_name, call) + if role is not None: + local = local_runner(role, config, fallback=host) + if local is not None: + return local + return host def _host_call(host_name, config):