From 859bd013f853cf2a6fbf5cd4313853cc0decbd5b Mon Sep 17 00:00:00 2001 From: Rahul Ranjan <11793455+rahul05ranjan@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:12:14 +0000 Subject: [PATCH 1/2] plugin: skip JSON lines that are not objects in the agent-output readers The readers of agent CLI output streams (probe.run_session, probe.inspect_session, sidecall.codex_call, sidecall.copilot_call, toolcheck._init_tools) parsed each line with json.loads and called .get on the result, so a line that parses but is not an object raised AttributeError and killed the whole probe, side call or tool check. inspect_session had no ValueError catch either, so one unparseable hook.end line crashed it too. #150 added the isinstance check to the trace readers; these five plugin-side readers of external process output are the same class. Add the guard everywhere, with regression tests. Fixes #182 Co-authored-by: rahul05ranjan <11793455+rahul05ranjan@users.noreply.github.com> --- tests/test_live_core.py | 49 +++++++++++++++++++++++++++++ tests/test_toolcheck_measure.py | 13 ++++++++ tokeneconomizer/plugin/probe.py | 8 ++++- tokeneconomizer/plugin/sidecall.py | 4 +++ tokeneconomizer/plugin/toolcheck.py | 2 ++ 5 files changed, 75 insertions(+), 1 deletion(-) diff --git a/tests/test_live_core.py b/tests/test_live_core.py index 4dae487..26fffb9 100644 --- a/tests/test_live_core.py +++ b/tests/test_live_core.py @@ -1494,3 +1494,52 @@ def fake_run(cmd, **kw): created.clear() caller("prompt") assert created and all(getattr(td, "_ignore_cleanup_errors", False) for td in created), caller + + +@pytest.mark.real_sidecalls +def test_the_side_calls_skip_json_lines_that_are_not_objects(monkeypatch, tmp_path): + """Anything sharing the CLI's stdout can print a line that parses but is not an object + (null, a bare string, a number, an array); the reply behind it must still be read, not an + AttributeError that loses the whole call (issue #182).""" + import subprocess + from tokeneconomizer.plugin import probe, sidecall + junk = (None, [1, 2], "text", 42) + monkeypatch.setenv("TOKENECONOMIZER_HOME", str(tmp_path / "te")) + monkeypatch.setattr(sidecall, "program", lambda cli: [cli]) + monkeypatch.setenv("COPILOT_HOME", str(tmp_path / "cp")) + copilot_events(str(tmp_path / "cp" / "session-state" / "side1" / "events.jsonl"), []) + + def fake_run(cmd, **kw): + if "codex" in (cmd[0] or ""): + out = "\n".join(json.dumps(e) for e in (*junk, { + "type": "item.completed", + "item": {"type": "agent_message", "text": "hi"}})) + else: + out = "\n".join(json.dumps(e) for e in (*junk, { + "type": "assistant.message", "data": {"content": "- use python -m pytest"}}, + {"type": "result", "sessionId": "side1", "exitCode": 0})) + return subprocess.CompletedProcess(cmd, 0, stdout=out, stderr="") + + monkeypatch.setattr(sidecall.subprocess, "run", fake_run) + monkeypatch.setattr(probe.subprocess, "run", fake_run) + assert sidecall.codex_call("say hi")[0] == "hi" + text, eff = sidecall.copilot_call("propose facts") + assert text == "- use python -m pytest" and eff and eff > 0 + sid, answer, _err = probe.run_session(str(tmp_path), {}, str(tmp_path), "") + assert sid == "side1" and answer == "- use python -m pytest" + + +@pytest.mark.real_sidecalls +def test_inspect_session_skips_hook_lines_that_do_not_parse(tmp_path, monkeypatch): + """The hook.end prefilter lets a corrupt or truncated line through; one such line must not + kill the whole inspection (issue #182).""" + from tokeneconomizer.plugin import probe + home = tmp_path / "cp" + log = home / "session-state" / "s1" / "events.jsonl" + log.parent.mkdir(parents=True) + log.write_text('{"type": "session.start", "data": {"context": {"cwd": "/r"}}}\n' + '"hook.end sessionStart"\n' # parses, not an object + '{broken "hook.end" "sessionStart"\n') # does not parse at all + monkeypatch.setenv("COPILOT_HOME", str(home)) + report = probe.inspect_session("s1") + assert report["session"] == "s1" and report["injected"] is False diff --git a/tests/test_toolcheck_measure.py b/tests/test_toolcheck_measure.py index c6bee44..258d6d6 100644 --- a/tests/test_toolcheck_measure.py +++ b/tests/test_toolcheck_measure.py @@ -99,3 +99,16 @@ def test_the_init_probe_returns_the_tools_when_the_init_event_arrives(monkeypatc started = time.monotonic() assert toolcheck._init_tools([], cli="claude", timeout=30) == (["Bash", "Read"], "2.1") assert time.monotonic() - started < 4 + + +def test_the_init_probe_skips_json_lines_that_are_not_objects(monkeypatch): + """A line the CLI prints that parses but is not an object (null, a bare string, a number, + an array) is skipped, not a crash that loses the whole probe (issue #182).""" + init = {"type": "system", "subtype": "init", "tools": ["Bash"], "claude_code_version": "2.1"} + script = ("import json, time\n" + "for junk in (None, [1, 2], 'text', 42):\n" + " print(json.dumps(junk), flush=True)\n" + f"print({json.dumps(init)!r}, flush=True)\n" + "time.sleep(5)") + _fake_probe(monkeypatch, script) + assert toolcheck._init_tools([], cli="claude", timeout=30) == (["Bash"], "2.1") diff --git a/tokeneconomizer/plugin/probe.py b/tokeneconomizer/plugin/probe.py index 588a061..5b19448 100644 --- a/tokeneconomizer/plugin/probe.py +++ b/tokeneconomizer/plugin/probe.py @@ -128,6 +128,8 @@ def run_session(repo, env, plugin, model, cli="copilot", timeout=600): ev = json.loads(line) except ValueError: continue + if not isinstance(ev, dict): + continue if ev.get("type") == "result": sid = ev.get("sessionId") elif ev.get("type") == "assistant.message": @@ -146,7 +148,11 @@ def inspect_session(sid): tests, tried_pytest = test_commands(cmds) for line in open(log, encoding="utf-8", errors="replace"): if '"hook.end"' in line and '"sessionStart"' in line: - out = (json.loads(line).get("data") or {}).get("output") or {} + try: + ev = json.loads(line) + except ValueError: + continue + out = (ev.get("data") or {}) if isinstance(ev, dict) else {} injected = injected or "fails here" in (out.get("additionalContext") or "") usage = trace.copilot_session_usage(log) first = tests[0] if tests else None diff --git a/tokeneconomizer/plugin/sidecall.py b/tokeneconomizer/plugin/sidecall.py index f30c114..e091215 100644 --- a/tokeneconomizer/plugin/sidecall.py +++ b/tokeneconomizer/plugin/sidecall.py @@ -312,6 +312,8 @@ def codex_call(prompt, model="", cli="codex", timeout=TIMEOUT_S): ev = json.loads(line) except ValueError: continue + if not isinstance(ev, dict): + continue item = ev.get("item") or {} if ev.get("type") == "item.completed" and item.get("type") == "agent_message": text = ev["item"].get("text") or text @@ -723,6 +725,8 @@ def copilot_call(prompt, model="", cli="copilot", timeout=TIMEOUT_S): ev = json.loads(line) except ValueError: continue + if not isinstance(ev, dict): + continue if ev.get("type") == "assistant.message": text = (ev.get("data") or {}).get("content") or text elif ev.get("type") == "result": diff --git a/tokeneconomizer/plugin/toolcheck.py b/tokeneconomizer/plugin/toolcheck.py index cc7f458..a4d0d83 100644 --- a/tokeneconomizer/plugin/toolcheck.py +++ b/tokeneconomizer/plugin/toolcheck.py @@ -78,6 +78,8 @@ def _init_tools(extra_args, cli=None, timeout=60): ev = json.loads(line) except ValueError: continue + if not isinstance(ev, dict): + continue if ev.get("type") == "system" and ev.get("subtype") == "init": tools, version = ev.get("tools"), ev.get("claude_code_version") break From 783633820f973fb79fc3bbab31b45aa4cac0816c Mon Sep 17 00:00:00 2001 From: Aaron Date: Tue, 10 Nov 2026 10:24:16 -0800 Subject: [PATCH 2/2] probe: read the session-start context under data.output again The non-object guard dropped .get('output'), so inspect_session always reported injected: False. Adds a test with a real hook.end line; both test logs end with session.shutdown so settle() does not wait 15 s. --- tests/test_live_core.py | 18 +++++++++++++++++- tokeneconomizer/plugin/probe.py | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/test_live_core.py b/tests/test_live_core.py index 26fffb9..cf67deb 100644 --- a/tests/test_live_core.py +++ b/tests/test_live_core.py @@ -1539,7 +1539,23 @@ def test_inspect_session_skips_hook_lines_that_do_not_parse(tmp_path, monkeypatc log.parent.mkdir(parents=True) log.write_text('{"type": "session.start", "data": {"context": {"cwd": "/r"}}}\n' '"hook.end sessionStart"\n' # parses, not an object - '{broken "hook.end" "sessionStart"\n') # does not parse at all + '{broken "hook.end" "sessionStart"\n' # does not parse at all + '{"type": "session.shutdown", "data": {}}\n') # so `settle` need not wait monkeypatch.setenv("COPILOT_HOME", str(home)) report = probe.inspect_session("s1") assert report["session"] == "s1" and report["injected"] is False + + +def test_inspect_session_reads_the_context_its_session_start_hook_returned(tmp_path, monkeypatch): + """The context sits under the hook.end event's `data.output`; a line beside it that parses + but is not an object changes nothing.""" + from tokeneconomizer.plugin import probe + home = tmp_path / "cp" + log = home / "session-state" / "s1" / "events.jsonl" + log.parent.mkdir(parents=True) + hook_end = {"type": "hook.end", "data": {"hookType": "sessionStart", "output": { + "additionalContext": "In this repository, pytest fails here: use unittest."}}} + log.write_text('"hook.end sessionStart"\n' + json.dumps(hook_end) + "\n" + + json.dumps({"type": "session.shutdown", "data": {}}) + "\n") + monkeypatch.setenv("COPILOT_HOME", str(home)) + assert probe.inspect_session("s1")["injected"] is True diff --git a/tokeneconomizer/plugin/probe.py b/tokeneconomizer/plugin/probe.py index 5b19448..6492689 100644 --- a/tokeneconomizer/plugin/probe.py +++ b/tokeneconomizer/plugin/probe.py @@ -152,7 +152,7 @@ def inspect_session(sid): ev = json.loads(line) except ValueError: continue - out = (ev.get("data") or {}) if isinstance(ev, dict) else {} + out = ((ev.get("data") or {}).get("output") or {}) if isinstance(ev, dict) else {} injected = injected or "fails here" in (out.get("additionalContext") or "") usage = trace.copilot_session_usage(log) first = tests[0] if tests else None