From f3f10fb1f3731e10f310d6c026f327ac8e971592 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Mon, 12 Oct 2026 02:09:34 -0700 Subject: [PATCH 01/12] plugin: show the user, after each turn, where an AxiomCode answer verifiably helped the agent (#1935) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agents finish a task with and without the graph and rarely credit the answer they acted on. A Stop hook (hooks/assist.py) now reads the session's transcript at the end of each turn and prints, under the reply, one line per chain an answer started: ◆ AxiomCode this turn • Helped: `impact _proxy_to_logger` showed _base.py (first seen there); edited it, read _native.py, twisted.py; both picked test files pass Labels, strongest first: Redirected (the work moved from files left behind to one the answer showed), Helped (a file first seen in the answer was edited and the edit is kept, or a test verb's picked tests passed), Useful (it was read, or its picked tests ran). "First seen" means no earlier prompt, tool call, tool result or hook context named the file. Nothing comes from the agent's text: its reply is on screen before a Stop hook runs. Every link is read from tool calls and results and the working-tree diff, and an unchecked link is not printed. A shell line that batches the query with grep or cat credits only rows shaped like an answer's. A shell edit counts only for the file the line writes, and a test run is credited once. Turns where nothing came of a query print nothing. Chains go to .axiomcode/assists.jsonl, which only the hook writes. A chain prints once, and again only when it gets stronger. `axiomcode report` renders them as .axiomcode/assists.html. AXIOMCODE_ASSISTS=off turns it off. tests/assists.py: 32 checks over synthetic transcripts, a case each rule must credit and a near case it must not, plus the hook end to end. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- plugins/axiomcode/hooks/assist.py | 46 ++ plugins/axiomcode/hooks/hooks.json | 9 + plugins/axiomcode/skills/axiomcode/SKILL.md | 7 + .../skills/axiomcode/scripts/ax_assists.py | 596 ++++++++++++++++++ .../skills/axiomcode/scripts/axiomcode | 6 +- skills/axiomcode/SKILL.md | 47 +- tests/assists.py | 245 +++++++ tests/surfaces.py | 5 +- 8 files changed, 954 insertions(+), 7 deletions(-) create mode 100644 plugins/axiomcode/hooks/assist.py create mode 100644 plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py create mode 100644 tests/assists.py diff --git a/plugins/axiomcode/hooks/assist.py b/plugins/axiomcode/hooks/assist.py new file mode 100644 index 000000000..ccdee0263 --- /dev/null +++ b/plugins/axiomcode/hooks/assist.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Stop: when the turn's work stood on an AxiomCode answer, say so under the reply (#1935). + +Agents finish with and without the graph, and they rarely credit the answer they acted on, so the user never sees what +it did. At the end of each turn this reads the session's transcript and prints, for each chain new in that turn, one +line the user can check (skills/axiomcode/scripts/ax_assists.py holds the rules): + + ◆ AxiomCode this turn + • Redirected: the agent was in OrderService.cs, Cart.cs; `path handleOrder PriceCalc` showed PriceCalc.cs … + +Rules it holds itself to: + · NOTHING FROM THE AGENT'S TEXT. The reply has already been shown when this runs; every link is read from the tool + calls and their results, and a link that was not checked is not printed. + · SILENT unless an answer led somewhere: no query, or a query nobody acted on, prints nothing. + · NEVER BLOCKS. It prints a message for the user and exits 0; the agent's turn is over. +AXIOMCODE_ASSISTS=off turns it off. +""" +import json, os, subprocess, sys +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'skills', 'axiomcode', 'scripts')) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _host + +if (os.environ.get('AXIOMCODE_ASSISTS') or '').lower() in ('off', '0', 'false', 'no'): + sys.exit(0) +try: + ev = _host.read() +except Exception: + sys.exit(0) +if _host.CURSOR: # Cursor's stop hook takes no message for the user + sys.exit(0) +tr = ev.get('transcript_path') or '' +if not tr or not os.path.isfile(tr): + sys.exit(0) +cwd = ev.get('cwd') or os.getcwd() +try: + r = subprocess.run(['git', '-C', cwd, 'rev-parse', '--show-toplevel'], capture_output=True, text=True, timeout=5) + root = r.stdout.strip() if r.returncode == 0 and r.stdout.strip() else cwd +except Exception: + root = cwd +try: + import ax_assists + msg = ax_assists.at_stop(tr, os.path.realpath(root), str(ev.get('session_id') or '')) +except Exception: + sys.exit(0) # a receipt is never worth a failed turn +if msg: + print(json.dumps({'systemMessage': msg})) diff --git a/plugins/axiomcode/hooks/hooks.json b/plugins/axiomcode/hooks/hooks.json index a97dd87f0..afe4941a9 100644 --- a/plugins/axiomcode/hooks/hooks.json +++ b/plugins/axiomcode/hooks/hooks.json @@ -72,6 +72,15 @@ "timeout": 5 } ] + }, + { + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/run.js\" assist.py", + "timeout": 15 + } + ] } ], "SessionStart": [ diff --git a/plugins/axiomcode/skills/axiomcode/SKILL.md b/plugins/axiomcode/skills/axiomcode/SKILL.md index 838be50f0..19c7057fd 100644 --- a/plugins/axiomcode/skills/axiomcode/SKILL.md +++ b/plugins/axiomcode/skills/axiomcode/SKILL.md @@ -123,6 +123,13 @@ With them, a call into a dependency resolves to its declaration and a chain is t types (`client.post(...).json()`). Library bodies are not walked, so a dependency calling back into the project is not found this way. +## report + +At the end of each turn the plugin prints, under your reply, where an answer verifiably led your work: a file the +answer showed first that you then read or edited, or the tests it picked that you ran (Redirected / Helped / Useful). +It is read from the session's tool calls, not from what you write, so there is nothing to claim yourself. +`axiomcode report` renders every recorded chain as one page, `.axiomcode/assists.html`. + ## What it cannot see — say so instead of guessing Reflection, string dispatch, event buses; receivers the engine could not type; callbacks invoked by a library; what a diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py new file mode 100644 index 000000000..73170c977 --- /dev/null +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py @@ -0,0 +1,596 @@ +#!/usr/bin/env python3 +"""Where an AxiomCode answer verifiably helped the agent, read from the session record (#1935). + +Agents finish a task with and without AxiomCode, and they rarely say which answer changed what they did, so the user +never sees it. This reads the host's transcript (the agent's tool calls and their results, in order) and finds chains +an answer started: + + cause an AxiomCode answer (context / impact / path / tests) names a file + action the file was FIRST SEEN in that answer (no earlier prompt, tool call, tool result or reply named it), and + the agent then read it, edited it, or ran the tests the answer picked + effect the edit is still in the working tree, and the picked tests that ran passed + +Each chain gets the strongest label it qualifies for: + + Redirected Helped, and before the answer the agent was reading or editing other files in that turn, which it never + touched again and which are not in the final diff: the answer moved the work + Helped a file first seen in the answer was edited and the edit is kept, or the answer's picked tests ran and passed + Useful a file first seen in the answer was read (or its picked tests ran), and nothing was edited from it + +Nothing here comes from what the agent SAYS: its reply has already been shown when a Stop hook runs, so a claim in it +could not be checked before the user read it. Only links that were checked are printed; a chain is cut back to the +last link that holds, and an answer that led to nothing prints nothing. No counterfactual is stated ("grep would have +missed it"): only what happened in the session. + +Chains are appended to /.axiomcode/assists.jsonl, which only this module writes. A chain is printed once, and +again only when it grows a stronger label or a new edited file, so each turn shows what is new in that turn. + + ax_assists.py report [repo] every recorded chain as HTML (/.axiomcode/assists.html) + ax_assists.py replay [root] what each turn of a past session would have printed +""" +import html, json, os, re, subprocess, sys, time + +LEDGER = os.path.join('.axiomcode', 'assists.jsonl') +RANK = {'Useful': 1, 'Helped': 2, 'Redirected': 3} + +# an answer: the MCP tools (current and older names), or the CLI verbs run through a shell +MCP = re.compile(r'^mcp__.*axiomcode.*__(?:axiomcode_)?(context|impact|path|tests|test_impact|changed)$') +CLI = re.compile(r'(?:^|[\s;&|(/])axiomcode(?:-(context|impact|path|test-impact|changed)|\s+(context|impact|path|tests|test-impact|changed))\b([^;&|\n]*)') + +EXT = (r'py|pyi|ts|tsx|mts|cts|js|jsx|mjs|cjs|vue|svelte|astro|java|kt|kts|scala|groovy|cs|cshtml|razor|fs|go|rs|rb|' + r'php|swift|m|mm|c|cc|cpp|cxx|h|hpp|sql|proto|graphql|gql') +PATH = re.compile(r'(?\n]+)') # the command, then its file arguments +EDIT_TOOLS = {'Edit', 'Write', 'MultiEdit', 'NotebookEdit', 'replace', 'write_file'} +READ_TOOLS = {'Read', 'read_file', 'NotebookRead'} +HARNESS = ('', '', '[SYSTEM NOTIFICATION', '"}: the text, with its line breaks""" + if t.startswith('{"result"'): + try: + v = json.loads(t).get('result') + if isinstance(v, str): + return v + except ValueError: + pass + return t + + +def steps(transcript): + """the session in order: prompts, replies and tool calls with their results. Side chains (subagents) are left out: + what a subagent saw, the agent saw only as the subagent's reply.""" + out, calls = [], {} + try: + f = open(transcript, encoding='utf-8', errors='replace') + except OSError: + return out + with f: + for line in f: + try: + e = json.loads(line) + except ValueError: + continue + if not isinstance(e, dict) or e.get('isSidechain'): + continue + if e.get('type') in ('attachment', 'system'): + # a hook's context, a mentioned file, a reminder: the agent saw it, so what it names is not first seen later + t = json.dumps(e.get('attachment') or e.get('content') or '', ensure_ascii=False) + if PATH.search(t): + out.append({'kind': 'note', 'text': t, 'ts': e.get('timestamp') or ''}) + continue + if e.get('type') not in ('user', 'assistant'): + continue + msg = e.get('message') or {} + content = msg.get('content') + ts = e.get('timestamp') or '' + if e['type'] == 'user': + if isinstance(content, str) or (isinstance(content, list) and not any( + isinstance(x, dict) and x.get('type') == 'tool_result' for x in content)): + t = _text(content).strip() + if not t: + continue + real = not e.get('isMeta') and not t.startswith(HARNESS) + out.append({'kind': 'prompt' if real else 'note', 'text': t, 'ts': ts}) + continue + for x in content: + if isinstance(x, dict) and x.get('type') == 'tool_result': + s = calls.get(x.get('tool_use_id')) + if s is not None: + s['result'] = _unwrap(_text(x.get('content'))) + s['error'] = bool(x.get('is_error')) + s['extra'] = json.dumps(e.get('toolUseResult') or '', ensure_ascii=False) + continue + for x in content if isinstance(content, list) else []: + if not isinstance(x, dict): + continue + if x.get('type') == 'text' and x.get('text'): + out.append({'kind': 'reply', 'text': x['text'], 'ts': ts}) + elif x.get('type') == 'tool_use': + s = {'kind': 'call', 'id': x.get('id'), 'name': x.get('name') or '', 'input': x.get('input') or {}, + 'result': '', 'error': False, 'ts': ts} + calls[s['id']] = s + out.append(s) + return out + + +# ---- paths ---------------------------------------------------------------------------------------------------------- + +def norm(p, root): + p = p.strip().strip('\'"`') + if not p: + return '' + if os.path.isabs(p): + rp = os.path.realpath(p) if os.path.exists(p) else os.path.normpath(p) + for r in {root, os.path.realpath(root)}: + if rp == r or rp.startswith(r.rstrip(os.sep) + os.sep): + return os.path.relpath(rp, r).replace(os.sep, '/') + return rp.replace(os.sep, '/') + return os.path.normpath(p).replace(os.sep, '/').lstrip('./') if not p.startswith('..') else os.path.normpath(p) + + +def paths_in(text, root): + return {n for n in (norm(m.group(1), root) for m in PATH.finditer(text or '')) if n} + + +def _seen_text(s): + if s['kind'] != 'call': + return s['text'] + return json.dumps(s['input'], ensure_ascii=False) + '\n' + (s['result'] or '') + '\n' + s.get('extra', '') + + +PLACE = re.compile(r'/|:\d') # an answer names a file as a place: a path, or file:line +SEGMENTS = re.compile(r'\s*(?:;|&&|\|\||\n)\s*') +QUIET = re.compile(r'^(?:cd|echo|printf|export|set|true|:)\b|^$') + + +# a row of an answer: `3. path/file.py:58 [resolved · Owner.name]`, or a further-away place ` path/file.py:56 name [hop 2]`. +# grep -n prints `path:58:text` and cat prints code, so in a shell line that also ran those, only these rows are the answer's. +ROW = re.compile(r'^[ \t]*(?:\d+\.[ \t]+)?((?:/|\.{1,2}/)?(?:[\w@.+-]+/)*[\w@+-][\w@.+-]*\.(?:' + EXT + r')):\d+[ \t]{2,}(?:\S+[ \t]{2,})?\[', re.M) + + +def mixed(s): + """a shell line that runs other commands beside the answer (cat, ls, grep …): their output shares the result, so + nothing in it can be credited to the answer""" + if s['name'] not in ('Bash', 'run_shell_command', 'Shell'): + return False + segs = [g for g in SEGMENTS.split(str(s['input'].get('command') or '')) if g.strip()] + return sum(1 for g in segs if not QUIET.match(g.strip()) and not CLI.search(g)) > 0 + + +def answer_of(s): + """(verb, argument) when this step is an AxiomCode answer, else None""" + if s['kind'] != 'call' or s['error']: + return None + m = MCP.match(s['name']) + if m: + args = ' '.join(str(v) for v in s['input'].values() if isinstance(v, str) and v.strip()) + return m.group(1).replace('_', '-'), args + if s['name'] in ('Bash', 'run_shell_command', 'Shell'): + m = CLI.search(str(s['input'].get('command') or '')) + if m: + verb = m.group(1) or m.group(2) + arg = re.sub(r'\s+(?:/|~)\S*$', '', (m.group(3) or '').strip()) # the trailing repository path + arg = re.sub(r'\s*\d?>&?\S*', '', arg).strip() + while arg.count(')') > arg.count('('): # `(time axiomcode impact X) 2>&1` + arg = arg[:arg.rindex(')')] + arg[arg.rindex(')') + 1:] + arg = arg.strip() + return verb, arg + return None + + +def _source(p, root): + """a source file of the repository, relative to it; '' for anything else (a tool's output file, a file elsewhere)""" + n = norm(p, root) + return n if n and not n.startswith(('/', '..')) and PATH.fullmatch(n) else '' + + +_Q = r'[\'"]([^\'"\n]+)[\'"]' +WRITES = [re.compile(r'open\(\s*' + _Q + r'\s*,\s*[\'"][wa]'), # open('f.py', 'w') + re.compile(r'Path\(\s*' + _Q + r'\s*\)\.write_(?:text|bytes)\('), # Path('f.py').write_text( + re.compile(r'(?:^|[^<>&\d])>>?\s*[\'"]?([^\s\'";&|)]+)'), # > f.py + re.compile(r'\btee\s+(?:-a\s+)?[\'"]?([^\s\'";&|]+)')] # tee f.py +VAR = re.compile(r'\b(\w+)\s*=\s*(?:Path\()?' + _Q) +SED_I = re.compile(r'\b(?:sed\s+(?:-[a-zE]*i\S*|--in-place)|perl\s+-[a-z]*i\w*)\s+([^;&|\n]+)') + + +def written(cmd, root): + """the source files a shell line writes: the target of open(…, 'w'), a redirection, tee or sed -i, by literal path or + through a variable the script assigned (p = 'f.py' … open(p, 'w')); a file the script only mentions is not written""" + out = set() + for r in WRITES: + out |= {m.group(1) for m in r.finditer(cmd)} + vars_ = {m.group(1): m.group(2) for m in VAR.finditer(cmd)} + for m in re.finditer(r'open\(\s*(\w+)\s*,\s*[\'"][wa]|\b(\w+)\.write_(?:text|bytes)\(', cmd): + v = m.group(1) or m.group(2) + if v in vars_: + out.add(vars_[v]) + for m in SED_I.finditer(cmd): + out |= {g.group(1) for g in PATH.finditer(m.group(1))} + return {f for f in (_source(x, root) for x in out) if f} + + +def touched(s, root): + """[(kind, files)] for one tool call, kind 'read' | 'edit' | 'test'. One shell line can be several (write a test, then + run it); a write counts even when the line exits non-zero, since the working tree decides whether it was kept.""" + if s['kind'] != 'call': + return [] + name, inp = s['name'], s['input'] + if name in EDIT_TOOLS: + p = _source(inp.get('file_path') or inp.get('notebook_path') or inp.get('path') or '', root) + return [('edit', {p})] if p and not s['error'] else [] + if name in READ_TOOLS: + p = _source(inp.get('file_path') or inp.get('path') or '', root) + return [('read', {p})] if p else [] + out = [] + if name in ('Bash', 'run_shell_command', 'Shell'): + cmd = str(inp.get('command') or '') + if answer_of(s): + return [] + w = written(cmd, root) + if w: + out.append(('edit', w)) + if TEST_RUN.search(cmd): + out.append(('test', {f for f in paths_in(cmd, root) if not f.startswith(('/', '..'))})) + r = {f for m in SHELL_READ.finditer(cmd) for f in (_source(g.group(1), root) for g in PATH.finditer(m.group(1))) if f} + if r: + out.append(('read', r)) + return out + + +def _same(a, b): + return a == b or a.endswith('/' + b) or b.endswith('/' + a) + + +def _hit(f, files): + return any(_same(f, g) for g in files) + + +# ---- the chains ----------------------------------------------------------------------------------------------------- + +def chains(ss, root, diff=None): + """every chain the session holds. `diff`: the files changed in the working tree now (None: not known, an edit + counts as kept).""" + seen_full, seen_base = set(), set() + first = {} # file -> index of the answer that first showed it + answers = {} # index -> (verb, arg) + turn_of = [] + turn = -1 + for i, s in enumerate(ss): + if s['kind'] == 'prompt': + turn += 1 + turn_of.append(turn) + a = answer_of(s) + if a: + answers[i] = a + for f in paths_in(json.dumps(s['input'], ensure_ascii=False), root): + seen_full.add(f) + seen_base.add(f.rsplit('/', 1)[-1]) + for m in (ROW.finditer(s['result'] or '') if mixed(s) else PATH.finditer(s['result'] or '')): + if not PLACE.search(m.group(0) + (s['result'] or '')[m.end():m.end() + 2]): + continue # a bare file name (a listing, prose) is not a place the answer gave + f = norm(m.group(1), root) + if not f: + continue + b = f.rsplit('/', 1)[-1] + if f in seen_full or f in first or (not GENERIC.match(b) and b in seen_base): + continue + if any(_same(f, g) for g in seen_full): + continue + first[f] = i + for f in paths_in(_seen_text(s), root): + seen_full.add(f) + seen_base.add(f.rsplit('/', 1)[-1]) + + acts = [(i, k, fs) for i, s in enumerate(ss) for k, fs in touched(s, root)] + edits_at = {j for j, k, fs in acts if k == 'edit'} + ids = lambda js: [ss[j]['id'] for j in js] + out = [] + claimed = set() # (test run, test file): a run is credited to the first answer that picked it + for i, (verb, arg) in sorted(answers.items()): + shown = sorted(f for f, j in first.items() if j == i) + named = paths_in(ss[i]['result'], root) + # the tests an answer picked: every test file a test verb names (picking them is its job); for the other verbs + # only a test file first seen in it, since a test the agent already knew of was not picked by the answer + tests_named = {f for f in named if TEST_FILE.search(f) and (verb in ('tests', 'test-impact') or f in shown)} + read, edited, ev_read, ev_edit = [], [], [], [] + for f in shown: + r = [j for j, k, fs in acts if k == 'read' and j > i and _hit(f, fs)] + e = [j for j, k, fs in acts if k == 'edit' and j > i and _hit(f, fs)] + kept = diff is None or _hit(f, diff) + if e and kept: + edited.append(f); ev_edit += e; ev_read += r + elif r or e: + read.append(f); ev_read += r or e + # the answer's picked tests: a later test command that names one of them + ran, passed, ev_test = set(), set(), [] + for j, k, fs in acts: + if k == 'test' and j > i: + hit = {t for t in tests_named if _hit(t, fs) and (j, t) not in claimed} + claimed |= {(j, t) for t in hit} + if hit: + ev_test.append(j) + ran |= hit + if not ss[j]['error']: + passed |= hit + passed_all = bool(ran) and ran == passed + if edited or (ran and passed_all and verb in ('tests', 'test-impact')): + label = 'Helped' + elif read or ran: + label = 'Useful' + else: + continue + frm, ev_from = [], [] + if edited: + # the search the answer ended: files read or edited since the prompt or the last edit before the answer, + # never touched again and not in the final diff + start = i + while start > 0 and ss[start - 1]['kind'] != 'prompt' and (start - 1) not in edits_at: + start -= 1 + before, hits = {}, {} + for j, k, fs in (a for a in acts if start <= a[0] < i): + if k in ('read', 'edit'): + for f in fs: + before.setdefault(f, j) + hits[f] = hits.get(f, 0) + 1 + later = set() + for s2 in ss[i + 1:]: + if s2['kind'] == 'call': + later |= paths_in(json.dumps(s2['input'], ensure_ascii=False), root) + frm = sorted(p for p in before if not _hit(p, shown) and not _hit(p, later) + and (diff is None or not _hit(p, diff)) and not _hit(p, named)) + ev_from = sorted({before[p] for p in frm}) + # a redirect is work that was going somewhere else: two or more files dropped, or one gone back to, and the + # change landing in code (updating the tests an answer listed is help, not a change of direction) + if frm and (len(frm) > 1 or hits[frm[0]] > 1) and any(not TEST_FILE.search(f) for f in edited): + label = 'Redirected' + else: + frm, ev_from = [], [] + out.append({'answer': ss[i]['id'], 'turn': turn_of[i], 'ts': ss[i]['ts'], 'verb': verb, 'arg': arg[:160], + 'label': label, 'shown': len(shown), 'edited': edited, 'read': read, 'from': frm, + 'tests_ran': len(ran), 'tests_passed': len(passed), + # the tool calls each link stands on, by the host's id, so every chain can be checked by hand + 'evidence': {'answer': ss[i]['id'], 'from': ids(ev_from), 'read': ids(sorted(set(ev_read))), + 'edit': ids(sorted(set(ev_edit))), 'test': ids(ev_test)}}) + return out + + +def working_diff(root): + try: + r = subprocess.run(['git', '-C', root, 'status', '--porcelain', '--untracked-files=all'], + capture_output=True, text=True, timeout=10) + except Exception: + return None + if r.returncode: + return None + return {norm(l[3:].split(' -> ')[-1], root) for l in r.stdout.splitlines() if len(l) > 3} + + +# ---- what is printed ------------------------------------------------------------------------------------------------ + +def _q(c): + # what was asked, without flags or paths (--json, an output file), and a shell loop's variable says nothing + arg = ' '.join(w for w in c['arg'].split() if not w.startswith(('-', '/', '$', '~'))) + v = c['verb'] + (f" {arg}" if arg else '') + return f'`{v[:70]}…`' if len(v) > 70 else f'`{v}`' + + +def _names(fs, n=3): + b = [f.rsplit('/', 1)[-1] for f in fs] + return ', '.join(b[:n]) + (f' +{len(b) - n}' if len(b) > n else '') + + +def _picked(c): + n, p = c['tests_ran'], c['tests_passed'] + res = ({1: 'it passes', 2: 'both pass'}.get(n, f'all {n} pass') if p == n else f'{p} of {n} pass') + return f"{_q(c)} picked {n} test file{'s' if n > 1 else ''}; {res}" + + +def line(c): + tests, n = '', c['tests_ran'] + if n: + every = {1: 'the picked test file passes', 2: 'both picked test files pass'}.get(n, f'all {n} picked test files pass') + tests = f'; {every}' if c['tests_passed'] == n else f"; {c['tests_passed']} of {n} picked test files pass" + if c['label'] == 'Redirected': + return (f"Redirected: the agent was in {_names(c['from'])}; {_q(c)} showed {_names(c['edited'])} " + f"(first seen there), and the change landed there{tests}") + if c['label'] == 'Helped': + if c['edited']: + extra = f", read {_names(c['read'])}" if c['read'] else '' + return (f"Helped: {_q(c)} showed {_names(c['edited'])} (first seen there); " + f"edited {'it' if len(c['edited']) == 1 else 'them'}{extra}{tests}") + return f"Helped: {_picked(c)}" + if not c['read']: + return f"Useful: {_picked(c)}" + return f"Useful: {_q(c)} showed {_names(c['read'])} (first seen there); read {'them' if len(c['read']) > 1 else 'it'}{tests}" + + +def _load(root): + p = os.path.join(root, LEDGER) + out = [] + try: + with open(p, encoding='utf-8') as f: + for l in f: + try: + out.append(json.loads(l)) + except ValueError: + pass + except OSError: + pass + return out + + +def new_chains(found, recorded, session): + """the chains not printed before, or grown since: a stronger label, or a file edited that was not""" + best = {} + for r in recorded: + if r.get('session') == session: + best[r['answer']] = r + out = [] + for c in found: + old = best.get(c['answer']) + if old and RANK[c['label']] <= RANK.get(old.get('label'), 0) and set(c['edited']) <= set(old.get('edited') or []): + continue + out.append(c) + return out + + +def receipt(fresh, totals=None): + if not fresh: + return '' + fresh = sorted(fresh, key=lambda c: -RANK[c['label']]) + said = [] + for c in fresh: + l = line(c) + if l not in said: + said.append(l) + lines = ['◆ AxiomCode this turn'] + [f' • {l}' for l in said[:5]] + if len(said) > 5: + lines.append(f' • and {len(said) - 5} more (axiomcode report)') + if totals and sum(totals.values()) > len(fresh): + lines.append(' session: ' + ' · '.join(f'{totals[k]} {k}' for k in ('Redirected', 'Helped', 'Useful') if totals.get(k))) + return '\n'.join(lines) + + +def record(root, session, fresh): + p = os.path.join(root, LEDGER) + try: + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, 'a', encoding='utf-8') as f: + for c in fresh: + f.write(json.dumps(dict(c, session=session, at=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()))) + '\n') + except OSError: + pass + + +def totals(recorded, session): + best = {} + for r in recorded: + if r.get('session') == session: + if RANK.get(r['label'], 0) >= RANK.get(best.get(r['answer'], ''), 0): + best[r['answer']] = r['label'] + t = {} + for l in best.values(): + t[l] = t.get(l, 0) + 1 + return t + + +def at_stop(transcript, root, session): + """the receipt for the turn that just ended, recorded; '' when there is nothing new""" + found = chains(steps(transcript), root, working_diff(root)) + recorded = _load(root) + fresh = new_chains(found, recorded, session) + if not fresh: + return '' + record(root, session, fresh) + return receipt(fresh, totals(recorded + [dict(c, session=session) for c in fresh], session)) + + +# ---- the report ----------------------------------------------------------------------------------------------------- + +CSS = """ +:root{--bg:#fbfbfa;--fg:#1d1d1b;--mut:#6b6a66;--card:#fff;--line:#e6e4df;--r:#7a3df0;--h:#127a55;--u:#8a6d1f} +@media (prefers-color-scheme:dark){:root{--bg:#141413;--fg:#ecebe6;--mut:#a3a29c;--card:#1d1d1b;--line:#33322f;--r:#b394ff;--h:#4fd1a1;--u:#e0c063}} +body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 -apple-system,system-ui,sans-serif} +main{max-width:860px;margin:0 auto;padding:32px 16px} +h1{font-size:22px;margin:0 0 4px}.sub{color:var(--mut);margin:0 0 24px} +.tiles{display:flex;gap:12px;flex-wrap:wrap;margin-bottom:28px} +.tile{flex:1;min-width:140px;background:var(--card);border:1px solid var(--line);border-radius:10px;padding:14px} +.tile b{display:block;font-size:28px}.tile span{color:var(--mut);font-size:13px} +.s{margin:28px 0 8px;color:var(--mut);font-size:13px;text-transform:uppercase;letter-spacing:.04em} +.c{background:var(--card);border:1px solid var(--line);border-radius:10px;padding:12px 14px;margin:8px 0} +.l{font-weight:600;font-size:13px}.Redirected{color:var(--r)}.Helped{color:var(--h)}.Useful{color:var(--u)} +.c p{margin:4px 0 0}.c small{color:var(--mut)}code{font:13px ui-monospace,Menlo,monospace} +""" + + +def report(root): + rec = _load(root) + best = {} + for r in rec: + k = (r.get('session'), r['answer']) + if k not in best or RANK.get(r['label'], 0) >= RANK.get(best[k]['label'], 0): + best[k] = r + rows = sorted(best.values(), key=lambda r: r.get('at') or '', reverse=True) + t = {} + for r in rows: + t[r['label']] = t.get(r['label'], 0) + 1 + sessions = {} + for r in rows: + sessions.setdefault(r.get('session') or '?', []).append(r) + e = html.escape + body = [f"

Where AxiomCode helped

{e(os.path.basename(os.path.abspath(root)))} · " + f"{len(rows)} verified chain(s) over {len(sessions)} session(s). Every chain is read from the session record: " + "an answer showed a file first, and the agent then read or changed it.

"] + for k in ('Redirected', 'Helped', 'Useful'): + body.append(f"
{t.get(k, 0)}{k}
") + body.append('
') + for s, rs in sessions.items(): + body.append(f"
session {e(s[:8])} · {e((rs[-1].get('at') or '')[:16].replace('T', ' '))}
") + for r in rs: + body.append(f"
{r['label']}" + f"

{re.sub(r'`([^`]+)`', r'\1', e(line(r).split(': ', 1)[1]))}

" + + (f"edited: {e(', '.join(r['edited']))}
" if r.get('edited') else '') + + (f"read: {e(', '.join(r['read']))}
" if r.get('read') else '') + + (f"left: {e(', '.join(r['from']))}" if r.get('from') else '') + '
') + page = (f"" + f"AxiomCode assists
{''.join(body)}
") + out = os.path.join(root, '.axiomcode', 'assists.html') + os.makedirs(os.path.dirname(out), exist_ok=True) + with open(out, 'w', encoding='utf-8') as f: + f.write(page) + return out, len(rows) + + +def replay(transcript, root): + """what each turn of a past session would have printed (the working tree then is not known: an edit counts as kept)""" + ss = steps(transcript) + ends = [i for i, s in enumerate(ss) if s['kind'] == 'prompt'][1:] + [len(ss)] + recorded, n = [], 0 + for e in ends: + n += 1 + fresh = new_chains(chains(ss[:e], root, None), recorded, 'replay') + recorded += [dict(c, session='replay') for c in fresh] + r = receipt(fresh, totals(recorded, 'replay')) + if r: + print(f'--- turn {n}\n{r}') + return recorded + + +if __name__ == '__main__': + a = sys.argv[1:] + if a[:1] == ['report']: + p, n = report(os.path.abspath(a[1] if len(a) > 1 else '.')) + print(f'{n} chain(s) → {p}') + elif a[:1] == ['replay'] and len(a) > 1: + rec = replay(a[1], os.path.abspath(a[2] if len(a) > 2 else '.')) + t = {} + for r in rec: + t[r['label']] = t.get(r['label'], 0) + 1 + print('total:', json.dumps(t)) + else: + print(__doc__.strip().splitlines()[-2:][0]); print(__doc__.strip().splitlines()[-1]); sys.exit(2) diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode index 6ab4eb364..083855653 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode @@ -17,6 +17,9 @@ # labelled [asserted]. Alone, lists every link and whether the graph took it; `link -` removes one. # axiomcode index [] [--lang [,…]] [--src ] [--library [,…]] # build the graph (the first query builds it too). defaults to the current directory. +# axiomcode report [] +# where an answer verifiably helped: every Redirected / Helped / Useful chain the end-of-turn receipts recorded, +# as one HTML page (.axiomcode/assists.html). # # Names are written as in the code: Owner.method, function, Type, or file.py:123. `axiomcode help ` for one verb. @@ -114,7 +117,7 @@ public_verbs(){ helptext | sed -n 's/^ axiomcode \([a-z][a-z-]*\).*/\1/p'; } # leading comment block. The verb documents itself once, where it is implemented. verbhelp(){ # a verb of the small surface is explained by its own entry in the help above: what it answers, no options - case "$1" in impact|path|tests|link) + case "$1" in impact|path|tests|link|report) helptext | awk -v v="$1" '$0 ~ "^ axiomcode "v"( |$)" {on=1; print; next} on && /^ axiomcode / {exit} on && /^$/ {exit} on {print}' return 0 ;; esac @@ -234,6 +237,7 @@ case "$cmd" in changed) exec python3 ${Q:+"$Q" "$FR"} "$H/axiomcode-changed" ${ARGS[@]+"${ARGS[@]}"} ;; test-impact|tests) exec ${G[@]+"${G[@]}"} python3 ${Q:+"$Q" "$FR"} "$H/axiomcode-test-impact" ${ARGS[@]+"${ARGS[@]}"} ;; link) exec python3 "$H/axiomcode-link" ${ARGS[@]+"${ARGS[@]}"} ;; + report) exec python3 "$H/ax_assists.py" report ${ARGS[@]+"${ARGS[@]}"} ;; --verbs) verbs ;; # the dispatch table, internal verbs included; tests/surfaces.py audits it ""|-h|--help|help) if [ ${#ARGS[@]} -gt 0 ]; then verbhelp "${ARGS[0]}"; else helptext; fi ;; *) echo "axiomcode: unknown subcommand '$cmd' — the entry point has: $(public_verbs | tr '\n' ' ')" >&2 diff --git a/skills/axiomcode/SKILL.md b/skills/axiomcode/SKILL.md index ba96f81fe..f512635b6 100644 --- a/skills/axiomcode/SKILL.md +++ b/skills/axiomcode/SKILL.md @@ -1,23 +1,25 @@ --- name: axiomcode description: >- - Use for any why, what or where question about code — how a codebase works, where something lives, who calls it, what a change to it breaks, which tests cover an edit. Also use when resolving an issue or bug report, which names a symptom rather than a file. Examples: "How does X work?", "Where do I change Y?", "What calls this?", "What breaks if I change Z?", "Which tests do I run?", "Fix this issue". Search with grep as usual; ask the graph for what grep cannot know — which declaration a call reaches and the callers that never spell the name. Answers come from a resolved call graph, so they include callers that never spell the name — through an interface, an override, a callback, dependency injection or a config key — and every place comes with the code of the function it sits in. Call the MCP tools directly, no need to load this skill first: impact(name) for who calls it and what a change reaches (with no name: your uncommitted edits), path(start, end) for how A reaches B, tests() for the tests your edits reach. Only when those tools are not in your list, the same from the shell: `axiomcode impact `, `axiomcode path `, `axiomcode tests`. Java, TypeScript, Python, JavaScript, C#. + Use for any why, what or where question about code — how a codebase works, where something lives, who calls it, what a change to it breaks, which tests cover an edit. Also use when resolving an issue or bug report, which names a symptom rather than a file. Examples: "How does X work?", "Where do I change Y?", "What calls this?", "What breaks if I change Z?", "Which tests do I run?", "Fix this issue". Search with grep as usual; ask the graph for what grep cannot know — which declaration a call reaches and the callers that never spell the name. Answers come from a resolved call graph, so they include callers that never spell the name — through an interface, an override, a callback, dependency injection or a config key — and every place comes with the code of the function it sits in. Call the MCP tools directly, no need to load this skill first: impact(name) for who calls it and what a change reaches (with no name: your uncommitted edits), path(start, end) for how A reaches B, tests() for the tests your edits reach, context(task, source=True) for how something works as a step-by-step call flow with each step's code. Only when those tools are not in your list, the same from the shell: `axiomcode impact `, `axiomcode path `, `axiomcode tests`, `axiomcode context "" --source`. Java, TypeScript, Python, JavaScript, C#. --- # axiomcode Search with grep as usual; the graph answers what grep cannot. Use the MCP tools when they are in your list (in Claude Code -`mcp__plugin_axiomcode_axiomcode__impact`, `__path`, `__tests`); otherwise run +`mcp__plugin_axiomcode_axiomcode__impact`, `__path`, `__tests`, `__context`); otherwise run `/../../plugins/axiomcode/skills/axiomcode/scripts/axiomcode ` from the repository root. Same answer either way. | the question | MCP tool | shell | |---|---|---| | where is the code for this task? | your own search (grep), then bring the name here | — | | who calls X, what does changing it reach, which tests? | `impact(name)` | `axiomcode impact ` | +| what is the value of constant X, and who reads it? | `impact(name)` | `axiomcode impact ` | | what do my uncommitted edits reach? | `impact()` | `axiomcode impact` | | how does A reach B? | `path(start, end)` | `axiomcode path ` | | which tests do my edits need, and how do I run them? | `tests()` | `axiomcode tests` | | an answer lists an unresolved call I can see the target of | `link(site, target)` | `axiomcode link ` | +| how does this work, start to finish? | `context(task, source=True)` | `axiomcode context "" --source` | Names are written as in the code: `Owner.method`, `function`, `Type`, or `file.py:123` for the declaration at that line. There is no setup step: the first question builds the graph, and it refreshes itself after every edit. @@ -45,7 +47,8 @@ from an empty answer. With a name: who calls it, what depends on it further out, and the tests that exercise it. Example: `impact(name="PriceService.total")`. With no name: the first line is `your edits:` (each declaration you changed and -how), then the same answer for all of them. +how), then the same answer for all of them. A constant answers with its value — `change: const MAX_ITEMS = 5` — +so a limit, a default or a threshold is read off the first line rather than from the file. ## path @@ -87,10 +90,46 @@ identifier as written in the code. ## index -`axiomcode index` builds the graph explicitly; `--lang`, `--src` and `--library` narrow it. Never re-run it on an +`axiomcode index` builds the graph explicitly; `--lang` and `--src` narrow it. Never re-run it on an existing graph: the graph rebuilds itself after edits, and an answer given before that finishes says so on a `graph refresh:` line. +By default the graph is built without the project's dependencies: a call into one is unresolved or named only as an +external boundary. When better coverage is needed, compile the dependencies and enable them with `--library`, given +once on `index` and kept by every rebuild after it: + + axiomcode index --library auto # the dependencies the project imports + axiomcode index --library .venv/lib/python3.12/site-packages/requests,/deps/ir/jdk # or name them + +`auto` finds what the project depends on, in every language: a Python package its source imports, in its virtual +environment (`.venv`, `venv`, `$VIRTUAL_ENV`); a JavaScript / TypeScript package it imports, under `node_modules`; a +Java dependency `pom.xml` or `build.gradle` declares, through its `-sources.jar` in the Maven repository or Gradle's +cache, or its class jar decompiled when it ships no sources (needs Vineflower: `mvn dependency:get +-Dartifact=org.vineflower:vineflower:1.10.1`); a NuGet package a `.csproj` references, decompiled from its assembly +(needs `dotnet tool install -g ilspycmd`). Whatever it cannot find (a package never restored, a jar without sources) +it names, with the command that fetches it. A named entry, comma-separated with no spaces, is a dependency's source +directory or a library IR (a directory of the parser's CSV tables, as `axiomcode parser --library` +writes it). Every library is compiled once into `~/.cache/axiomcode/libir/` and reused until its files change; one no +build has used for 30 days is removed. Entries are relative to the repository or absolute; write `~` out, since one +after a comma is not expanded. `AXIOMCODE_LIBRARY` takes the same list. + +The choice is kept with the graph. Every rebuild after it stages the libraries again (`auto` discovers them +again, so a new dependency is picked up), and with libraries on, a dependency change alone (a manifest or lockfile +edit, `pip install -U`, `npm install`, `dotnet restore`, a Maven or Gradle fetch of a dependency that was missing) +also marks the graph stale and rebuilds it. A graph built +without `--library` never gains libraries by itself; `AXIOMCODE_REINDEX=1 axiomcode index` turns them off again. + +With them, a call into a dependency resolves to its declaration and a chain is typed through its declared return +types (`client.post(...).json()`). Library bodies are not walked, so a dependency calling back into the project is +not found this way. + +## report + +At the end of each turn the plugin prints, under your reply, where an answer verifiably led your work: a file the +answer showed first that you then read or edited, or the tests it picked that you ran (Redirected / Helped / Useful). +It is read from the session's tool calls, not from what you write, so there is nothing to claim yourself. +`axiomcode report` renders every recorded chain as one page, `.axiomcode/assists.html`. + ## What it cannot see — say so instead of guessing Reflection, string dispatch, event buses; receivers the engine could not type; callbacks invoked by a library; what a diff --git a/tests/assists.py b/tests/assists.py new file mode 100644 index 000000000..aeabbe784 --- /dev/null +++ b/tests/assists.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +"""tests/assists.py — the Stop hook's receipt (#1935) credits an answer only for what the session shows it led to. + +Each check writes a small transcript in the host's shape (prompts, tool calls, their results) into a scratch git +repository, and asks the rules what they find. A rule is checked from both sides: the case it must credit, and a near +case that differs only where the rule reads and must not be credited. No engine and no model are needed. + + python3 tests/assists.py +""" +import json, os, subprocess, sys, tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +HOOKS = os.path.join(ROOT, 'plugins', 'axiomcode', 'hooks') +sys.path.insert(0, os.path.join(ROOT, 'plugins', 'axiomcode', 'skills', 'axiomcode', 'scripts')) +import ax_assists as A + +fails = [] +def check(why, cond, detail=''): + print(('ok ' if cond else 'FAIL ') + why + (f'\n {detail}' if not cond and detail else '')) + if not cond: + fails.append(why) + + +MCP = 'mcp__plugin_axiomcode_axiomcode__impact' +IMPACT_A = '1. src/pkg/alpha.py:10 [resolved · Alpha.run]\n ```python\n → 10 self.beta()\n ```\nverified: ✓' +TESTS_A = '1. tests/test_alpha.py:5 [test]\nrun them: pytest tests/test_alpha.py' + + +class T: + """a transcript under construction""" + def __init__(self, root): + self.root, self.lines, self.n = root, [], 0 + + def prompt(self, text): + self.lines.append({'type': 'user', 'message': {'role': 'user', 'content': text}}) + return self + + def call(self, name, inp, result='', error=False, side=False): + self.n += 1 + tid = f'toolu_{self.n:03d}' + self.lines.append({'type': 'assistant', 'isSidechain': side, + 'message': {'role': 'assistant', 'content': [{'type': 'tool_use', 'id': tid, 'name': name, 'input': inp}]}}) + self.lines.append({'type': 'user', 'isSidechain': side, + 'message': {'role': 'user', 'content': [{'type': 'tool_result', 'tool_use_id': tid, + 'content': result, 'is_error': error}]}}) + return self + + def read(self, rel): + return self.call('Read', {'file_path': os.path.join(self.root, rel)}, 'contents') + + def edit(self, rel): + return self.call('Edit', {'file_path': os.path.join(self.root, rel), 'old_string': 'a', 'new_string': 'b'}, 'ok') + + def save(self): + p = os.path.join(self.root, '..', f'transcript-{id(self)}.jsonl') + with open(p, 'w') as f: + for l in self.lines: + f.write(json.dumps(l) + '\n') + return p + + +def repo(): + d = os.path.realpath(tempfile.mkdtemp(prefix='ax-assists-')) + r = os.path.join(d, 'repo') + os.makedirs(os.path.join(r, 'src', 'pkg')); os.makedirs(os.path.join(r, 'tests')) + for f in ('src/pkg/alpha.py', 'src/pkg/beta.py', 'src/pkg/gamma.py', 'tests/test_alpha.py'): + open(os.path.join(r, f), 'w').write('x = 1\n') + for c in (['init', '-q'], ['add', '-A'], ['-c', 'user.email=t@t', '-c', 'user.name=t', 'commit', '-qm', 'base']): + subprocess.run(['git', '-C', r] + c, check=True) + return r + + +def change(r, rel): + open(os.path.join(r, rel), 'a').write('y = 2\n') + + +def found(t, r, diff=True): + return A.chains(A.steps(t.save()), r, A.working_diff(r) if diff else None) + + +# --- Helped: first seen in the answer, then read and edited, and the edit kept +r = repo(); change(r, 'src/pkg/alpha.py') +t = T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, IMPACT_A).read('src/pkg/alpha.py').edit('src/pkg/alpha.py') +c = found(t, r) +check('an answer that showed a file the agent then edited is Helped', [x['label'] for x in c] == ['Helped'], c) +check('the chain carries the tool calls it stands on', c and c[0]['evidence']['edit'] and c[0]['evidence']['answer'] == 'toolu_001', c) + +# --- an MCP answer wrapped as {"result": …}, the file at the start of a line +r = repo(); change(r, 'src/pkg/alpha.py') +t = (T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, json.dumps({'result': 'further away:\nsrc/pkg/alpha.py:10 run [hop 2]'})) + .edit('src/pkg/alpha.py')) +check('an MCP answer wrapped in {"result": …} is read with its line breaks', [x['label'] for x in found(t, r)] == ['Helped'], found(t, r)) + +# --- the same, but the prompt already named the file: not first seen, no credit +r = repo(); change(r, 'src/pkg/alpha.py') +t = T(r).prompt('fix the bug in src/pkg/alpha.py').call(MCP, {'name': 'Alpha.run'}, IMPACT_A).edit('src/pkg/alpha.py') +check('a file the prompt named is not credited to the answer', found(t, r) == [], found(t, r)) + +# --- an earlier tool result named it (a grep hit): no credit +r = repo(); change(r, 'src/pkg/alpha.py') +t = (T(r).prompt('fix the bug in run').call('Grep', {'pattern': 'run'}, 'src/pkg/alpha.py:10: def run') + .call(MCP, {'name': 'Alpha.run'}, IMPACT_A).edit('src/pkg/alpha.py')) +check('a file an earlier grep showed is not credited', found(t, r) == [], found(t, r)) + +# --- the edit was reverted: what is left is a read, so Useful +r = repo() +t = T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, IMPACT_A).read('src/pkg/alpha.py').edit('src/pkg/alpha.py') +check('an edit no longer in the working tree is not Helped (Useful: it was read)', [x['label'] for x in found(t, r)] == ['Useful'], found(t, r)) + +# --- an answer nobody acted on prints nothing +r = repo() +t = T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, IMPACT_A) +check('an answer nothing came from is not a chain', found(t, r) == [], found(t, r)) + +# --- Redirected: two files dropped, the change lands in the one the answer showed +r = repo(); change(r, 'src/pkg/alpha.py') +t = (T(r).prompt('fix the bug in run').read('src/pkg/beta.py').read('src/pkg/gamma.py') + .call(MCP, {'name': 'Alpha.run'}, IMPACT_A).read('src/pkg/alpha.py').edit('src/pkg/alpha.py')) +c = found(t, r) +check('dropped files plus a change landing in the answer\'s file is Redirected', + [x['label'] for x in c] == ['Redirected'] and c[0]['from'] == ['src/pkg/beta.py', 'src/pkg/gamma.py'], c) + +# --- control: one of those files is touched again after the answer, and one file alone was looked at once +r = repo(); change(r, 'src/pkg/alpha.py') +t = (T(r).prompt('fix the bug in run').read('src/pkg/beta.py').read('src/pkg/gamma.py') + .call(MCP, {'name': 'Alpha.run'}, IMPACT_A).edit('src/pkg/alpha.py').read('src/pkg/gamma.py')) +c = found(t, r) +check('a file gone back to after the answer was not left behind (one dropped read is not a redirect)', + [x['label'] for x in c] == ['Helped'], c) + +# --- control: the earlier file is in the final diff, so the work did not move away from it +r = repo(); change(r, 'src/pkg/alpha.py'); change(r, 'src/pkg/beta.py') +t = (T(r).prompt('fix the bug in run').read('src/pkg/beta.py').read('src/pkg/beta.py') + .call(MCP, {'name': 'Alpha.run'}, IMPACT_A).edit('src/pkg/alpha.py')) +check('a file still in the diff was not left behind', [x['label'] for x in found(t, r)] == ['Helped'], found(t, r)) + +# --- picked tests: ran and passed is Helped; ran and failed is Useful +r = repo() +t = (T(r).prompt('which tests cover my change').call('mcp__plugin_axiomcode_axiomcode__tests', {}, TESTS_A) + .call('Bash', {'command': 'pytest tests/test_alpha.py -q'}, '1 passed')) +c = found(t, r) +check('picked tests that ran and passed are Helped', [x['label'] for x in c] == ['Helped'] and c[0]['tests_passed'] == 1, c) +r = repo() +t = (T(r).prompt('which tests cover my change').call('mcp__plugin_axiomcode_axiomcode__tests', {}, TESTS_A) + .call('Bash', {'command': 'pytest tests/test_alpha.py -q'}, 'Exit code 1\n1 failed', error=True)) +check('picked tests that failed are only Useful', [x['label'] for x in found(t, r)] == ['Useful'], found(t, r)) + +# --- a test file the agent already knew of is not picked by an impact answer; a tests answer picks what it names +r = repo() +IMPACT_T = IMPACT_A + '\ntests:\n tests/test_alpha.py:5 test_run [test]' +t = (T(r).prompt('fix the bug in run').read('tests/test_alpha.py').call(MCP, {'name': 'Alpha.run'}, IMPACT_T) + .call('Bash', {'command': 'pytest tests/test_alpha.py -q'}, '1 passed')) +check('an impact answer is not credited for running a test file the agent had already opened', found(t, r, diff=False) == [], + found(t, r, diff=False)) +r = repo() +t = (T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, IMPACT_T) + .call('Bash', {'command': 'pytest tests/test_alpha.py -q'}, '1 passed')) +check('…but is when the answer showed it first', [x['label'] for x in found(t, r, diff=False)] == ['Useful'], found(t, r, diff=False)) + +# --- one shell line that writes a file and runs the tests, failing, still wrote the file +r = repo(); change(r, 'src/pkg/alpha.py') +t = (T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, IMPACT_A) + .call('Bash', {'command': "python3 - <<'EOF'\np='src/pkg/alpha.py'\ns=open(p).read()\nopen(p,'w').write(s)\nEOF\npytest -q"}, + 'Exit code 1', error=True)) +check('a write in a shell line that then fails its tests is still an edit', [x['label'] for x in found(t, r)] == ['Helped'], found(t, r)) +r = repo(); change(r, 'src/pkg/alpha.py') +t = (T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, IMPACT_A) + .call('Bash', {'command': "cat >> notes.jsonl <<'EOF'\n{\"file\": \"src/pkg/alpha.py\"}\nEOF"}, '')) +check('a shell line that only mentions a file (a log it appends to) did not edit it', found(t, r) == [], found(t, r)) + +# --- one test run is credited once, to the first answer that picked it +r = repo() +t = (T(r).prompt('which tests cover my change').call('mcp__plugin_axiomcode_axiomcode__tests', {}, TESTS_A) + .call('Bash', {'command': 'axiomcode tests --json /tmp/out.json'}, TESTS_A) + .call('Bash', {'command': 'pytest tests/test_alpha.py -q'}, '1 passed')) +c = found(t, r) +check('a test run two answers picked is credited once', len(c) == 1, c) +check('a flag or an output path is not shown as what was asked', A._q(dict(verb='tests', arg='--json /tmp/out.json')) == '`tests`', + A._q(dict(verb='tests', arg='--json /tmp/out.json'))) + +# --- the CLI through a shell counts as an answer; a shell line that also lists files does not +r = repo(); change(r, 'src/pkg/alpha.py') +t = (T(r).prompt('fix the bug in run').call('Bash', {'command': 'axiomcode impact Alpha.run 2>&1 | head -40'}, IMPACT_A) + .edit('src/pkg/alpha.py')) +c = found(t, r) +check('`axiomcode impact` in a shell is an answer, its argument without the redirection', + [x['label'] for x in c] == ['Helped'] and c[0]['arg'] == 'Alpha.run', c) +r = repo(); change(r, 'src/pkg/alpha.py') +t = (T(r).prompt('fix the bug in run').call('Bash', {'command': 'axiomcode impact Alpha.run; ls src/pkg'}, + 'no callers found\nsrc/pkg/alpha.py\nsrc/pkg/beta.py') + .edit('src/pkg/alpha.py')) +check('a shell line that also lists files credits nothing from the listing', found(t, r) == [], found(t, r)) +r = repo(); change(r, 'src/pkg/gamma.py') +t = (T(r).prompt('fix the bug in run').call('Bash', {'command': 'axiomcode impact Alpha.run; grep -rn run src'}, + 'src/pkg/gamma.py:3:run()\n' + IMPACT_A).edit('src/pkg/gamma.py')) +check('in a shell line that also greps, a file only grep printed is not credited', found(t, r) == [], found(t, r)) +r = repo(); change(r, 'src/pkg/alpha.py') +t = (T(r).prompt('fix the bug in run').call('Bash', {'command': 'axiomcode impact Alpha.run; grep -rn zzz src'}, + 'no match\n' + IMPACT_A).edit('src/pkg/alpha.py')) +check('…but a row of the answer in that line is', [x['label'] for x in found(t, r)] == ['Helped'], found(t, r)) + +# --- a bare file name (a listing, prose) is not a place the answer gave +r = repo(); change(r, 'src/pkg/alpha.py') +t = (T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, 'Alpha.run is defined in alpha.py') + .edit('src/pkg/alpha.py')) +check('a bare file name in an answer is not credited', found(t, r) == [], found(t, r)) + +# --- what a subagent saw is not the session's +r = repo(); change(r, 'src/pkg/alpha.py') +t = (T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, IMPACT_A, side=True).edit('src/pkg/alpha.py')) +check('an answer inside a subagent is not credited to the session', found(t, r) == [], found(t, r)) + +# --- the ledger: printed once; printed again only when it grows +r = repo(); change(r, 'src/pkg/alpha.py') +t = T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, IMPACT_A).read('src/pkg/alpha.py') +p = t.save() +subprocess.run(['git', '-C', r, 'checkout', '-q', '--', '.']) +first = A.at_stop(p, r, 's1') +again = A.at_stop(p, r, 's1') +change(r, 'src/pkg/alpha.py'); t.edit('src/pkg/alpha.py'); p = t.save() +grown = A.at_stop(p, r, 's1') +check('a chain prints once (Useful)', 'Useful' in first and again == '', (first, again)) +check('it prints again when it grows a stronger label', 'Helped' in grown, grown) +check('the ledger keeps both', len(A._load(r)) == 2, A._load(r)) +out, n = A.report(r) +page = open(out).read() +check('the report renders the session\'s chains', n == 1 and 'Helped' in page and 'impact Alpha.run' in page, (out, n)) + +# --- the hook end to end: a Stop event in, a message for the user out; switched off, nothing +r = repo(); change(r, 'src/pkg/alpha.py') +t = T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, IMPACT_A).edit('src/pkg/alpha.py') +ev = {'hook_event_name': 'Stop', 'session_id': 's2', 'transcript_path': t.save(), 'cwd': r, 'stop_hook_active': False} +h = subprocess.run([sys.executable, os.path.join(HOOKS, 'assist.py')], input=json.dumps(ev), capture_output=True, text=True) +msg = json.loads(h.stdout).get('systemMessage', '') if h.stdout.strip() else '' +check('the Stop hook prints the receipt as a message for the user', h.returncode == 0 and msg.startswith('◆ AxiomCode') and 'Helped' in msg, + (h.returncode, h.stdout, h.stderr)) +ev['session_id'] = 's3' +h = subprocess.run([sys.executable, os.path.join(HOOKS, 'assist.py')], input=json.dumps(ev), capture_output=True, text=True, + env=dict(os.environ, AXIOMCODE_ASSISTS='off')) +check('AXIOMCODE_ASSISTS=off prints nothing', h.returncode == 0 and not h.stdout.strip(), h.stdout) +h = subprocess.run([sys.executable, os.path.join(HOOKS, 'assist.py')], input='{"hook_event_name":"Stop"}', capture_output=True, text=True) +check('a Stop event with no transcript is silent and never fails', h.returncode == 0 and not h.stdout.strip(), (h.returncode, h.stdout)) + +print(f'\n{len(fails)} failed' if fails else '\nall passed') +sys.exit(1 if fails else 0) diff --git a/tests/surfaces.py b/tests/surfaces.py index c4eb20ab8..9674cea67 100644 --- a/tests/surfaces.py +++ b/tests/surfaces.py @@ -29,8 +29,9 @@ MCP = os.path.join(PLUG, 'mcp', 'server.py') CLI = os.path.join(ROOT, 'bin', 'axiomcode') # the command an install puts on $PATH -PUBLIC = ['index', 'impact', 'path', 'tests', 'context', 'link'] -NO_MCP = {'index': 'setup, not a question: the first query through the MCP server builds the graph itself'} +PUBLIC = ['index', 'impact', 'path', 'tests', 'context', 'link', 'report'] +NO_MCP = {'index': 'setup, not a question: the first query through the MCP server builds the graph itself', + 'report': 'a page for the person, not a question for the agent: the end-of-turn receipts already reach both'} # dispatched, not advertised: verb -> why INTERNAL = { 'build': 'the old name of index', From 72b2678eee948931454c72eb44bef8477f1ccf10 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Mon, 12 Oct 2026 10:37:36 -0700 Subject: [PATCH 02/12] plugin: the receipt counts the answers that led the work and names the files and tests behind each (#1935) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each chain now reads cause → action → effect with the concrete files edited or read and the picked test files that passed. The header counts the answers that led the agent's work; answers that led nowhere are never counted or shown, and a label no chain earned gets no tile in the report. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- plugins/axiomcode/hooks/assist.py | 7 +- .../skills/axiomcode/scripts/ax_assists.py | 183 +++++++++++++----- tests/assists.py | 14 ++ 3 files changed, 148 insertions(+), 56 deletions(-) diff --git a/plugins/axiomcode/hooks/assist.py b/plugins/axiomcode/hooks/assist.py index ccdee0263..b02ada2fa 100644 --- a/plugins/axiomcode/hooks/assist.py +++ b/plugins/axiomcode/hooks/assist.py @@ -5,13 +5,14 @@ it did. At the end of each turn this reads the session's transcript and prints, for each chain new in that turn, one line the user can check (skills/axiomcode/scripts/ax_assists.py holds the rules): - ◆ AxiomCode this turn - • Redirected: the agent was in OrderService.cs, Cart.cs; `path handleOrder PriceCalc` showed PriceCalc.cs … + ◆ AxiomCode this turn · 1 answer led the agent's work + • Helped: `impact total` showed pricing.py (new to the agent) → it edited it → both picked test files (…) pass Rules it holds itself to: · NOTHING FROM THE AGENT'S TEXT. The reply has already been shown when this runs; every link is read from the tool calls and their results, and a link that was not checked is not printed. - · SILENT unless an answer led somewhere: no query, or a query nobody acted on, prints nothing. + · SILENT unless an answer led somewhere: no query, or a query nobody acted on, prints nothing. Only what helped is + counted; the answers that led nowhere are never shown. · NEVER BLOCKS. It prints a message for the user and exits 0; the agent's turn is over. AXIOMCODE_ASSISTS=off turns it off. """ diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py index 73170c977..07bfe1ecb 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py @@ -372,7 +372,8 @@ def chains(ss, root, diff=None): frm, ev_from = [], [] out.append({'answer': ss[i]['id'], 'turn': turn_of[i], 'ts': ss[i]['ts'], 'verb': verb, 'arg': arg[:160], 'label': label, 'shown': len(shown), 'edited': edited, 'read': read, 'from': frm, - 'tests_ran': len(ran), 'tests_passed': len(passed), + 'tests_ran': len(ran), 'tests_passed': len(passed), 'tests': sorted(ran), + 'tests_failed': sorted(ran - passed), # the tool calls each link stands on, by the host's id, so every chain can be checked by hand 'evidence': {'answer': ss[i]['id'], 'from': ids(ev_from), 'read': ids(sorted(set(ev_read))), 'edit': ids(sorted(set(ev_edit))), 'test': ids(ev_test)}}) @@ -404,29 +405,45 @@ def _names(fs, n=3): return ', '.join(b[:n]) + (f' +{len(b) - n}' if len(b) > n else '') -def _picked(c): - n, p = c['tests_ran'], c['tests_passed'] - res = ({1: 'it passes', 2: 'both pass'}.get(n, f'all {n} pass') if p == n else f'{p} of {n} pass') - return f"{_q(c)} picked {n} test file{'s' if n > 1 else ''}; {res}" +def _tests(c): + """the effect of the tests an answer picked: which ran, and whether they pass""" + n, t = c['tests_ran'], c.get('tests') or [] + if not n: + return '' + names = f" ({_names(t)})" if t else '' + if c['tests_passed'] == n: + return f" → {({1: 'its', 2: 'both'}.get(n, f'all {n}'))} picked test file{'s' if n > 1 else ''}{names} {'passes' if n == 1 else 'pass'}" + ok = [f for f in t if f not in (c.get('tests_failed') or [])] + return f" → picked test file{'s' if len(ok) > 1 else ''} {_names(ok)} {'pass' if len(ok) > 1 else 'passes'}" if ok else "" def line(c): - tests, n = '', c['tests_ran'] - if n: - every = {1: 'the picked test file passes', 2: 'both picked test files pass'}.get(n, f'all {n} picked test files pass') - tests = f'; {every}' if c['tests_passed'] == n else f"; {c['tests_passed']} of {n} picked test files pass" + """one chain as cause → action → effect, naming the files and tests it stands on""" + q, ed, rd = _q(c), c['edited'], c['read'] + them = lambda fs: 'it' if len(fs) == 1 else 'them' if c['label'] == 'Redirected': - return (f"Redirected: the agent was in {_names(c['from'])}; {_q(c)} showed {_names(c['edited'])} " - f"(first seen there), and the change landed there{tests}") - if c['label'] == 'Helped': - if c['edited']: - extra = f", read {_names(c['read'])}" if c['read'] else '' - return (f"Helped: {_q(c)} showed {_names(c['edited'])} (first seen there); " - f"edited {'it' if len(c['edited']) == 1 else 'them'}{extra}{tests}") - return f"Helped: {_picked(c)}" - if not c['read']: - return f"Useful: {_picked(c)}" - return f"Useful: {_q(c)} showed {_names(c['read'])} (first seen there); read {'them' if len(c['read']) > 1 else 'it'}{tests}" + return (f"Redirected: the agent was in {_names(c['from'])} → {q} showed {_names(ed)} (new to it) " + f"→ it left those and edited {them(ed)}{_tests(c)}") + if ed: + extra = f", read {_names(rd)}" if rd else '' + return f"{c['label']}: {q} showed {_names(ed)} (new to the agent) → it edited {them(ed)}{extra}{_tests(c)}" + if rd: + return f"{c['label']}: {q} showed {_names(rd)} (new to the agent) → it read {them(rd)}{_tests(c)}" + n = c['tests_ran'] + ok = 'all pass' if c['tests_passed'] == n else '' + return (f"{c['label']}: {q} picked {n} test file{'s' if n > 1 else ''} ({_names(c.get('tests') or [])}) " + f"→ the agent ran {'it' if n == 1 else 'them'}" + (f" → {'it passes' if n == 1 else ok}" if ok else '')) + + +def asked(ss): + """the answers the session holds: [(turn, step id)]""" + out, turn = [], -1 + for s in ss: + if s['kind'] == 'prompt': + turn += 1 + elif answer_of(s): + out.append((turn, s['id'])) + return out def _load(root): @@ -436,9 +453,11 @@ def _load(root): with open(p, encoding='utf-8') as f: for l in f: try: - out.append(json.loads(l)) + r = json.loads(l) except ValueError: - pass + continue + if 'answer' in r: + out.append(r) except OSError: pass return out @@ -448,7 +467,7 @@ def new_chains(found, recorded, session): """the chains not printed before, or grown since: a stronger label, or a file edited that was not""" best = {} for r in recorded: - if r.get('session') == session: + if r.get('session') == session and 'answer' in r: best[r['answer']] = r out = [] for c in found: @@ -459,7 +478,8 @@ def new_chains(found, recorded, session): return out -def receipt(fresh, totals=None): +def receipt(fresh, totals=None, turn=None, session=None): + """`turn` / `session`: (answers asked, answers that led to work), for the count lines""" if not fresh: return '' fresh = sorted(fresh, key=lambda c: -RANK[c['label']]) @@ -468,11 +488,21 @@ def receipt(fresh, totals=None): l = line(c) if l not in said: said.append(l) - lines = ['◆ AxiomCode this turn'] + [f' • {l}' for l in said[:5]] + head = '◆ AxiomCode this turn' + if turn and turn[1]: + head += f" · {turn[1]} answer{'s' if turn[1] > 1 else ''} led the agent's work" + earlier = len({c['answer'] for c in fresh} - set(turn[2] if turn and len(turn) > 2 else [])) + if turn and earlier: + head += (', and ' if turn[1] else ' · ') + f"{earlier} earlier answer{'s' if earlier > 1 else ''} went further" + lines = [head] + [f' • {l}' for l in said[:5]] if len(said) > 5: lines.append(f' • and {len(said) - 5} more (axiomcode report)') - if totals and sum(totals.values()) > len(fresh): - lines.append(' session: ' + ' · '.join(f'{totals[k]} {k}' for k in ('Redirected', 'Helped', 'Useful') if totals.get(k))) + if totals and session and session[1] > len({c['answer'] for c in fresh}): + t = totals + lines.append(f" session: {session[1]} answer{'s' if session[1] > 1 else ''} led the work · " + ', '.join( + f'{t[k]} {k}' for k in ('Redirected', 'Helped', 'Useful') if t.get(k)) + + (f" · {t['_edited']} file{'s' if t['_edited'] > 1 else ''} edited from them" if t.get('_edited') else '') + + (f" · {t['_tests']} picked test file{'s' if t['_tests'] > 1 else ''} run" if t.get('_tests') else '')) return '\n'.join(lines) @@ -487,27 +517,48 @@ def record(root, session, fresh): pass -def totals(recorded, session): +def _best(recorded, session=None): + """each answer's strongest recorded chain""" best = {} for r in recorded: - if r.get('session') == session: - if RANK.get(r['label'], 0) >= RANK.get(best.get(r['answer'], ''), 0): - best[r['answer']] = r['label'] - t = {} - for l in best.values(): - t[l] = t.get(l, 0) + 1 + if 'answer' in r and (session is None or r.get('session') == session): + k = (r.get('session'), r['answer']) + if k not in best or RANK.get(r['label'], 0) >= RANK.get(best[k]['label'], 0): + best[k] = r + return best + + +def totals(recorded, session): + """labels per answer, plus `_edited` (files edited from an answer) and `_tests` (picked test files run)""" + t, edited, tests = {}, set(), set() + for r in _best(recorded, session).values(): + t[r['label']] = t.get(r['label'], 0) + 1 + for r in recorded: + if r.get('session') == session and 'answer' in r: + edited |= set(r.get('edited') or []) + tests |= set(r.get('tests') or []) + if edited: + t['_edited'] = len(edited) + if tests: + t['_tests'] = len(tests) return t def at_stop(transcript, root, session): """the receipt for the turn that just ended, recorded; '' when there is nothing new""" - found = chains(steps(transcript), root, working_diff(root)) + ss = steps(transcript) + found = chains(ss, root, working_diff(root)) recorded = _load(root) fresh = new_chains(found, recorded, session) + q = asked(ss) + last = sum(1 for s in ss if s['kind'] == 'prompt') - 1 + record(root, session, fresh) if not fresh: return '' - record(root, session, fresh) - return receipt(fresh, totals(recorded + [dict(c, session=session) for c in fresh], session)) + led = {c['answer'] for c in found} + now = [i for t, i in q if t == last] + return receipt(fresh, totals(recorded + [dict(c, session=session) for c in fresh], session), + turn=(len(now), len(led & set(now)), now), session=(len(q), len(led))) # ---- the report ----------------------------------------------------------------------------------------------------- @@ -524,39 +575,62 @@ def at_stop(transcript, root, session): .s{margin:28px 0 8px;color:var(--mut);font-size:13px;text-transform:uppercase;letter-spacing:.04em} .c{background:var(--card);border:1px solid var(--line);border-radius:10px;padding:12px 14px;margin:8px 0} .l{font-weight:600;font-size:13px}.Redirected{color:var(--r)}.Helped{color:var(--h)}.Useful{color:var(--u)} -.c p{margin:4px 0 0}.c small{color:var(--mut)}code{font:13px ui-monospace,Menlo,monospace} +.c p{margin:4px 0 0}.c ol{margin:6px 0 0;padding-left:20px;color:var(--mut);font-size:13px}.c ol b{color:var(--fg);font-weight:600}.tile small{font-size:14px;color:var(--mut)}.c small{color:var(--mut)}code{font:13px ui-monospace,Menlo,monospace} """ +def _steps_html(r): + """a chain as its links, each one checkable: what the answer gave, what the agent did, what came of it""" + e = html.escape + out = [f"
  • asked {e(_q(r).strip('`'))}
    • "] + if r.get('from'): + out.append(f"
  • before the agent was in {e(', '.join(r['from']))}
    • ") + gave = (r.get('edited') or []) + (r.get('read') or []) + if gave: + out.append(f"
  • showed {e(', '.join(gave))}, new to the agent
    • ") + if r.get('edited'): + out.append(f"
  • edited {e(', '.join(r['edited']))} (kept in the working tree)
    • ") + if r.get('read'): + out.append(f"
  • read {e(', '.join(r['read']))}
    • ") + if r.get('tests_ran'): + t = ', '.join(r.get('tests') or []) or f"{r['tests_ran']} file(s)" + ok = r['tests_passed'] == r['tests_ran'] + out.append(f"
  • ran the picked tests {e(t)}" + (" → all pass" if ok else '') + "
    • ") + return '
      ' + ''.join(out) + '
    ' + + def report(root): - rec = _load(root) - best = {} - for r in rec: - k = (r.get('session'), r['answer']) - if k not in best or RANK.get(r['label'], 0) >= RANK.get(best[k]['label'], 0): - best[k] = r - rows = sorted(best.values(), key=lambda r: r.get('at') or '', reverse=True) + rows = sorted(_best(_load(root)).values(), key=lambda r: r.get('at') or '', reverse=True) t = {} for r in rows: t[r['label']] = t.get(r['label'], 0) + 1 sessions = {} for r in rows: sessions.setdefault(r.get('session') or '?', []).append(r) + edited = {(r.get('session'), f) for r in _load(root) for f in r.get('edited') or []} + tests = {(r.get('session'), f) for r in _load(root) for f in r.get('tests') or [] if f not in (r.get('tests_failed') or [])} e = html.escape body = [f"

    Where AxiomCode helped

    {e(os.path.basename(os.path.abspath(root)))} · " - f"{len(rows)} verified chain(s) over {len(sessions)} session(s). Every chain is read from the session record: " - "an answer showed a file first, and the agent then read or changed it.

    "] + f"{len(sessions)} session(s). Every line is read from the session record, not from what the agent wrote: " + "an answer named a file the agent had not seen, and the agent then read or changed it, or ran the tests it " + "picked.

    "] + for v, k in ((len(rows), 'answers that led the agent\'s work'), + (len({f for _, f in edited}), 'files edited that an answer showed first'), + (len({f for _, f in tests}), 'test files it picked, run and passing')): + if v: + body.append(f"
    {v}{k}
    ") + body.append('
    ') for k in ('Redirected', 'Helped', 'Useful'): - body.append(f"
    {t.get(k, 0)}{k}
    ") + if t.get(k): # only what happened: a label no chain earned is not shown + body.append(f"
    {t[k]}{k}
    ") body.append('
    ') for s, rs in sessions.items(): - body.append(f"
    session {e(s[:8])} · {e((rs[-1].get('at') or '')[:16].replace('T', ' '))}
    ") + body.append(f"
    session {e(s[:8])} · {e((rs[-1].get('at') or '')[:16].replace('T', ' '))}" + + f" · {len(rs)} answer{'s' if len(rs) > 1 else ''} led the work
    ") for r in rs: body.append(f"
    {r['label']}" f"

    {re.sub(r'`([^`]+)`', r'\1', e(line(r).split(': ', 1)[1]))}

    " - + (f"edited: {e(', '.join(r['edited']))}
    " if r.get('edited') else '') - + (f"read: {e(', '.join(r['read']))}
    " if r.get('read') else '') - + (f"left: {e(', '.join(r['from']))}" if r.get('from') else '') + '
    ') + f"{_steps_html(r)}
    ") page = (f"" f"AxiomCode assists
    {''.join(body)}
    ") out = os.path.join(root, '.axiomcode', 'assists.html') @@ -575,7 +649,10 @@ def replay(transcript, root): n += 1 fresh = new_chains(chains(ss[:e], root, None), recorded, 'replay') recorded += [dict(c, session='replay') for c in fresh] - r = receipt(fresh, totals(recorded, 'replay')) + q = asked(ss[:e]) + led = {c['answer'] for c in recorded} + now = [i for t, i in q if t == n - 1] + r = receipt(fresh, totals(recorded, 'replay'), turn=(len(now), len(led & set(now)), now), session=(len(q), len(led))) if r: print(f'--- turn {n}\n{r}') return recorded diff --git a/tests/assists.py b/tests/assists.py index aeabbe784..3247ffaf2 100644 --- a/tests/assists.py +++ b/tests/assists.py @@ -140,6 +140,7 @@ def found(t, r, diff=True): .call('Bash', {'command': 'pytest tests/test_alpha.py -q'}, '1 passed')) c = found(t, r) check('picked tests that ran and passed are Helped', [x['label'] for x in c] == ['Helped'] and c[0]['tests_passed'] == 1, c) +check('the line names the picked test file and its result', 'test_alpha.py' in A.line(c[0]) and 'it passes' in A.line(c[0]), A.line(c[0])) r = repo() t = (T(r).prompt('which tests cover my change').call('mcp__plugin_axiomcode_axiomcode__tests', {}, TESTS_A) .call('Bash', {'command': 'pytest tests/test_alpha.py -q'}, 'Exit code 1\n1 failed', error=True)) @@ -234,6 +235,19 @@ def found(t, r, diff=True): msg = json.loads(h.stdout).get('systemMessage', '') if h.stdout.strip() else '' check('the Stop hook prints the receipt as a message for the user', h.returncode == 0 and msg.startswith('◆ AxiomCode') and 'Helped' in msg, (h.returncode, h.stdout, h.stderr)) +check('it counts the answers that led the work, never those that did not', '1 answer led' in msg and ' of ' not in msg.splitlines()[0], msg) + +# --- two answers, one acted on: the count says so; a later turn that only grows an earlier chain says that instead +r = repo(); change(r, 'src/pkg/alpha.py') +t = (T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, IMPACT_A) + .call(MCP, {'name': 'Gamma.go'}, 'no callers found').read('src/pkg/alpha.py')) +first = A.at_stop(t.save(), r, 's4') +t.prompt('now fix it').edit('src/pkg/alpha.py') +later = A.at_stop(t.save(), r, 's4') +check('one of two answers acted on counts only the one', '1 answer led' in first and ' of 2' not in first, first) +check('a turn that only grows an earlier chain names it as earlier', 'earlier answer went further' in later and 'Helped' in later, later) +page = open(A.report(r)[0]).read() +check('the report counts answers asked and the files and tests behind them', ' of 2' not in page and 'edited' in page, page[:300]) ev['session_id'] = 's3' h = subprocess.run([sys.executable, os.path.join(HOOKS, 'assist.py')], input=json.dumps(ev), capture_output=True, text=True, env=dict(os.environ, AXIOMCODE_ASSISTS='off')) From 43bcdec0814445431ed6c14b7574d307ccbed726 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Mon, 12 Oct 2026 10:49:47 -0700 Subject: [PATCH 03/12] plugin: a test file an answer names twice is one test file in the receipt (#1935) Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py | 9 ++++++--- tests/assists.py | 8 ++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py index 07bfe1ecb..b51b0b3cf 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py @@ -317,6 +317,8 @@ def chains(ss, root, diff=None): # the tests an answer picked: every test file a test verb names (picking them is its job); for the other verbs # only a test file first seen in it, since a test the agent already knew of was not picked by the answer tests_named = {f for f in named if TEST_FILE.search(f) and (verb in ('tests', 'test-impact') or f in shown)} + # one file named twice (`test_x.py` in a run line, `tests/test_x.py` in a row) is one test file + tests_named = {f for f in tests_named if not any(g != f and g.endswith('/' + f) for g in tests_named)} read, edited, ev_read, ev_edit = [], [], [], [] for f in shown: r = [j for j, k, fs in acts if k == 'read' and j > i and _hit(f, fs)] @@ -614,9 +616,10 @@ def report(root): f"{len(sessions)} session(s). Every line is read from the session record, not from what the agent wrote: " "an answer named a file the agent had not seen, and the agent then read or changed it, or ran the tests it " "picked.

    "] - for v, k in ((len(rows), 'answers that led the agent\'s work'), - (len({f for _, f in edited}), 'files edited that an answer showed first'), - (len({f for _, f in tests}), 'test files it picked, run and passing')): + ne, nt = len({f for _, f in edited}), len({f for _, f in tests}) + for v, k in ((len(rows), f"answer{'s' if len(rows) != 1 else ''} that led the agent's work"), + (ne, f"file{'s' if ne != 1 else ''} edited that an answer showed first"), + (nt, f"test file{'s' if nt != 1 else ''} it picked, run and passing")): if v: body.append(f"
    {v}{k}
    ") body.append('
    ') diff --git a/tests/assists.py b/tests/assists.py index 3247ffaf2..d53e4026b 100644 --- a/tests/assists.py +++ b/tests/assists.py @@ -146,6 +146,14 @@ def found(t, r, diff=True): .call('Bash', {'command': 'pytest tests/test_alpha.py -q'}, 'Exit code 1\n1 failed', error=True)) check('picked tests that failed are only Useful', [x['label'] for x in found(t, r)] == ['Useful'], found(t, r)) +# --- a test file named twice in one answer (bare, and with its directory) is one test file +r = repo() +t = (T(r).prompt('which tests cover my change').call('mcp__plugin_axiomcode_axiomcode__tests', {}, + TESTS_A + '\nrun: pytest test_alpha.py') + .call('Bash', {'command': 'pytest tests/test_alpha.py -q'}, '1 passed')) +c = found(t, r) +check('a test file named twice is counted once', c and c[0]['tests_ran'] == 1 and c[0]['tests'] == ['tests/test_alpha.py'], c) + # --- a test file the agent already knew of is not picked by an impact answer; a tests answer picks what it names r = repo() IMPACT_T = IMPACT_A + '\ntests:\n tests/test_alpha.py:5 test_run [test]' From 148e9c5c092ed43129779e41be40a2d217e01f19 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Mon, 12 Oct 2026 10:57:22 -0700 Subject: [PATCH 04/12] plugin: the receipt credits core files the graph named, from its notes and from shell edits (#1935) - A graph note the plugin's own hooks put in front of the agent (the callers an edit reaches, where a task lands) is an answer like a tool call's; its hook_success copy no longer marks its files seen first. - A shell line that edits files and also runs axiomcode keeps its edits, and a helper the script defines and calls with a path (def sub(path, ...): ...write_text...) writes that path. - A row about a name the task already gave is the task's lead, not the answer's (the name asked about is exempt: it is on every caller's line); a note credits a file:line, not a file summary. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- .../skills/axiomcode/scripts/ax_assists.py | 61 ++++++++++++++++++- tests/assists.py | 50 +++++++++++++++ 2 files changed, 108 insertions(+), 3 deletions(-) diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py index b51b0b3cf..91278e4b7 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py @@ -81,6 +81,18 @@ def _unwrap(t): return t +HOOK = 'axiomcode-hook' + + +def _graph_note(a): + """the hook event when this attachment is a graph answer the plugin's hooks gave the agent, else ''""" + if a.get('type') == 'hook_additional_context' or (a.get('type') == 'hook_success' + and a.get('hookEvent') == 'UserPromptSubmit'): + if _text(a.get('content')).lstrip().startswith('graph:'): + return str(a.get('hookEvent') or a.get('hookName') or '').split(':')[0] + return '' + + def steps(transcript): """the session in order: prompts, replies and tool calls with their results. Side chains (subagents) are left out: what a subagent saw, the agent saw only as the subagent's reply.""" @@ -98,6 +110,18 @@ def steps(transcript): if not isinstance(e, dict) or e.get('isSidechain'): continue if e.get('type') in ('attachment', 'system'): + a = e.get('attachment') if isinstance(e.get('attachment'), dict) else {} + hook = _graph_note(a) + if hook: + # the plugin's own hooks put a graph answer in front of the agent: after an edit, the callers it + # reaches; on a prompt, where the task lands. That is an answer like a tool call's. + out.append({'kind': 'call', 'id': 'hook:' + str(e.get('uuid') or len(out)), 'name': HOOK, + 'input': {'event': hook}, 'result': _text(a.get('content')), 'error': False, + 'ts': e.get('timestamp') or ''}) + continue + if a.get('type') == 'hook_success' and 'graph:' in str(a.get('stdout') or '') and \ + str(a.get('hookEvent')) != 'UserPromptSubmit': + continue # the same text as the hook_additional_context that follows it # a hook's context, a mentioned file, a reminder: the agent saw it, so what it names is not first seen later t = json.dumps(e.get('attachment') or e.get('content') or '', ensure_ascii=False) if PATH.search(t): @@ -163,6 +187,9 @@ def _seen_text(s): return json.dumps(s['input'], ensure_ascii=False) + '\n' + (s['result'] or '') + '\n' + s.get('extra', '') +# a code name as a task writes it: snake_case, CamelCase or Owner.member — not a plain word (find, where, define) +IDENT = re.compile(r'(? index of the answer that first showed it answers = {} # index -> (verb, arg) turn_of = [] @@ -283,19 +318,34 @@ def chains(ss, root, diff=None): for i, s in enumerate(ss): if s['kind'] == 'prompt': turn += 1 + asked_for |= set(IDENT.findall(s['text'])) | {w for c in re.findall(r'`([^`\n]{1,80})`', s['text']) + for w in re.findall(r'[A-Za-z_]\w{2,}', c)} turn_of.append(turn) a = answer_of(s) if a: answers[i] = a + verb = a[0] for f in paths_in(json.dumps(s['input'], ensure_ascii=False), root): seen_full.add(f) seen_base.add(f.rsplit('/', 1)[-1]) + res = s['result'] or '' + # the name asked about is on every caller's line by construction; the task naming it is no lead to its callers + lead = asked_for - {w for x in re.findall(r'[\w.]+', a[1]) for w in (x, *x.split('.'))} + led = set() + for ln in res.splitlines(): + if lead & {w for x in IDENT.findall(ln) for w in (x, *x.split('.'))}: + led |= {norm(g.group(1), root) for g in PATH.finditer(ln)} for m in (ROW.finditer(s['result'] or '') if mixed(s) else PATH.finditer(s['result'] or '')): - if not PLACE.search(m.group(0) + (s['result'] or '')[m.end():m.end() + 2]): + after = (s['result'] or '')[m.end():m.end() + 2] + if not PLACE.search(m.group(0) + after): continue # a bare file name (a listing, prose) is not a place the answer gave + if verb == 'note' and not re.match(r':\d', after): + continue # a note nobody asked for credits a place (file:line), not a file summary f = norm(m.group(1), root) if not f: continue + if f in led: + continue # the task already named what a row on this file is about: its lead b = f.rsplit('/', 1)[-1] if f in seen_full or f in first or (not GENERIC.match(b) and b in seen_base): continue @@ -395,7 +445,12 @@ def working_diff(root): # ---- what is printed ------------------------------------------------------------------------------------------------ +NOTES = {'PostToolUse': "AxiomCode's after-edit check", 'UserPromptSubmit': "AxiomCode's note on the task"} + + def _q(c): + if c['verb'] == 'note': + return NOTES.get(c['arg'], 'an AxiomCode note') # what was asked, without flags or paths (--json, an output file), and a shell loop's variable says nothing arg = ' '.join(w for w in c['arg'].split() if not w.startswith(('-', '/', '$', '~'))) v = c['verb'] + (f" {arg}" if arg else '') diff --git a/tests/assists.py b/tests/assists.py index d53e4026b..cb48c01b8 100644 --- a/tests/assists.py +++ b/tests/assists.py @@ -51,6 +51,15 @@ def read(self, rel): def edit(self, rel): return self.call('Edit', {'file_path': os.path.join(self.root, rel), 'old_string': 'a', 'new_string': 'b'}, 'ok') + def note(self, text, kind='hook_additional_context', event='PostToolUse', stdout=None): + """a hook's output put in front of the agent, as the host records it""" + self.n += 1 + a = {'type': kind, 'hookName': event, 'hookEvent': event, 'content': [text] if kind == 'hook_additional_context' else text} + if stdout is not None: + a['stdout'] = stdout + self.lines.append({'type': 'attachment', 'uuid': f'u{self.n:03d}', 'attachment': a}) + return self + def save(self): p = os.path.join(self.root, '..', f'transcript-{id(self)}.jsonl') with open(p, 'w') as f: @@ -219,6 +228,47 @@ def found(t, r, diff=True): t = (T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, IMPACT_A, side=True).edit('src/pkg/alpha.py')) check('an answer inside a subagent is not credited to the session', found(t, r) == [], found(t, r)) +# --- the plugin's own hooks: a graph note put in front of the agent is an answer +NOTE = 'graph: after that command, 1 declaration(s) changed —\n reads / uses it (1): [resolved] Beta.go src/pkg/beta.py:3' +r = repo(); change(r, 'src/pkg/beta.py') +t = T(r).prompt('add a flag to run').edit('src/pkg/alpha.py').note(NOTE).edit('src/pkg/beta.py') +c = found(t, r) +check('the after-edit note that named a caller, then edited, is Helped', [(x['verb'], x['label'], x['edited']) for x in c] + == [('note', 'Helped', ['src/pkg/beta.py'])] and "after-edit check" in A.line(c[0]), c) +r = repo(); change(r, 'src/pkg/beta.py') +t = T(r).prompt('add a flag to run').note(NOTE, kind='hook_success', stdout=json.dumps({'additionalContext': NOTE})) \ + .note(NOTE).edit('src/pkg/beta.py') +check('the hook_success copy of a note does not make its files seen before the note', [x['verb'] for x in found(t, r)] == ['note'], found(t, r)) +r = repo(); change(r, 'src/pkg/beta.py') +t = T(r).prompt('add a flag to run').note('graph: where this task lands —\n src/pkg/beta.py (4 symbols)').edit('src/pkg/beta.py') +check('a note naming a file without a line (a summary) is not credited', found(t, r) == [], found(t, r)) +r = repo(); change(r, 'src/pkg/beta.py') +t = T(r).prompt('add a flag to run').note('reminder: src/pkg/beta.py:3 was opened').edit('src/pkg/beta.py') +check('an attachment that is not a graph note is no answer', found(t, r) == [], found(t, r)) + +# --- the task already named what a row is about: that row is the task's lead, not the answer's +ROWS = '1. src/pkg/beta.py:3 [resolved · Beta.go]\n2. src/pkg/gamma.py:7 [resolved · Gamma.stop]' +r = repo(); change(r, 'src/pkg/beta.py'); change(r, 'src/pkg/gamma.py') +t = (T(r).prompt('fix `Beta.go` so it stops').call(MCP, {'name': 'Alpha.run'}, ROWS) + .edit('src/pkg/beta.py').edit('src/pkg/gamma.py')) +check('a row about a name the task gave is not credited; the other row is', [x['edited'] for x in found(t, r)] == [['src/pkg/gamma.py']], found(t, r)) +r = repo(); change(r, 'src/pkg/beta.py') +t = (T(r).prompt('add a flag to `Alpha.run`').call(MCP, {'name': 'Alpha.run'}, + '1. src/pkg/beta.py:3 [resolved · Beta.go]\n → 3 self.run() # Alpha.run') + .edit('src/pkg/beta.py')) +check('the name asked about, on its callers\' lines, does not hide them', [x['edited'] for x in found(t, r)] == [['src/pkg/beta.py']], found(t, r)) + +# --- an edit in a shell line that also asks the graph, and an edit through a helper the script defines +r = repo(); change(r, 'src/pkg/alpha.py') +t = (T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, IMPACT_A) + .call('Bash', {'command': "sed -i '' 's/a/b/' src/pkg/alpha.py; axiomcode tests"}, '1. tests/test_alpha.py:5 [test]')) +check('an edit in a line that also runs axiomcode is still an edit', [x['label'] for x in found(t, r) if x['verb'] == 'impact'] == ['Helped'], found(t, r)) +HELPER = ("python3 - <<'EOF'\nimport pathlib\ndef sub(path, old, new):\n p = pathlib.Path(path); s = p.read_text()\n" + " p.write_text(s.replace(old, new))\nsub('src/pkg/alpha.py', 'x', 'y')\nprint(open('src/pkg/gamma.py').read())\nEOF") +r = repo(); change(r, 'src/pkg/alpha.py'); change(r, 'src/pkg/gamma.py') +check('a helper that writes its path argument writes the files it is called with, not the ones only read', + A.written(HELPER, r) == {'src/pkg/alpha.py'}, A.written(HELPER, r)) + # --- the ledger: printed once; printed again only when it grows r = repo(); change(r, 'src/pkg/alpha.py') t = T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, IMPACT_A).read('src/pkg/alpha.py') From 49a96f06b150282ebfe622f79a3f8f42de2eb0ef Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:38:45 -0700 Subject: [PATCH 05/12] plugin: the assists report tells what the agent did with each answer, with every action inspectable (#1935) - The report opens on the outcome (sessions, answers acted on, previously unseen files edited, selected tests passing), then the three ways answers showed, featured timelines, every answer expandable to its recorded actions by sequence number, and the rules each label is assigned by. - Receipts and report say what happened: the agent acted on N answers; an answer pointed the agent to a file directly, before it had come across it. - camelCase names a task gives are the task's lead; a runner config under test/ is not a test file; a run that exits 0 but reports failures is not a pass. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- .../skills/axiomcode/scripts/ax_assists.py | 292 ++++++++++++++---- tests/assists.py | 18 +- 2 files changed, 245 insertions(+), 65 deletions(-) diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py index 91278e4b7..eb5e934de 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py @@ -41,11 +41,23 @@ r'php|swift|m|mm|c|cc|cpp|cxx|h|hpp|sql|proto|graphql|gql') PATH = re.compile(r'(?\n]+)') # the command, then its file arguments EDIT_TOOLS = {'Edit', 'Write', 'MultiEdit', 'NotebookEdit', 'replace', 'write_file'} @@ -139,7 +151,10 @@ def steps(transcript): if not t: continue real = not e.get('isMeta') and not t.startswith(HARNESS) - out.append({'kind': 'prompt' if real else 'note', 'text': t, 'ts': ts}) + k = len(out) + while real and k and out[k - 1].get('name') == HOOK and out[k - 1]['input'].get('event') == 'UserPromptSubmit': + k -= 1 # the host records a prompt's own hook note before the prompt; it answers it + out.insert(k, {'kind': 'prompt' if real else 'note', 'text': t, 'ts': ts}) continue for x in content: if isinstance(x, dict) and x.get('type') == 'tool_result': @@ -187,8 +202,9 @@ def _seen_text(s): return json.dumps(s['input'], ensure_ascii=False) + '\n' + (s['result'] or '') + '\n' + s.get('extra', '') -# a code name as a task writes it: snake_case, CamelCase or Owner.member — not a plain word (find, where, define) +# a code name as a task writes it: snake_case, CamelCase, camelCase or Owner.member — not a plain word (find, where, define) IDENT = re.compile(r'(? 1 else ''} ({_names(c.get('tests') or [])}) " @@ -547,7 +592,7 @@ def receipt(fresh, totals=None, turn=None, session=None): said.append(l) head = '◆ AxiomCode this turn' if turn and turn[1]: - head += f" · {turn[1]} answer{'s' if turn[1] > 1 else ''} led the agent's work" + head += f" · the agent acted on {turn[1]} answer{'s' if turn[1] > 1 else ''}" earlier = len({c['answer'] for c in fresh} - set(turn[2] if turn and len(turn) > 2 else [])) if turn and earlier: head += (', and ' if turn[1] else ' · ') + f"{earlier} earlier answer{'s' if earlier > 1 else ''} went further" @@ -556,7 +601,7 @@ def receipt(fresh, totals=None, turn=None, session=None): lines.append(f' • and {len(said) - 5} more (axiomcode report)') if totals and session and session[1] > len({c['answer'] for c in fresh}): t = totals - lines.append(f" session: {session[1]} answer{'s' if session[1] > 1 else ''} led the work · " + ', '.join( + lines.append(f" session: the agent acted on {session[1]} answer{'s' if session[1] > 1 else ''} · " + ', '.join( f'{t[k]} {k}' for k in ('Redirected', 'Helped', 'Useful') if t.get(k)) + (f" · {t['_edited']} file{'s' if t['_edited'] > 1 else ''} edited from them" if t.get('_edited') else '') + (f" · {t['_tests']} picked test file{'s' if t['_tests'] > 1 else ''} run" if t.get('_tests') else '')) @@ -569,7 +614,8 @@ def record(root, session, fresh): os.makedirs(os.path.dirname(p), exist_ok=True) with open(p, 'a', encoding='utf-8') as f: for c in fresh: - f.write(json.dumps(dict(c, session=session, at=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()))) + '\n') + f.write(json.dumps(dict(c, session=session, repo=os.path.basename(root.rstrip('/')), + at=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()))) + '\n') except OSError: pass @@ -621,76 +667,198 @@ def at_stop(transcript, root, session): # ---- the report ----------------------------------------------------------------------------------------------------- CSS = """ -:root{--bg:#fbfbfa;--fg:#1d1d1b;--mut:#6b6a66;--card:#fff;--line:#e6e4df;--r:#7a3df0;--h:#127a55;--u:#8a6d1f} -@media (prefers-color-scheme:dark){:root{--bg:#141413;--fg:#ecebe6;--mut:#a3a29c;--card:#1d1d1b;--line:#33322f;--r:#b394ff;--h:#4fd1a1;--u:#e0c063}} -body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 -apple-system,system-ui,sans-serif} -main{max-width:860px;margin:0 auto;padding:32px 16px} -h1{font-size:22px;margin:0 0 4px}.sub{color:var(--mut);margin:0 0 24px} -.tiles{display:flex;gap:12px;flex-wrap:wrap;margin-bottom:28px} -.tile{flex:1;min-width:140px;background:var(--card);border:1px solid var(--line);border-radius:10px;padding:14px} -.tile b{display:block;font-size:28px}.tile span{color:var(--mut);font-size:13px} -.s{margin:28px 0 8px;color:var(--mut);font-size:13px;text-transform:uppercase;letter-spacing:.04em} -.c{background:var(--card);border:1px solid var(--line);border-radius:10px;padding:12px 14px;margin:8px 0} -.l{font-weight:600;font-size:13px}.Redirected{color:var(--r)}.Helped{color:var(--h)}.Useful{color:var(--u)} -.c p{margin:4px 0 0}.c ol{margin:6px 0 0;padding-left:20px;color:var(--mut);font-size:13px}.c ol b{color:var(--fg);font-weight:600}.tile small{font-size:14px;color:var(--mut)}.c small{color:var(--mut)}code{font:13px ui-monospace,Menlo,monospace} +:root{--bg:#fafaf8;--fg:#1b1b19;--mut:#66655f;--card:#fff;--line:#e5e3dd;--soft:#f2f1ec;--acc:#2f6fed;--ok:#127a55;--r:#7a3df0;--u:#8a6d1f} +@media (prefers-color-scheme:dark){:root:not([data-theme=light]){--bg:#121211;--fg:#ecebe6;--mut:#a3a29c;--card:#1b1b1a;--line:#302f2c;--soft:#22221f;--acc:#7aa5ff;--ok:#4fd1a1;--r:#b394ff;--u:#e0c063}} +:root[data-theme=dark]{--bg:#121211;--fg:#ecebe6;--mut:#a3a29c;--card:#1b1b1a;--line:#302f2c;--soft:#22221f;--acc:#7aa5ff;--ok:#4fd1a1;--r:#b394ff;--u:#e0c063} +*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.55 -apple-system,system-ui,sans-serif} +main{max-width:940px;margin:0 auto;padding:40px 16px 80px} +code,pre{font:12.5px/1.5 ui-monospace,Menlo,monospace}pre{white-space:pre-wrap;word-break:break-word;margin:6px 0 0;padding:8px 10px;background:var(--soft);border-radius:6px} +.eyebrow{font-size:12px;letter-spacing:.12em;text-transform:uppercase;color:var(--acc);font-weight:700} +h1{font-size:34px;line-height:1.15;margin:8px 0 12px;letter-spacing:-.01em}h2{font-size:21px;margin:0 0 4px} +.sec{margin-top:56px}.num{color:var(--mut);font-size:13px;font-weight:600;letter-spacing:.06em} +.lede{font-size:18px;color:var(--fg);max-width:720px;margin:0 0 24px}.sub{color:var(--mut);margin:0 0 18px} +.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(190px,1fr));gap:12px} +.tile{background:var(--card);border:1px solid var(--line);border-radius:12px;padding:16px}.tile b{display:block;font-size:34px;line-height:1.1} +.tile span{color:var(--mut);font-size:13.5px}.verified{margin-top:14px;font-size:13.5px;color:var(--ok);font-weight:600} +.cap{background:var(--card);border:1px solid var(--line);border-radius:12px;padding:16px}.cap b{font-size:26px;color:var(--acc)} +.cap h3{margin:4px 0 6px;font-size:16px}.cap p{margin:0;color:var(--mut);font-size:14px} +.case{background:var(--card);border:1px solid var(--line);border-radius:14px;padding:20px;margin:14px 0} +.case .top{display:flex;justify-content:space-between;gap:12px;flex-wrap:wrap;align-items:baseline} +.case .repo{font-size:18px;font-weight:700}.case .q{color:var(--mut);font-size:13.5px} +.badge{font-size:12px;font-weight:700;padding:3px 9px;border-radius:99px;background:var(--soft);color:var(--ok);white-space:nowrap} +.tl{list-style:none;margin:14px 0 0;padding:0;counter-reset:s}.tl li{position:relative;padding:0 0 14px 34px;counter-increment:s} +.tl li:before{content:counter(s);position:absolute;left:0;top:0;width:22px;height:22px;border-radius:50%;background:var(--acc);color:#fff;font-size:12px;font-weight:700;display:flex;align-items:center;justify-content:center} +.tl li:after{content:"";position:absolute;left:10.5px;top:24px;bottom:2px;width:1px;background:var(--line)}.tl li:last-child:after{display:none} +.tl .t{font-weight:600}.tl .d{color:var(--mut);font-size:13.5px}.files{margin:4px 0 0;padding:0;list-style:none}.files li{padding:0!important;counter-increment:none}.files li:before,.files li:after{display:none} +.pass{color:var(--ok);font-weight:600;font-size:13px}.outcome{margin-top:6px;padding:12px 14px;border-left:3px solid var(--ok);background:var(--soft);border-radius:0 8px 8px 0;font-size:14.5px} +details{background:var(--card);border:1px solid var(--line);border-radius:10px;margin:8px 0}summary{cursor:pointer;padding:12px 14px;list-style:none} +summary::-webkit-details-marker{display:none}summary .l{font-size:12px;font-weight:700;margin-right:8px}.Redirected{color:var(--r)}.Helped{color:var(--ok)}.Useful{color:var(--u)} +details .body{padding:0 14px 14px}.ev{border-top:1px solid var(--line);padding:8px 0}.ev .k{display:inline-block;min-width:74px;font-size:12px;font-weight:700;color:var(--mut);text-transform:uppercase} +.ev .n{color:var(--mut);font-size:12px;margin-right:6px}.filters{display:flex;gap:8px;flex-wrap:wrap;margin:10px 0} +.filters button{font:inherit;font-size:13px;padding:5px 12px;border-radius:99px;border:1px solid var(--line);background:var(--card);color:var(--fg);cursor:pointer} +.filters button.on{background:var(--acc);border-color:var(--acc);color:#fff}.method{background:var(--card);border:1px solid var(--line);border-radius:12px;padding:18px 20px} +.method h3{font-size:15px;margin:14px 0 4px}.method h3:first-child{margin-top:0}.method ul{margin:4px 0;padding-left:20px}.method p{margin:4px 0} +@media (max-width:560px){h1{font-size:27px}.tile b{font-size:28px}} """ +KIND = {'answer': 'asked', 'before': 'before', 'read': 'read', 'edit': 'edited', 'test': 'ran tests'} + + +def _caps(r): + """which of the three ways a chain shows: found files first, connected a change across files, tests run""" + c = set() + if r.get('edited') or r.get('read'): + c.add('found') + if r['verb'] in ('impact', 'path') and len((r.get('edited') or []) + (r.get('read') or [])) > 1 or \ + (r['verb'] == 'note' and r.get('arg') == 'PostToolUse') or r.get('from'): + c.add('connected') + if r.get('tests_passed'): + c.add('tests') + return c + + +def _score(r): + return 4 * len(r.get('edited') or []) + len(r.get('read') or []) + min(r.get('tests_passed', 0), 3) + \ + (6 if r['label'] == 'Redirected' else 0) + -def _steps_html(r): - """a chain as its links, each one checkable: what the answer gave, what the agent did, what came of it""" +def _story(r): + """the featured timeline: what was asked, what the answer named first, what the agent then did, and what came of it""" e = html.escape - out = [f"
  • asked {e(_q(r).strip('`'))}
    • "] + ed, rd, n = r.get('edited') or [], r.get('read') or [], len(r.get('edited') or []) + len(r.get('read') or []) + fl = lambda fs: '
      ' + ''.join(f'
    • {e(f)}
      • ' for f in fs) + '
    ' + st = [] if r.get('from'): - out.append(f"
  • before the agent was in {e(', '.join(r['from']))}
    • ") - gave = (r.get('edited') or []) + (r.get('read') or []) - if gave: - out.append(f"
  • showed {e(', '.join(gave))}, new to the agent
    • ") - if r.get('edited'): - out.append(f"
  • edited {e(', '.join(r['edited']))} (kept in the working tree)
    • ") - if r.get('read'): - out.append(f"
  • read {e(', '.join(r['read']))}
    • ") + st.append(f"
  • The agent was working in other files
    {fl(r['from'])}
    • ") + st.append(f"
  • The agent queried AxiomCode
    {e(_q(r).strip('`'))}
    • ") + if n: + word = {1: 'one', 2: 'two', 3: 'three', 4: 'four', 5: 'five'}.get(n, str(n)) + st.append(f"
  • AxiomCode identified {word} file{'s' if n > 1 else ''} the agent had not yet " + f"come across
    {fl(ed + rd)}
    • ") + if ed: + whole = len(ed) == n + st.append(f"
  • The agent edited {('all ' + str(len(ed)) + ' of them' if len(ed) > 2 else ('both' if len(ed) == 2 else 'it')) if whole else _names(ed)}" + f"{' and read ' + _names(rd) if rd else ''}
    The edits remained in the working tree.
    • ") + elif rd: + st.append(f"
  • The agent read {'them' if len(rd) > 1 else 'it'}
    • ") if r.get('tests_ran'): - t = ', '.join(r.get('tests') or []) or f"{r['tests_ran']} file(s)" - ok = r['tests_passed'] == r['tests_ran'] - out.append(f"
  • ran the picked tests {e(t)}" + (" → all pass" if ok else '') + "
    • ") - return '
      ' + ''.join(out) + '
    ' + ok = set(r.get('tests') or []) - set(r.get('tests_failed') or []) + st.append("
  • The agent ran the tests AxiomCode selected
      " + ''.join( + f"
    • {e(f.rsplit('/', 1)[-1])} passed
      • " for f in sorted(ok)) + '
    • ') + out = [] + if n: + out.append(f"AxiomCode identified {n} file{'s' if n > 1 else ''} before the agent encountered " + f"{'them' if n > 1 else 'it'}") + if ed: + out.append(f"the agent edited {('all ' + str(len(ed)) if len(ed) > 2 else ('both' if len(ed) == 2 else 'it')) if len(ed) == n else _names(ed)}" + + (f" and read {_names(rd)}" if rd else '')) + elif rd: + out.append('the agent read ' + ('them' if len(rd) > 1 else 'it')) + if r.get('tests_passed'): + k = r['tests_passed'] + out.append(f"then ran {k} selected test file{'s' if k > 1 else ''} successfully") + txt = ', '.join(out) + return '
      ' + ''.join(st) + '
    ' + (f"
    Observed outcome. {e(txt[0].upper() + txt[1:])}.
    " if txt else '') + + +def _evidence(r): + """every link as the session record shows it, by sequence number""" + e = html.escape + out = [] + for x in r.get('trail') or []: + o = f"
    {e(x['out'])}
    " if x.get('out') else '' + out.append(f"
    #{x['n']}{KIND.get(x['kind'], x['kind'])}" + f"{e(x['text'])}{o}
    ") + return ''.join(out) or '
    Evidence ids: ' + e(json.dumps(r.get('evidence'))) + '
    ' def report(root): rows = sorted(_best(_load(root)).values(), key=lambda r: r.get('at') or '', reverse=True) - t = {} - for r in rows: - t[r['label']] = t.get(r['label'], 0) + 1 + e = html.escape sessions = {} for r in rows: sessions.setdefault(r.get('session') or '?', []).append(r) edited = {(r.get('session'), f) for r in _load(root) for f in r.get('edited') or []} tests = {(r.get('session'), f) for r in _load(root) for f in r.get('tests') or [] if f not in (r.get('tests_failed') or [])} - e = html.escape - body = [f"

    Where AxiomCode helped

    {e(os.path.basename(os.path.abspath(root)))} · " - f"{len(sessions)} session(s). Every line is read from the session record, not from what the agent wrote: " - "an answer named a file the agent had not seen, and the agent then read or changed it, or ran the tests it " - "picked.

    "] - ne, nt = len({f for _, f in edited}), len({f for _, f in tests}) - for v, k in ((len(rows), f"answer{'s' if len(rows) != 1 else ''} that led the agent's work"), - (ne, f"file{'s' if ne != 1 else ''} edited that an answer showed first"), - (nt, f"test file{'s' if nt != 1 else ''} it picked, run and passing")): + ns, ne, nt = len(sessions), len({f for _, f in edited}), len({f for _, f in tests}) + caps = {k: [r for r in rows if k in _caps(r)] for k in ('found', 'connected', 'tests')} + pl = lambda n, w: f"{n} {w}{'s' if n != 1 else ''}" + body = [f"
    01
    AxiomCode · agent impact report

    From code understanding to verified agent actions.

    " + f"

    Across {pl(ns, 'coding session')}, AxiomCode identified files agents had not yet explored, " + f"guided the edits that followed, and selected tests the agents ran and passed.

    "] + for v, k in ((ns, 'Coding sessions'), (len(rows), 'Answers the agent acted on'), (ne, 'Previously unseen files edited'), + (nt, 'Selected test files passing')): if v: body.append(f"
    {v}{k}
    ") - body.append('
    ') - for k in ('Redirected', 'Helped', 'Useful'): - if t.get(k): # only what happened: a label no chain earned is not shown - body.append(f"
    {t[k]}{k}
    ") - body.append('
    ') + body.append("
    ✓ Verified against recorded agent actions, not agent self-reports.
    ") + + body.append("
    02

    How AxiomCode influenced agent behavior

    " + "

    Each answer below is counted under every way it showed.

    ") + for k, t, d in (('found', 'Found relevant files before the agent discovered them', + 'AxiomCode named code locations the agent had not yet encountered; the agent then inspected or edited them.'), + ('connected', 'Connected changes across the codebase', + 'Impact and path answers, and the check after each edit, named related implementation files and tests, ' + 'moving the agent beyond where it started.'), + ('tests', 'Turned analysis into test execution', + 'Agents ran the tests AxiomCode selected, and those runs passed.')): + if caps[k]: + body.append(f"
    {len(caps[k])}

    {t}

    {d}

    ") + body.append('
    ') + + feat, seen = [], set() + for r in sorted(rows, key=_score, reverse=True): + if r.get('session') in seen or not (r.get('edited') or r.get('read')): + continue + feat.append(r); seen.add(r.get('session')) + if len(feat) == 3: + break + if feat: + body.append("
    03

    Featured evidence

    " + "

    The strongest chains, each read step by step from the session record.

    ") + for r in feat: + badge = 'Observed edit + tests passed' if r.get('edited') and r.get('tests_passed') else ( + 'Observed edit' if r.get('edited') else 'Observed read') + body.append(f"
    {e(r.get('repo') or '')}
    " + f"
    {e({'impact': 'Impact analysis', 'path': 'Call path', 'context': 'Code context', 'tests': 'Test selection', 'note': 'Graph note'}.get(r['verb'], r['verb']))}" + f" · {e(_q(r).strip('`'))}
    {badge}
    {_story(r)}" + f"
    Show the recorded actions
    {_evidence(r)}
    ") + body.append('
    ') + + body.append("
    04

    All verified sessions

    " + "

    Open any answer to see the query, AxiomCode's rows, and each later action with its sequence " + "number in the session.

    " + "" + "
    ") for s, rs in sessions.items(): - body.append(f"
    session {e(s[:8])} · {e((rs[-1].get('at') or '')[:16].replace('T', ' '))}" - + f" · {len(rs)} answer{'s' if len(rs) > 1 else ''} led the work
    ") for r in rs: - body.append(f"
    {r['label']}" - f"

    {re.sub(r'`([^`]+)`', r'\1', e(line(r).split(': ', 1)[1]))}

    " - f"{_steps_html(r)}
    ") + tags = ' '.join(t for t, ok in (('edited', r.get('edited')), ('read', r.get('read')), ('tests', r.get('tests_passed'))) if ok) + body.append(f"
    {r['label']}" + f"{e(r.get('repo') or s[:8])} · " + f"{re.sub(r'`([^`]+)`', r'\1', e(line(r).split(': ', 1)[1]))}" + f"
    {_evidence(r)}
    ") + body.append('
    ') + + body.append("""
    05

    Measured from agent actions, not agent claims

    +

    An answer is credited only when all of these hold in the session record

      +
    • The AxiomCode answer named a file (as a place: a path or file:line) before any earlier prompt, tool call, +tool output or reply had named it.
      • +
    • The task had not already named what that row is about; a function the issue names is the task's lead, not AxiomCode's.
      • +
    • Afterwards the agent read or edited that file, or ran a test file the answer selected.
      • +
    • Edits are confirmed in the working tree when the turn ends; test runs and their results come from the tool output.
    +

    Labels are assigned mechanically

      +
    • Helped: a file the answer named first was edited and the edit was kept, or the tests a test-selection +answer picked ran and passed.
      • +
    • Useful: a file the answer named first was read, or tests it picked were run, without an edit from it.
      • +
    • Redirected: Helped, and before the answer the agent had been reading or editing two or more other files +that it never returned to and did not change.
    +

    Scope

    This report shows that AxiomCode's answers were followed by these agent actions. It does not measure what the +agent would have done without them, or whether the edits were correct; a passing test supports only that the selected test +run passed.

    """) + js = ("") page = (f"" - f"AxiomCode assists
    {''.join(body)}
    ") + f"Agent Impact Report
    {''.join(body)}
    {js}") out = os.path.join(root, '.axiomcode', 'assists.html') os.makedirs(os.path.dirname(out), exist_ok=True) with open(out, 'w', encoding='utf-8') as f: diff --git a/tests/assists.py b/tests/assists.py index cb48c01b8..b80dd0e06 100644 --- a/tests/assists.py +++ b/tests/assists.py @@ -153,6 +153,10 @@ def found(t, r, diff=True): r = repo() t = (T(r).prompt('which tests cover my change').call('mcp__plugin_axiomcode_axiomcode__tests', {}, TESTS_A) .call('Bash', {'command': 'pytest tests/test_alpha.py -q'}, 'Exit code 1\n1 failed', error=True)) +r = repo() +t = (T(r).prompt('which tests cover my change').call('mcp__plugin_axiomcode_axiomcode__tests', {}, TESTS_A) + .call('Bash', {'command': 'pytest tests/test_alpha.py -q; echo done'}, '1 failed, 3 passed\ndone')) +check('a run that exits 0 but reports failures is not a pass', [x['label'] for x in found(t, r)] == ['Useful'], found(t, r)) check('picked tests that failed are only Useful', [x['label'] for x in found(t, r)] == ['Useful'], found(t, r)) # --- a test file named twice in one answer (bare, and with its directory) is one test file @@ -163,6 +167,9 @@ def found(t, r, diff=True): c = found(t, r) check('a test file named twice is counted once', c and c[0]['tests_ran'] == 1 and c[0]['tests'] == ['tests/test_alpha.py'], c) +check('a test runner config beside the tests is not a test file', not A.TEST_FILE.search('test/jest.config.js') + and A.TEST_FILE.search('test/registry.test.ts') and A.TEST_FILE.search('tests/test_alpha.py')) + # --- a test file the agent already knew of is not picked by an impact answer; a tests answer picks what it names r = repo() IMPACT_T = IMPACT_A + '\ntests:\n tests/test_alpha.py:5 test_run [test]' @@ -251,6 +258,9 @@ def found(t, r, diff=True): r = repo(); change(r, 'src/pkg/beta.py'); change(r, 'src/pkg/gamma.py') t = (T(r).prompt('fix `Beta.go` so it stops').call(MCP, {'name': 'Alpha.run'}, ROWS) .edit('src/pkg/beta.py').edit('src/pkg/gamma.py')) +r2 = repo(); change(r2, 'src/pkg/beta.py') +t2 = T(r2).prompt('`getBeta` returns the wrong value').call(MCP, {'name': 'Alpha.run'}, '1. src/pkg/beta.py:3 [resolved · getBeta]').edit('src/pkg/beta.py') +check('a camelCase name the task gave is the task\'s lead too', found(t2, r2) == [], found(t2, r2)) check('a row about a name the task gave is not credited; the other row is', [x['edited'] for x in found(t, r)] == [['src/pkg/gamma.py']], found(t, r)) r = repo(); change(r, 'src/pkg/beta.py') t = (T(r).prompt('add a flag to `Alpha.run`').call(MCP, {'name': 'Alpha.run'}, @@ -293,7 +303,7 @@ def found(t, r, diff=True): msg = json.loads(h.stdout).get('systemMessage', '') if h.stdout.strip() else '' check('the Stop hook prints the receipt as a message for the user', h.returncode == 0 and msg.startswith('◆ AxiomCode') and 'Helped' in msg, (h.returncode, h.stdout, h.stderr)) -check('it counts the answers that led the work, never those that did not', '1 answer led' in msg and ' of ' not in msg.splitlines()[0], msg) +check('it counts the answers that led the work, never those that did not', 'acted on 1 answer' in msg and ' of ' not in msg.splitlines()[0], msg) # --- two answers, one acted on: the count says so; a later turn that only grows an earlier chain says that instead r = repo(); change(r, 'src/pkg/alpha.py') @@ -302,10 +312,12 @@ def found(t, r, diff=True): first = A.at_stop(t.save(), r, 's4') t.prompt('now fix it').edit('src/pkg/alpha.py') later = A.at_stop(t.save(), r, 's4') -check('one of two answers acted on counts only the one', '1 answer led' in first and ' of 2' not in first, first) +check('one of two answers acted on counts only the one', 'acted on 1 answer' in first and ' of 2' not in first, first) check('a turn that only grows an earlier chain names it as earlier', 'earlier answer went further' in later and 'Helped' in later, later) page = open(A.report(r)[0]).read() -check('the report counts answers asked and the files and tests behind them', ' of 2' not in page and 'edited' in page, page[:300]) +check('the report has no unfilled template text', '\\1' not in page and "{'" not in page and "'}" not in page and ' if ' not in page, page[:300]) +check('the report counts answers asked and the files and tests behind them', ' of 2' not in page and 'Previously unseen files edited' in page and 'edited' in page + and 'Featured evidence' in page and 'Measured from agent actions' in page, page[:300]) ev['session_id'] = 's3' h = subprocess.run([sys.executable, os.path.join(HOOKS, 'assist.py')], input=json.dumps(ev), capture_output=True, text=True, env=dict(os.environ, AXIOMCODE_ASSISTS='off')) From e6f9d6803f44485a34e410a1836ea34f7a0a18c9 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:49:47 -0700 Subject: [PATCH 06/12] plugin: the assists page rebuilds itself whenever a turn adds to the record, and the receipt links it (#1935) Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- plugins/axiomcode/skills/axiomcode/SKILL.md | 3 ++- .../axiomcode/skills/axiomcode/scripts/ax_assists.py | 11 +++++++++-- skills/axiomcode/SKILL.md | 3 ++- tests/assists.py | 1 + 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/plugins/axiomcode/skills/axiomcode/SKILL.md b/plugins/axiomcode/skills/axiomcode/SKILL.md index 19c7057fd..fba3cebb4 100644 --- a/plugins/axiomcode/skills/axiomcode/SKILL.md +++ b/plugins/axiomcode/skills/axiomcode/SKILL.md @@ -128,7 +128,8 @@ not found this way. At the end of each turn the plugin prints, under your reply, where an answer verifiably led your work: a file the answer showed first that you then read or edited, or the tests it picked that you ran (Redirected / Helped / Useful). It is read from the session's tool calls, not from what you write, so there is nothing to claim yourself. -`axiomcode report` renders every recorded chain as one page, `.axiomcode/assists.html`. +Each repository keeps its own record (`.axiomcode/assists.jsonl` at its root) and its own page, +`.axiomcode/assists.html`, rebuilt whenever a turn adds to the record; `axiomcode report` rebuilds it by hand. ## What it cannot see — say so instead of guessing diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py index eb5e934de..e600ca057 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py @@ -660,8 +660,15 @@ def at_stop(transcript, root, session): return '' led = {c['answer'] for c in found} now = [i for t, i in q if t == last] - return receipt(fresh, totals(recorded + [dict(c, session=session) for c in fresh], session), - turn=(len(now), len(led & set(now)), now), session=(len(q), len(led))) + msg = receipt(fresh, totals(recorded + [dict(c, session=session) for c in fresh], session), + turn=(len(now), len(led & set(now)), now), session=(len(q), len(led))) + try: + # the page is rebuilt whenever the ledger grows, so it is current without anyone asking for it + page, _ = report(root) + msg += f'\n report: file://{page}' + except Exception: + pass + return msg # ---- the report ----------------------------------------------------------------------------------------------------- diff --git a/skills/axiomcode/SKILL.md b/skills/axiomcode/SKILL.md index f512635b6..6a94665e1 100644 --- a/skills/axiomcode/SKILL.md +++ b/skills/axiomcode/SKILL.md @@ -128,7 +128,8 @@ not found this way. At the end of each turn the plugin prints, under your reply, where an answer verifiably led your work: a file the answer showed first that you then read or edited, or the tests it picked that you ran (Redirected / Helped / Useful). It is read from the session's tool calls, not from what you write, so there is nothing to claim yourself. -`axiomcode report` renders every recorded chain as one page, `.axiomcode/assists.html`. +Each repository keeps its own record (`.axiomcode/assists.jsonl` at its root) and its own page, +`.axiomcode/assists.html`, rebuilt whenever a turn adds to the record; `axiomcode report` rebuilds it by hand. ## What it cannot see — say so instead of guessing diff --git a/tests/assists.py b/tests/assists.py index b80dd0e06..fce459115 100644 --- a/tests/assists.py +++ b/tests/assists.py @@ -303,6 +303,7 @@ def found(t, r, diff=True): msg = json.loads(h.stdout).get('systemMessage', '') if h.stdout.strip() else '' check('the Stop hook prints the receipt as a message for the user', h.returncode == 0 and msg.startswith('◆ AxiomCode') and 'Helped' in msg, (h.returncode, h.stdout, h.stderr)) +check('the Stop hook rebuilds the report and links it', 'report: file://' in msg and os.path.isfile(os.path.join(r, '.axiomcode', 'assists.html')), msg) check('it counts the answers that led the work, never those that did not', 'acted on 1 answer' in msg and ' of ' not in msg.splitlines()[0], msg) # --- two answers, one acted on: the count says so; a later turn that only grows an earlier chain says that instead From af4da877e46e890214dcc581338e98991fc89e30 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:20:03 -0700 Subject: [PATCH 07/12] plugin: the agent ends its reply with the assist receipt, and calls sent with a query are not credited to it (#1935) The Stop hook hands the receipt back to the agent once (decision "block"), which ends its reply with it, so it is part of the answer in every client. The second stop checks the reply carries every line and shows it as a message when it does not. Gemini keeps the message. A file the agent read in the same message as a query was named before the answer came back; it is no longer credited to that answer. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- plugins/axiomcode/hooks/assist.py | 58 +++++++++++++++++- plugins/axiomcode/skills/axiomcode/SKILL.md | 7 ++- .../skills/axiomcode/scripts/ax_assists.py | 20 +++++-- skills/axiomcode/SKILL.md | 7 ++- tests/assists.py | 59 +++++++++++++++---- 5 files changed, 127 insertions(+), 24 deletions(-) diff --git a/plugins/axiomcode/hooks/assist.py b/plugins/axiomcode/hooks/assist.py index b02ada2fa..1660571c8 100644 --- a/plugins/axiomcode/hooks/assist.py +++ b/plugins/axiomcode/hooks/assist.py @@ -13,10 +13,13 @@ calls and their results, and a link that was not checked is not printed. · SILENT unless an answer led somewhere: no query, or a query nobody acted on, prints nothing. Only what helped is counted; the answers that led nowhere are never shown. - · NEVER BLOCKS. It prints a message for the user and exits 0; the agent's turn is over. + · THE AGENT QUOTES IT. In Claude Code the receipt goes back to the agent once (decision "block"), which ends its + reply with it word for word, so it is part of the answer in every client (`-p`, the SDK, an IDE). The second stop + (stop_hook_active) checks the reply carries every line; when it does not, the receipt is shown as a message + instead. Cursor and Gemini cannot hand it back, so they get the message. AXIOMCODE_ASSISTS=off turns it off. """ -import json, os, subprocess, sys +import json, os, subprocess, sys, time sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'skills', 'axiomcode', 'scripts')) sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import _host @@ -38,10 +41,59 @@ root = r.stdout.strip() if r.returncode == 0 and r.stdout.strip() else cwd except Exception: root = cwd +ASK = ('AxiomCode checked the tool calls of this turn and recorded where its answers led your work. End your reply with ' + 'the block below, copied exactly as written (inside a ``` block), and nothing else: run no tools, add no claims ' + 'of your own about it.\n\n') + + +def _pending(root): + return os.path.join(root, '.axiomcode', 'assists.pending.json') + + +def _quoted(transcript, msg): + """whether the agent's text after the hand-back carries every line of the receipt""" + want = [l.strip() for l in msg.splitlines() if l.strip()] + for _ in range(6): # the host may still be writing the reply's last message + said, after = [], False + for line in open(transcript, encoding='utf-8', errors='replace'): + try: + e = json.loads(line) + except ValueError: + continue + c = (e.get('message') or {}).get('content') if isinstance(e, dict) else None + if e.get('type') == 'user' and ASK[:40] in json.dumps(c, ensure_ascii=False): + said, after = [], True + elif after and e.get('type') == 'assistant' and isinstance(c, list): + said += [x.get('text') or '' for x in c if isinstance(x, dict) and x.get('type') == 'text'] + text = '\n'.join(said) + if all(w in text for w in want): + return True + time.sleep(0.5) + return False + + +if ev.get('stop_hook_active'): + # the agent was handed the receipt; show it as a message only when its reply did not carry it + try: + pend = json.load(open(_pending(root))) + os.remove(_pending(root)) + except Exception: + sys.exit(0) + if pend.get('session') == str(ev.get('session_id') or '') and not _quoted(tr, pend['msg']): + print(json.dumps({'systemMessage': pend['msg']})) + sys.exit(0) try: import ax_assists msg = ax_assists.at_stop(tr, os.path.realpath(root), str(ev.get('session_id') or '')) except Exception: sys.exit(0) # a receipt is never worth a failed turn -if msg: +if not msg: + sys.exit(0) +if _host.GEMINI: + print(json.dumps({'systemMessage': msg})) + sys.exit(0) +try: + json.dump({'session': str(ev.get('session_id') or ''), 'msg': msg}, open(_pending(root), 'w')) + print(json.dumps({'decision': 'block', 'reason': ASK + msg})) +except Exception: print(json.dumps({'systemMessage': msg})) diff --git a/plugins/axiomcode/skills/axiomcode/SKILL.md b/plugins/axiomcode/skills/axiomcode/SKILL.md index fba3cebb4..b18ed7031 100644 --- a/plugins/axiomcode/skills/axiomcode/SKILL.md +++ b/plugins/axiomcode/skills/axiomcode/SKILL.md @@ -125,9 +125,10 @@ not found this way. ## report -At the end of each turn the plugin prints, under your reply, where an answer verifiably led your work: a file the -answer showed first that you then read or edited, or the tests it picked that you ran (Redirected / Helped / Useful). -It is read from the session's tool calls, not from what you write, so there is nothing to claim yourself. +At the end of a turn where an answer verifiably led your work (a file the answer showed first that you then read or +edited, or the tests it picked that you ran: Redirected / Helped / Useful), the plugin hands you a receipt once. End +your reply with it exactly as written. It is read from the session's tool calls, not from what you write, so there is +nothing to claim yourself; if the reply does not carry it, the user is shown it anyway. Each repository keeps its own record (`.axiomcode/assists.jsonl` at its root) and its own page, `.axiomcode/assists.html`, rebuilt whenever a turn adds to the record; `axiomcode report` rebuilds it by hand. diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py index e600ca057..0fe4d44d6 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py @@ -170,8 +170,9 @@ def steps(transcript): if x.get('type') == 'text' and x.get('text'): out.append({'kind': 'reply', 'text': x['text'], 'ts': ts}) elif x.get('type') == 'tool_use': + # mid: the calls of one message go out together, before any of their answers is seen s = {'kind': 'call', 'id': x.get('id'), 'name': x.get('name') or '', 'input': x.get('input') or {}, - 'result': '', 'error': False, 'ts': ts} + 'result': '', 'error': False, 'ts': ts, 'mid': msg.get('id')} calls[s['id']] = s out.append(s) return out @@ -322,6 +323,12 @@ def _hit(f, files): # ---- the chains ----------------------------------------------------------------------------------------------------- +def _batch(ss, i): + """the call at i and the calls sent in the same message""" + m = ss[i].get('mid') + return [x for x in ss if x['kind'] == 'call' and x.get('mid') == m] if m else [ss[i]] + + def chains(ss, root, diff=None): """every chain the session holds. `diff`: the files changed in the working tree now (None: not known, an edit counts as kept).""" @@ -341,7 +348,8 @@ def chains(ss, root, diff=None): if a: answers[i] = a verb = a[0] - for f in paths_in(json.dumps(s['input'], ensure_ascii=False), root): + # what this call and the calls sent with it name, the agent named before it saw the answer + for f in paths_in(' '.join(json.dumps(x['input'], ensure_ascii=False) for x in _batch(ss, i)), root): seen_full.add(f) seen_base.add(f.rsplit('/', 1)[-1]) res = s['result'] or '' @@ -373,6 +381,8 @@ def chains(ss, root, diff=None): seen_base.add(f.rsplit('/', 1)[-1]) acts = [(i, k, fs) for i, s in enumerate(ss) for k, fs in touched(s, root)] + # j acted on the answer at i only when it was sent after i's answer came back, not alongside the call + sent_after = lambda j, i: j > i and not (ss[i].get('mid') and ss[j].get('mid') == ss[i].get('mid')) edits_at = {j for j, k, fs in acts if k == 'edit'} ids = lambda js: [ss[j]['id'] for j in js] out = [] @@ -387,8 +397,8 @@ def chains(ss, root, diff=None): tests_named = {f for f in tests_named if not any(g != f and g.endswith('/' + f) for g in tests_named)} read, edited, ev_read, ev_edit = [], [], [], [] for f in shown: - r = [j for j, k, fs in acts if k == 'read' and j > i and _hit(f, fs)] - e = [j for j, k, fs in acts if k == 'edit' and j > i and _hit(f, fs)] + r = [j for j, k, fs in acts if k == 'read' and sent_after(j, i) and _hit(f, fs)] + e = [j for j, k, fs in acts if k == 'edit' and sent_after(j, i) and _hit(f, fs)] kept = diff is None or _hit(f, diff) if e and kept: edited.append(f); ev_edit += e; ev_read += r @@ -397,7 +407,7 @@ def chains(ss, root, diff=None): # the answer's picked tests: a later test command that names one of them ran, passed, ev_test = set(), set(), [] for j, k, fs in acts: - if k == 'test' and j > i: + if k == 'test' and sent_after(j, i): hit = {t for t in tests_named if _hit(t, fs) and (j, t) not in claimed} claimed |= {(j, t) for t in hit} if hit: diff --git a/skills/axiomcode/SKILL.md b/skills/axiomcode/SKILL.md index 6a94665e1..5a6c6db1c 100644 --- a/skills/axiomcode/SKILL.md +++ b/skills/axiomcode/SKILL.md @@ -125,9 +125,10 @@ not found this way. ## report -At the end of each turn the plugin prints, under your reply, where an answer verifiably led your work: a file the -answer showed first that you then read or edited, or the tests it picked that you ran (Redirected / Helped / Useful). -It is read from the session's tool calls, not from what you write, so there is nothing to claim yourself. +At the end of a turn where an answer verifiably led your work (a file the answer showed first that you then read or +edited, or the tests it picked that you ran: Redirected / Helped / Useful), the plugin hands you a receipt once. End +your reply with it exactly as written. It is read from the session's tool calls, not from what you write, so there is +nothing to claim yourself; if the reply does not carry it, the user is shown it anyway. Each repository keeps its own record (`.axiomcode/assists.jsonl` at its root) and its own page, `.axiomcode/assists.html`, rebuilt whenever a turn adds to the record; `axiomcode report` rebuilds it by hand. diff --git a/tests/assists.py b/tests/assists.py index fce459115..cff156dd9 100644 --- a/tests/assists.py +++ b/tests/assists.py @@ -35,11 +35,13 @@ def prompt(self, text): self.lines.append({'type': 'user', 'message': {'role': 'user', 'content': text}}) return self - def call(self, name, inp, result='', error=False, side=False): + def call(self, name, inp, result='', error=False, side=False, mid=None): + """mid: a message id shared with other calls, for calls sent together in one message""" self.n += 1 tid = f'toolu_{self.n:03d}' self.lines.append({'type': 'assistant', 'isSidechain': side, - 'message': {'role': 'assistant', 'content': [{'type': 'tool_use', 'id': tid, 'name': name, 'input': inp}]}}) + 'message': {'role': 'assistant', 'id': mid or f'msg_{self.n:03d}', + 'content': [{'type': 'tool_use', 'id': tid, 'name': name, 'input': inp}]}}) self.lines.append({'type': 'user', 'isSidechain': side, 'message': {'role': 'user', 'content': [{'type': 'tool_result', 'tool_use_id': tid, 'content': result, 'is_error': error}]}}) @@ -295,16 +297,55 @@ def found(t, r, diff=True): page = open(out).read() check('the report renders the session\'s chains', n == 1 and 'Helped' in page and 'impact Alpha.run' in page, (out, n)) -# --- the hook end to end: a Stop event in, a message for the user out; switched off, nothing +# --- calls sent in one message go out before any answer comes back: a file read alongside the query was not shown by it +r = repo(); change(r, 'src/pkg/alpha.py') +t = (T(r).prompt('fix the bug').call(MCP, {'name': 'Alpha.run'}, IMPACT_A, mid='m1') + .call('Bash', {'command': 'cat src/pkg/alpha.py'}, 'contents', mid='m1').edit('src/pkg/alpha.py')) +got = A.chains(A.steps(t.save()), r) +check('a file read in the same message as the query is not credited to its answer', not got, got) +t = (T(r).prompt('fix the bug').call(MCP, {'name': 'Alpha.run'}, IMPACT_A, mid='m1') + .call('Bash', {'command': 'cat src/pkg/alpha.py'}, 'contents', mid='m2').edit('src/pkg/alpha.py')) +got = A.chains(A.steps(t.save()), r) +check('control: the same read in the next message is', [c['label'] for c in got] == ['Helped'], got) + +# --- the hook end to end: a Stop event in, the receipt handed back to the agent, which ends its reply with it; when +# the reply does not carry it, the second stop shows it as a message instead; switched off, nothing +def hook(ev, **env): + h = subprocess.run([sys.executable, os.path.join(HOOKS, 'assist.py')], input=json.dumps(ev), capture_output=True, + text=True, env=dict(os.environ, **env)) + return h, (json.loads(h.stdout) if h.stdout.strip() else {}) + + +def handed_back(t, reason, reply): + t.lines.append({'type': 'user', 'isMeta': True, 'message': {'role': 'user', 'content': 'Stop hook feedback:\n' + reason}}) + t.lines.append({'type': 'assistant', 'message': {'role': 'assistant', 'content': [{'type': 'text', 'text': reply}]}}) + return t.save() + + r = repo(); change(r, 'src/pkg/alpha.py') t = T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, IMPACT_A).edit('src/pkg/alpha.py') ev = {'hook_event_name': 'Stop', 'session_id': 's2', 'transcript_path': t.save(), 'cwd': r, 'stop_hook_active': False} -h = subprocess.run([sys.executable, os.path.join(HOOKS, 'assist.py')], input=json.dumps(ev), capture_output=True, text=True) -msg = json.loads(h.stdout).get('systemMessage', '') if h.stdout.strip() else '' -check('the Stop hook prints the receipt as a message for the user', h.returncode == 0 and msg.startswith('◆ AxiomCode') and 'Helped' in msg, - (h.returncode, h.stdout, h.stderr)) +h, out = hook(ev) +reason = out.get('reason', '') +msg = reason[reason.find('◆ AxiomCode'):] +check('the Stop hook hands the receipt back to the agent to end its reply with', h.returncode == 0 and out.get('decision') == 'block' + and msg.startswith('◆ AxiomCode') and 'Helped' in msg and 'exactly as written' in reason, (h.returncode, h.stdout, h.stderr)) check('the Stop hook rebuilds the report and links it', 'report: file://' in msg and os.path.isfile(os.path.join(r, '.axiomcode', 'assists.html')), msg) check('it counts the answers that led the work, never those that did not', 'acted on 1 answer' in msg and ' of ' not in msg.splitlines()[0], msg) +h2, out2 = hook(dict(ev, stop_hook_active=True, transcript_path=handed_back(t, reason, 'Done.\n\n```\n' + msg + '\n```'))) +check('a reply that quotes the receipt ends the turn with nothing more', h2.returncode == 0 and out2 == {}, h2.stdout) +check('the hand-back is not a prompt, so a later turn still credits by the task alone', + [s['kind'] for s in A.steps(t.save())].count('prompt') == 1, A.steps(t.save())) +r = repo(); change(r, 'src/pkg/alpha.py') +t = T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, IMPACT_A).edit('src/pkg/alpha.py') +ev = dict(ev, session_id='s5', transcript_path=t.save(), cwd=r) +h, out = hook(ev) +reason = out.get('reason', '') +msg = reason[reason.find('◆ AxiomCode'):] +h2, out2 = hook(dict(ev, stop_hook_active=True, transcript_path=handed_back(t, reason, 'AxiomCode helped a lot here.'))) +check('a reply that drops or rewrites the receipt gets it shown as a message instead', out2.get('systemMessage') == msg, (h2.stdout, msg)) +h3, out3 = hook(dict(ev, stop_hook_active=True)) +check('the second stop happens once: a third is silent', out3 == {}, h3.stdout) # --- two answers, one acted on: the count says so; a later turn that only grows an earlier chain says that instead r = repo(); change(r, 'src/pkg/alpha.py') @@ -319,9 +360,7 @@ def found(t, r, diff=True): check('the report has no unfilled template text', '\\1' not in page and "{'" not in page and "'}" not in page and ' if ' not in page, page[:300]) check('the report counts answers asked and the files and tests behind them', ' of 2' not in page and 'Previously unseen files edited' in page and 'edited' in page and 'Featured evidence' in page and 'Measured from agent actions' in page, page[:300]) -ev['session_id'] = 's3' -h = subprocess.run([sys.executable, os.path.join(HOOKS, 'assist.py')], input=json.dumps(ev), capture_output=True, text=True, - env=dict(os.environ, AXIOMCODE_ASSISTS='off')) +h, _ = hook(dict(ev, session_id='s3'), AXIOMCODE_ASSISTS='off') check('AXIOMCODE_ASSISTS=off prints nothing', h.returncode == 0 and not h.stdout.strip(), h.stdout) h = subprocess.run([sys.executable, os.path.join(HOOKS, 'assist.py')], input='{"hook_event_name":"Stop"}', capture_output=True, text=True) check('a Stop event with no transcript is silent and never fails', h.returncode == 0 and not h.stdout.strip(), (h.returncode, h.stdout)) From 1c6254e7404d8dc6e74015586efd30cd18f88303 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:26:33 -0700 Subject: [PATCH 08/12] plugin: a picked test passes only when the runner says so, on its latest run (#1935) A test line exits 0 when it ends in `| tail` or `; ruff check`, including a run that never started (a missing module, a collection error, every test deselected), so exit 0 was counted as a pass. A pass now needs the runner's own pass line with no failure or error beside it, and a test file passes when its latest run does. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- .../axiomcode/skills/axiomcode/scripts/ax_assists.py | 11 +++++++++-- tests/assists.py | 12 ++++++++++++ tests/surfaces.py | 2 +- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py index 0fe4d44d6..15f32d3c8 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py @@ -58,6 +58,10 @@ def search(f): r'dotnet\s+test|mvn\b[^;&|]*\b(?:test|verify)\b|mvnw\b[^;&|]*\b(?:test|verify)\b|gradlew?\b[^;&|]*\btest\b|' r'(?:npm|pnpm|yarn|bun)\s+(?:run\s+)?test\b|python3?\s+-m\s+(?:pytest|unittest))') FAILED = re.compile(r'\b[1-9]\d* (?:failed|failing)\b|Tests:\s+[1-9]\d* failed|\bFAIL\s') +# a pass is the runner saying so (pytest, jest, vitest, mocha, unittest, go, dotnet, maven); an exit 0 alone is not one, +# since a test line often ends in `| tail` or `; ruff check`, and a run that never started exits 0 there too +PASSED = re.compile(r'\b[1-9]\d* (?:passed|passing)\b|^\s*PASS\s|^OK\b|^ok\s|\bPassed!|Tests run: [1-9]\d*, Failures: 0, Errors: 0', re.M) +BROKE = re.compile(r'\b[1-9]\d* errors?\b|\bno tests ran\b|ERROR collecting|ModuleNotFoundError|ImportError while') SHELL_READ = re.compile(r'(?:^|[;&|(]|\n|&&)\s*(?:cat|sed\s+-n\s+\S+|head(?:\s+-n?\s*\d+)?|tail(?:\s+-n?\s*\d+)?|less|bat|nl|' r'grep\s+-n\S*\s+\S+|rg\s+-n\S*\s+\S+)\s+([^;&|<>\n]+)') # the command, then its file arguments EDIT_TOOLS = {'Edit', 'Write', 'MultiEdit', 'NotebookEdit', 'replace', 'write_file'} @@ -413,8 +417,11 @@ def chains(ss, root, diff=None): if hit: ev_test.append(j) ran |= hit - if not ss[j]['error'] and not FAILED.search(ss[j]['result'] or ''): - passed |= hit # a run that exits 0 but reports failures (a before/after script) is no pass + said = ss[j]['result'] or '' + if not ss[j]['error'] and PASSED.search(said) and not FAILED.search(said) and not BROKE.search(said): + passed |= hit + else: + passed -= hit # a test file passes when its latest run does passed_all = bool(ran) and ran == passed if edited or (ran and passed_all and verb in ('tests', 'test-impact')): label = 'Helped' diff --git a/tests/assists.py b/tests/assists.py index cff156dd9..60f7535aa 100644 --- a/tests/assists.py +++ b/tests/assists.py @@ -160,6 +160,18 @@ def found(t, r, diff=True): .call('Bash', {'command': 'pytest tests/test_alpha.py -q; echo done'}, '1 failed, 3 passed\ndone')) check('a run that exits 0 but reports failures is not a pass', [x['label'] for x in found(t, r)] == ['Useful'], found(t, r)) check('picked tests that failed are only Useful', [x['label'] for x in found(t, r)] == ['Useful'], found(t, r)) +PICK = lambda r: T(r).prompt('which tests cover my change').call('mcp__plugin_axiomcode_axiomcode__tests', {}, TESTS_A) +r = repo() +t = PICK(r).call('Bash', {'command': 'pytest tests/test_alpha.py -q 2>&1 | tail -3; ruff check .'}, + "ModuleNotFoundError: No module named 'trio'\nAll checks passed!") +check('a run that never started is no pass, though the line exits 0 and something else says passed', + [x['label'] for x in found(t, r)] == ['Useful'], found(t, r)) +t = (PICK(repo()).call('Bash', {'command': 'pytest tests/test_alpha.py -q'}, '1 failed, 2 passed') + .call('Bash', {'command': 'pytest tests/test_alpha.py -q'}, '3 passed')) +check('a test file passes when its latest run does: failed, then fixed', [x['label'] for x in found(t, t.root)] == ['Helped'], found(t, t.root)) +t = (PICK(repo()).call('Bash', {'command': 'pytest tests/test_alpha.py -q'}, '3 passed') + .call('Bash', {'command': 'pytest tests/test_alpha.py -q'}, '1 failed, 2 passed')) +check('passed, then broken, is no pass', [x['label'] for x in found(t, t.root)] == ['Useful'], found(t, t.root)) # --- a test file named twice in one answer (bare, and with its directory) is one test file r = repo() diff --git a/tests/surfaces.py b/tests/surfaces.py index 9674cea67..4e26b675a 100644 --- a/tests/surfaces.py +++ b/tests/surfaces.py @@ -31,7 +31,7 @@ PUBLIC = ['index', 'impact', 'path', 'tests', 'context', 'link', 'report'] NO_MCP = {'index': 'setup, not a question: the first query through the MCP server builds the graph itself', - 'report': 'a page for the person, not a question for the agent: the end-of-turn receipts already reach both'} + 'report': 'a page for the user, not a question for the agent: the end-of-turn receipt already reaches both'} # dispatched, not advertised: verb -> why INTERNAL = { 'build': 'the old name of index', From b39c043e99c09154f9b33833292f7688ff8860c4 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:29:52 -0700 Subject: [PATCH 09/12] plugin: a test run is credited to the latest answer that picked its tests (#1935) A follow-up turn that asks `tests` again and runs what it picked was credited to the first turn's answer, so the turn that acted on it printed nothing. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- .../skills/axiomcode/scripts/ax_assists.py | 28 +++++++++++++------ tests/assists.py | 7 +++++ 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py index 15f32d3c8..4ec62ae75 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py @@ -390,15 +390,28 @@ def chains(ss, root, diff=None): edits_at = {j for j, k, fs in acts if k == 'edit'} ids = lambda js: [ss[j]['id'] for j in js] out = [] - claimed = set() # (test run, test file): a run is credited to the first answer that picked it - for i, (verb, arg) in sorted(answers.items()): - shown = sorted(f for f, j in first.items() if j == i) - named = paths_in(ss[i]['result'], root) + def picked(i): # the tests an answer picked: every test file a test verb names (picking them is its job); for the other verbs # only a test file first seen in it, since a test the agent already knew of was not picked by the answer - tests_named = {f for f in named if TEST_FILE.search(f) and (verb in ('tests', 'test-impact') or f in shown)} + verb = answers[i][0] + t = {f for f in paths_in(ss[i]['result'], root) + if TEST_FILE.search(f) and (verb in ('tests', 'test-impact') or first.get(f) == i)} # one file named twice (`test_x.py` in a run line, `tests/test_x.py` in a row) is one test file - tests_named = {f for f in tests_named if not any(g != f and g.endswith('/' + f) for g in tests_named)} + return {f for f in t if not any(g != f and g.endswith('/' + f) for g in t)} + picks = {i: picked(i) for i in answers} + # (test run, test file) -> the answer it is credited to: the latest one that picked the file before the run + owner = {} + for j, k, fs in acts: + if k == 'test': + for i in sorted(answers): + if sent_after(j, i): + for t in picks[i]: + if _hit(t, fs): + owner[(j, t)] = i + for i, (verb, arg) in sorted(answers.items()): + shown = sorted(f for f, j in first.items() if j == i) + tests_named = picks[i] + named = paths_in(ss[i]['result'], root) read, edited, ev_read, ev_edit = [], [], [], [] for f in shown: r = [j for j, k, fs in acts if k == 'read' and sent_after(j, i) and _hit(f, fs)] @@ -412,8 +425,7 @@ def chains(ss, root, diff=None): ran, passed, ev_test = set(), set(), [] for j, k, fs in acts: if k == 'test' and sent_after(j, i): - hit = {t for t in tests_named if _hit(t, fs) and (j, t) not in claimed} - claimed |= {(j, t) for t in hit} + hit = {t for t in tests_named if _hit(t, fs) and owner.get((j, t)) == i} if hit: ev_test.append(j) ran |= hit diff --git a/tests/assists.py b/tests/assists.py index 60f7535aa..35d54a27e 100644 --- a/tests/assists.py +++ b/tests/assists.py @@ -172,6 +172,13 @@ def found(t, r, diff=True): t = (PICK(repo()).call('Bash', {'command': 'pytest tests/test_alpha.py -q'}, '3 passed') .call('Bash', {'command': 'pytest tests/test_alpha.py -q'}, '1 failed, 2 passed')) check('passed, then broken, is no pass', [x['label'] for x in found(t, t.root)] == ['Useful'], found(t, t.root)) +TA = 'mcp__plugin_axiomcode_axiomcode__tests' +t = (PICK(repo()).prompt('one more change').call(TA, {}, TESTS_A).call('Bash', {'command': 'pytest tests/test_alpha.py -q'}, '3 passed')) +c = found(t, t.root) +check('a run goes to the latest answer that picked its tests', [(x['answer'], x['label']) for x in c] == [('toolu_002', 'Helped')], c) +t = PICK(repo()).call('Bash', {'command': 'pytest tests/test_alpha.py -q'}, '3 passed').prompt('again').call(TA, {}, TESTS_A) +c = found(t, t.root) +check('control: a run before the second answer stays with the first', [(x['answer'], x['label']) for x in c] == [('toolu_001', 'Helped')], c) # --- a test file named twice in one answer (bare, and with its directory) is one test file r = repo() From d734ff740017092aa5b8c3ad8a03bf1f221f2fa0 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Mon, 12 Oct 2026 14:18:48 -0700 Subject: [PATCH 10/12] plugin: the terminal shows the assist receipt once (#1935) The terminal shows a Stop hook's hand-back reason as well as the reply, so a receipt handed back to the agent there was printed twice. In the terminal it is a message under the reply again; headless runs and IDEs, where a hook message may never reach the user, still have the agent end its reply with it. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- plugins/axiomcode/hooks/assist.py | 12 +++++++----- plugins/axiomcode/skills/axiomcode/SKILL.md | 7 ++++--- skills/axiomcode/SKILL.md | 7 ++++--- tests/assists.py | 8 +++++++- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/plugins/axiomcode/hooks/assist.py b/plugins/axiomcode/hooks/assist.py index 1660571c8..ffd16e974 100644 --- a/plugins/axiomcode/hooks/assist.py +++ b/plugins/axiomcode/hooks/assist.py @@ -13,10 +13,12 @@ calls and their results, and a link that was not checked is not printed. · SILENT unless an answer led somewhere: no query, or a query nobody acted on, prints nothing. Only what helped is counted; the answers that led nowhere are never shown. - · THE AGENT QUOTES IT. In Claude Code the receipt goes back to the agent once (decision "block"), which ends its - reply with it word for word, so it is part of the answer in every client (`-p`, the SDK, an IDE). The second stop - (stop_hook_active) checks the reply carries every line; when it does not, the receipt is shown as a message - instead. Cursor and Gemini cannot hand it back, so they get the message. + · SHOWN ONCE. In the terminal (CLAUDE_CODE_ENTRYPOINT=cli) it is a message under the reply: the terminal shows + a Stop hook's message, and would also show a hand-back's reason, so handing it back there prints it twice. + Elsewhere (`-p`, the SDK, an IDE) a hook's message may never reach the user, so the receipt goes back to the + agent once (decision "block"), which ends its reply with it word for word. The second stop (stop_hook_active) + checks the reply carries every line; when it does not, the receipt is shown as a message instead. Gemini gets + the message. AXIOMCODE_ASSISTS=off turns it off. """ import json, os, subprocess, sys, time @@ -89,7 +91,7 @@ def _quoted(transcript, msg): sys.exit(0) # a receipt is never worth a failed turn if not msg: sys.exit(0) -if _host.GEMINI: +if _host.GEMINI or os.environ.get('CLAUDE_CODE_ENTRYPOINT') == 'cli': print(json.dumps({'systemMessage': msg})) sys.exit(0) try: diff --git a/plugins/axiomcode/skills/axiomcode/SKILL.md b/plugins/axiomcode/skills/axiomcode/SKILL.md index b18ed7031..531b28fec 100644 --- a/plugins/axiomcode/skills/axiomcode/SKILL.md +++ b/plugins/axiomcode/skills/axiomcode/SKILL.md @@ -126,9 +126,10 @@ not found this way. ## report At the end of a turn where an answer verifiably led your work (a file the answer showed first that you then read or -edited, or the tests it picked that you ran: Redirected / Helped / Useful), the plugin hands you a receipt once. End -your reply with it exactly as written. It is read from the session's tool calls, not from what you write, so there is -nothing to claim yourself; if the reply does not carry it, the user is shown it anyway. +edited, or the tests it picked that you ran: Redirected / Helped / Useful), the plugin shows the user a receipt. In +the terminal it appears under your reply; elsewhere (headless, an IDE) it is handed to you once: end your reply with +it exactly as written. It is read from the session's tool calls, not from what you write, so there is nothing to claim +yourself. Each repository keeps its own record (`.axiomcode/assists.jsonl` at its root) and its own page, `.axiomcode/assists.html`, rebuilt whenever a turn adds to the record; `axiomcode report` rebuilds it by hand. diff --git a/skills/axiomcode/SKILL.md b/skills/axiomcode/SKILL.md index 5a6c6db1c..001ec0453 100644 --- a/skills/axiomcode/SKILL.md +++ b/skills/axiomcode/SKILL.md @@ -126,9 +126,10 @@ not found this way. ## report At the end of a turn where an answer verifiably led your work (a file the answer showed first that you then read or -edited, or the tests it picked that you ran: Redirected / Helped / Useful), the plugin hands you a receipt once. End -your reply with it exactly as written. It is read from the session's tool calls, not from what you write, so there is -nothing to claim yourself; if the reply does not carry it, the user is shown it anyway. +edited, or the tests it picked that you ran: Redirected / Helped / Useful), the plugin shows the user a receipt. In +the terminal it appears under your reply; elsewhere (headless, an IDE) it is handed to you once: end your reply with +it exactly as written. It is read from the session's tool calls, not from what you write, so there is nothing to claim +yourself. Each repository keeps its own record (`.axiomcode/assists.jsonl` at its root) and its own page, `.axiomcode/assists.html`, rebuilt whenever a turn adds to the record; `axiomcode report` rebuilds it by hand. diff --git a/tests/assists.py b/tests/assists.py index 35d54a27e..753af4d13 100644 --- a/tests/assists.py +++ b/tests/assists.py @@ -330,8 +330,9 @@ def found(t, r, diff=True): # --- the hook end to end: a Stop event in, the receipt handed back to the agent, which ends its reply with it; when # the reply does not carry it, the second stop shows it as a message instead; switched off, nothing def hook(ev, **env): + env = dict(os.environ, **dict({'CLAUDE_CODE_ENTRYPOINT': 'sdk-cli'}, **env)) h = subprocess.run([sys.executable, os.path.join(HOOKS, 'assist.py')], input=json.dumps(ev), capture_output=True, - text=True, env=dict(os.environ, **env)) + text=True, env=env) return h, (json.loads(h.stdout) if h.stdout.strip() else {}) @@ -365,6 +366,11 @@ def handed_back(t, reason, reply): check('a reply that drops or rewrites the receipt gets it shown as a message instead', out2.get('systemMessage') == msg, (h2.stdout, msg)) h3, out3 = hook(dict(ev, stop_hook_active=True)) check('the second stop happens once: a third is silent', out3 == {}, h3.stdout) +r = repo(); change(r, 'src/pkg/alpha.py') +t = T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, IMPACT_A).edit('src/pkg/alpha.py') +h, out = hook(dict(ev, session_id='s6', transcript_path=t.save(), cwd=r), CLAUDE_CODE_ENTRYPOINT='cli') +check('in the terminal the receipt is one message under the reply, never handed back (that would show it twice)', + 'decision' not in out and out.get('systemMessage', '').startswith('◆ AxiomCode'), h.stdout) # --- two answers, one acted on: the count says so; a later turn that only grows an earlier chain says that instead r = repo(); change(r, 'src/pkg/alpha.py') From 60c38d0a9d8a5d9d976e05ea150da46d663ac48b Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Mon, 12 Oct 2026 14:53:15 -0700 Subject: [PATCH 11/12] Revert "plugin: the terminal shows the assist receipt once (#1935)" The terminal did not show the Stop hook's message, so the receipt disappeared there. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- plugins/axiomcode/hooks/assist.py | 12 +++++------- plugins/axiomcode/skills/axiomcode/SKILL.md | 7 +++---- skills/axiomcode/SKILL.md | 7 +++---- tests/assists.py | 8 +------- 4 files changed, 12 insertions(+), 22 deletions(-) diff --git a/plugins/axiomcode/hooks/assist.py b/plugins/axiomcode/hooks/assist.py index ffd16e974..1660571c8 100644 --- a/plugins/axiomcode/hooks/assist.py +++ b/plugins/axiomcode/hooks/assist.py @@ -13,12 +13,10 @@ calls and their results, and a link that was not checked is not printed. · SILENT unless an answer led somewhere: no query, or a query nobody acted on, prints nothing. Only what helped is counted; the answers that led nowhere are never shown. - · SHOWN ONCE. In the terminal (CLAUDE_CODE_ENTRYPOINT=cli) it is a message under the reply: the terminal shows - a Stop hook's message, and would also show a hand-back's reason, so handing it back there prints it twice. - Elsewhere (`-p`, the SDK, an IDE) a hook's message may never reach the user, so the receipt goes back to the - agent once (decision "block"), which ends its reply with it word for word. The second stop (stop_hook_active) - checks the reply carries every line; when it does not, the receipt is shown as a message instead. Gemini gets - the message. + · THE AGENT QUOTES IT. In Claude Code the receipt goes back to the agent once (decision "block"), which ends its + reply with it word for word, so it is part of the answer in every client (`-p`, the SDK, an IDE). The second stop + (stop_hook_active) checks the reply carries every line; when it does not, the receipt is shown as a message + instead. Cursor and Gemini cannot hand it back, so they get the message. AXIOMCODE_ASSISTS=off turns it off. """ import json, os, subprocess, sys, time @@ -91,7 +89,7 @@ def _quoted(transcript, msg): sys.exit(0) # a receipt is never worth a failed turn if not msg: sys.exit(0) -if _host.GEMINI or os.environ.get('CLAUDE_CODE_ENTRYPOINT') == 'cli': +if _host.GEMINI: print(json.dumps({'systemMessage': msg})) sys.exit(0) try: diff --git a/plugins/axiomcode/skills/axiomcode/SKILL.md b/plugins/axiomcode/skills/axiomcode/SKILL.md index 531b28fec..b18ed7031 100644 --- a/plugins/axiomcode/skills/axiomcode/SKILL.md +++ b/plugins/axiomcode/skills/axiomcode/SKILL.md @@ -126,10 +126,9 @@ not found this way. ## report At the end of a turn where an answer verifiably led your work (a file the answer showed first that you then read or -edited, or the tests it picked that you ran: Redirected / Helped / Useful), the plugin shows the user a receipt. In -the terminal it appears under your reply; elsewhere (headless, an IDE) it is handed to you once: end your reply with -it exactly as written. It is read from the session's tool calls, not from what you write, so there is nothing to claim -yourself. +edited, or the tests it picked that you ran: Redirected / Helped / Useful), the plugin hands you a receipt once. End +your reply with it exactly as written. It is read from the session's tool calls, not from what you write, so there is +nothing to claim yourself; if the reply does not carry it, the user is shown it anyway. Each repository keeps its own record (`.axiomcode/assists.jsonl` at its root) and its own page, `.axiomcode/assists.html`, rebuilt whenever a turn adds to the record; `axiomcode report` rebuilds it by hand. diff --git a/skills/axiomcode/SKILL.md b/skills/axiomcode/SKILL.md index 001ec0453..5a6c6db1c 100644 --- a/skills/axiomcode/SKILL.md +++ b/skills/axiomcode/SKILL.md @@ -126,10 +126,9 @@ not found this way. ## report At the end of a turn where an answer verifiably led your work (a file the answer showed first that you then read or -edited, or the tests it picked that you ran: Redirected / Helped / Useful), the plugin shows the user a receipt. In -the terminal it appears under your reply; elsewhere (headless, an IDE) it is handed to you once: end your reply with -it exactly as written. It is read from the session's tool calls, not from what you write, so there is nothing to claim -yourself. +edited, or the tests it picked that you ran: Redirected / Helped / Useful), the plugin hands you a receipt once. End +your reply with it exactly as written. It is read from the session's tool calls, not from what you write, so there is +nothing to claim yourself; if the reply does not carry it, the user is shown it anyway. Each repository keeps its own record (`.axiomcode/assists.jsonl` at its root) and its own page, `.axiomcode/assists.html`, rebuilt whenever a turn adds to the record; `axiomcode report` rebuilds it by hand. diff --git a/tests/assists.py b/tests/assists.py index 753af4d13..35d54a27e 100644 --- a/tests/assists.py +++ b/tests/assists.py @@ -330,9 +330,8 @@ def found(t, r, diff=True): # --- the hook end to end: a Stop event in, the receipt handed back to the agent, which ends its reply with it; when # the reply does not carry it, the second stop shows it as a message instead; switched off, nothing def hook(ev, **env): - env = dict(os.environ, **dict({'CLAUDE_CODE_ENTRYPOINT': 'sdk-cli'}, **env)) h = subprocess.run([sys.executable, os.path.join(HOOKS, 'assist.py')], input=json.dumps(ev), capture_output=True, - text=True, env=env) + text=True, env=dict(os.environ, **env)) return h, (json.loads(h.stdout) if h.stdout.strip() else {}) @@ -366,11 +365,6 @@ def handed_back(t, reason, reply): check('a reply that drops or rewrites the receipt gets it shown as a message instead', out2.get('systemMessage') == msg, (h2.stdout, msg)) h3, out3 = hook(dict(ev, stop_hook_active=True)) check('the second stop happens once: a third is silent', out3 == {}, h3.stdout) -r = repo(); change(r, 'src/pkg/alpha.py') -t = T(r).prompt('fix the bug in run').call(MCP, {'name': 'Alpha.run'}, IMPACT_A).edit('src/pkg/alpha.py') -h, out = hook(dict(ev, session_id='s6', transcript_path=t.save(), cwd=r), CLAUDE_CODE_ENTRYPOINT='cli') -check('in the terminal the receipt is one message under the reply, never handed back (that would show it twice)', - 'decision' not in out and out.get('systemMessage', '').startswith('◆ AxiomCode'), h.stdout) # --- two answers, one acted on: the count says so; a later turn that only grows an earlier chain says that instead r = repo(); change(r, 'src/pkg/alpha.py') From 6e8cbd45f8e95d02a60f48032b522f1824005a40 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:10:08 -0700 Subject: [PATCH 12/12] plugin: the receipt says an answer came from an earlier turn, not that it went further (#1935) Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- .../skills/axiomcode/scripts/ax_assists.py | 14 +++++++++----- tests/assists.py | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py index 4ec62ae75..49bca6d16 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_assists.py @@ -620,11 +620,15 @@ def receipt(fresh, totals=None, turn=None, session=None): if l not in said: said.append(l) head = '◆ AxiomCode this turn' - if turn and turn[1]: - head += f" · the agent acted on {turn[1]} answer{'s' if turn[1] > 1 else ''}" - earlier = len({c['answer'] for c in fresh} - set(turn[2] if turn and len(turn) > 2 else [])) - if turn and earlier: - head += (', and ' if turn[1] else ' · ') + f"{earlier} earlier answer{'s' if earlier > 1 else ''} went further" + now = turn[1] if turn else 0 + # an answer from an earlier turn the agent acted on only now (it read a file then, and edits it in this turn) + earlier = len({c['answer'] for c in fresh} - set(turn[2] if turn and len(turn) > 2 else [])) if turn else 0 + if now and earlier: + head += f" · the agent acted on {now} answer{'s' if now > 1 else ''} from this turn and {earlier} from an earlier turn" + elif now: + head += f" · the agent acted on {now} answer{'s' if now > 1 else ''}" + elif earlier: + head += f" · the agent acted on {earlier} answer{'s' if earlier > 1 else ''} from an earlier turn" lines = [head] + [f' • {l}' for l in said[:5]] if len(said) > 5: lines.append(f' • and {len(said) - 5} more (axiomcode report)') diff --git a/tests/assists.py b/tests/assists.py index 35d54a27e..a278cef51 100644 --- a/tests/assists.py +++ b/tests/assists.py @@ -374,7 +374,7 @@ def handed_back(t, reason, reply): t.prompt('now fix it').edit('src/pkg/alpha.py') later = A.at_stop(t.save(), r, 's4') check('one of two answers acted on counts only the one', 'acted on 1 answer' in first and ' of 2' not in first, first) -check('a turn that only grows an earlier chain names it as earlier', 'earlier answer went further' in later and 'Helped' in later, later) +check('a turn that only grows an earlier chain names it as earlier', 'acted on 1 answer from an earlier turn' in later.splitlines()[0] and 'Helped' in later, later) page = open(A.report(r)[0]).read() check('the report has no unfilled template text', '\\1' not in page and "{'" not in page and "'}" not in page and ' if ' not in page, page[:300]) check('the report counts answers asked and the files and tests behind them', ' of 2' not in page and 'Previously unseen files edited' in page and 'edited' in page