|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""tests/mcp_first.py — every surface an agent reads before its first call names the MCP tool first (#1425). |
| 3 | +
|
| 4 | +`axiomcode install` pre-approves the plugin's MCP tools and nothing else. The skill's description, the |
| 5 | +prompt-time orientation and the pre-search directive used to spell the call as a shell command, with the tool |
| 6 | +in parentheses or not at all, so an agent took the shell spelling; under the install's permissions that first |
| 7 | +call is the one that stops for a prompt, and in a headless session it is denied outright. |
| 8 | +
|
| 9 | +Each check is one surface: the tool's name comes before the shell form of the same verb. The control on every |
| 10 | +surface is the other half: the shell form is still there, after it. A host without the MCP server, or a session |
| 11 | +where the server did not start, has nothing else to call, and the skill has to keep working there. |
| 12 | +
|
| 13 | + python3 tests/mcp_first.py indexes one case, so it needs the engine, as hosts.py does |
| 14 | +""" |
| 15 | +import json, os, re, shutil, subprocess, sys, tempfile |
| 16 | + |
| 17 | +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| 18 | +PLUG = os.path.join(ROOT, 'plugins', 'axiomcode') |
| 19 | +HOOKS = os.path.join(PLUG, 'hooks') |
| 20 | +SCRIPTS = os.path.join(PLUG, 'skills', 'axiomcode', 'scripts') |
| 21 | +AX = os.path.join(SCRIPTS, 'axiomcode') |
| 22 | +CASE = os.path.join(ROOT, 'tests', 'cases', 'java', 'injected-into-a-field') |
| 23 | + |
| 24 | +fails, checked = [], [] |
| 25 | +def check(why, cond, detail=''): |
| 26 | + checked.append(why) |
| 27 | + print(('ok ' if cond else 'FAIL ') + why + (f'\n {detail}' if not cond and detail else '')) |
| 28 | + if not cond: |
| 29 | + fails.append(why) |
| 30 | + |
| 31 | + |
| 32 | +def tool_first(surface, text, verbs): |
| 33 | + """for each verb: the MCP tool is named, the shell form is named, and the tool comes first""" |
| 34 | + for v in verbs: |
| 35 | + tool, shell = f"axiomcode_{v.replace('-', '_')}", f"`axiomcode {v}" |
| 36 | + t, s = text.find(tool), text.find(shell) |
| 37 | + check(f'{surface}: {v} is named as the {tool} tool', t >= 0, text[:200]) |
| 38 | + check(f'{surface}: {v} keeps its shell form, for a host without the MCP server', s >= 0, text[:200]) |
| 39 | + if t >= 0 and s >= 0: |
| 40 | + check(f'{surface}: {tool} comes before `axiomcode {v}`', t < s, f'tool at {t}, shell at {s}') |
| 41 | + |
| 42 | + |
| 43 | +def fire(hook, ev): |
| 44 | + r = subprocess.run([sys.executable, os.path.join(HOOKS, hook)], input=json.dumps(ev), |
| 45 | + capture_output=True, text=True, timeout=120) |
| 46 | + out = r.stdout.strip() |
| 47 | + try: |
| 48 | + return r.returncode, json.loads(out)['hookSpecificOutput']['additionalContext'] |
| 49 | + except (ValueError, KeyError, TypeError): |
| 50 | + return r.returncode, out |
| 51 | + |
| 52 | + |
| 53 | +# 1. the description, which is the one part of the skill every session sees |
| 54 | +skill = open(os.path.join(PLUG, 'skills', 'axiomcode', 'SKILL.md'), encoding='utf-8').read() |
| 55 | +m = re.search(r'^description: >-\n(.*?)\n---', skill, re.S | re.M) |
| 56 | +check('SKILL.md has a description block', bool(m)) |
| 57 | +if m: |
| 58 | + tool_first('SKILL.md description', ' '.join(m.group(1).split()), ('context', 'impact', 'path')) |
| 59 | + |
| 60 | +# 2. the block `axiomcode install` writes into CLAUDE.md, beside the permission it grants |
| 61 | +r = subprocess.run([sys.executable, os.path.join(SCRIPTS, 'axiomcode-install'), '--print'], |
| 62 | + capture_output=True, text=True, timeout=30) |
| 63 | +check('install --print prints the block', r.returncode == 0 and 'BEGIN axiomcode' in r.stdout, r.stderr[-200:]) |
| 64 | +tool_first('install block', r.stdout, ('context', 'impact', 'path')) |
| 65 | + |
| 66 | +# 3. the directive before the first search; it needs only a graph file to exist |
| 67 | +with tempfile.TemporaryDirectory() as repo: |
| 68 | + os.makedirs(os.path.join(repo, '.axiomcode', 'out')) |
| 69 | + open(os.path.join(repo, '.axiomcode', 'out', 'graph.sqlite'), 'w').close() |
| 70 | + rc, said = fire('direct.py', {'hook_event_name': 'PreToolUse', 'tool_name': 'Grep', |
| 71 | + 'tool_input': {'pattern': 'f'}, 'cwd': repo, 'session_id': 'm1'}) |
| 72 | + check('direct: the first search hears the directive', rc == 0 and bool(said), f'rc={rc}') |
| 73 | + tool_first('direct', said, ('impact', 'path', 'context', 'changed', 'test-impact')) |
| 74 | + |
| 75 | +# 4. the orientation on the first prompt, both branches it can reach: a change question and a how-question |
| 76 | +with tempfile.TemporaryDirectory() as work: |
| 77 | + repo = os.path.join(work, 'case') |
| 78 | + shutil.copytree(CASE, repo) |
| 79 | + b = subprocess.run(['bash', AX, 'index', repo], capture_output=True, text=True, timeout=900) |
| 80 | + built = os.path.exists(os.path.join(repo, '.axiomcode', 'out', 'graph.sqlite')) |
| 81 | + check('the case indexes', built, (b.stdout + b.stderr)[-300:]) |
| 82 | + if built: |
| 83 | + rc, said = fire('orient.py', {'hook_event_name': 'UserPromptSubmit', 'cwd': repo, 'session_id': 'o1', |
| 84 | + 'prompt': 'What breaks if I change Repo.find and its callers?'}) |
| 85 | + check('orient: a change question is oriented', rc == 0 and 'next,' in said, said[:300]) |
| 86 | + tool_first('orient (change)', said, ('impact',)) |
| 87 | + rc, said = fire('orient.py', {'hook_event_name': 'UserPromptSubmit', 'cwd': repo, 'session_id': 'o2', |
| 88 | + 'prompt': 'How does Consumer.go work, step by step?'}) |
| 89 | + check('orient: a how-question is oriented to the flow', rc == 0 and 'next:' in said, said[:300]) |
| 90 | + tool_first('orient (how)', said, ('context',)) |
| 91 | + |
| 92 | +# 5. orient's third hint, for a verb that refuses without a scope: no verb refuses that way today, so it cannot be |
| 93 | +# fired; the order is checked in the source line that prints it. |
| 94 | +src = open(os.path.join(HOOKS, 'orient.py'), encoding='utf-8').read() |
| 95 | +i = src.find("the axiomcode_context tool with in_path=<one of these>") |
| 96 | +check('orient (scope refused): its hint is still in the source', i >= 0) |
| 97 | +if i >= 0: |
| 98 | + tool_first('orient (scope refused)', src[i:src.find('\n', src.find("')", i))], ('context',)) |
| 99 | + |
| 100 | +print(f'\n{len(checked) - len(fails)} of {len(checked)} check(s) held') |
| 101 | +sys.exit(1 if fails else 0) |
0 commit comments