Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions docs/MCP.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,12 +93,10 @@ Tools from connected servers are not advertised to the model up front; they are
registered for dispatch as soon as the server connects (including later in the
same turn) and surfaced on demand through dynamic tool discovery
(`tool_search`). A match is a ranked handful of names plus capped
descriptions. The top two cards also include a compact input schema so
the first call is formable. `tool_search` does not promote
names onto the next inference or into `run.json`. Calling a discovered
name declares that one schema on the next infer (promote-on-execute),
persists it in the session's `run.json`, and re-advertises it on resume
and after rebuilds. Fold drops idle execute-promoted schemas and persists
descriptions. The top five cards also include a compact input schema and
are loaded onto the next inference's advertised tail (Codex/Pi deferred)
and into `run.json`. Remaining hits stay name + description until called
(promote-on-execute). Fold drops idle execute-promoted schemas and persists
that empty set so a crash or resume does not restore them.

For integrations a project calls constantly, `pinnedTools` in local
Expand Down
5 changes: 3 additions & 2 deletions e2e/mcp-late-dispatch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,9 @@ function toolDoneContents(events: ReactorEmittedEvent[]): string[] {
);
}

// Wires production promote-on-execute: search returns cards only; a call to a
// registered-but-unadvertised name declares that one schema, then dispatches.
// Wires production load-on-search: top ranked hits join the advertised tail
// for the next infer; a call to a remaining unadvertised name still
// promote-on-execute.
function promoteDynamicTools(session: IntegrationSession): void {
const advertised = createAdvertisedToolset({
sessionMode: "orchestrator",
Expand Down
25 changes: 14 additions & 11 deletions src/agent/mcp-promote-on-execute.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,9 @@
* CL-9704: Linear MCP discovery-to-invocation on the primary session.
*
* Regression lock for "discoverable but not callable": tool_search finds
* `mcp__linear__*`, and promote-on-execute must then commit a callable
* schema for exactly the called name and dispatch it — list_teams first,
* then save_issue. Search alone never promotes (the wire stays
* built-ins-only until a call), and promoting one name never implies its
* siblings: the primary session mounts MCP tools on demand, mirroring the
* worker requires_tools gate.
* `mcp__linear__*`, loads the top ranked hits onto the next infer's tail,
* and promote-on-execute still commits a called name that was not in that
* prefix — list_teams via search-load, then save_issue via execute.
*/

import { afterEach, describe, expect, test } from "bun:test";
Expand Down Expand Up @@ -126,20 +123,26 @@ describe("CL-9704 linear MCP discovery-to-invocation", () => {
toolset.dynamicRunner
.currentDefinitions()
.find((d) => d.name === name),
promote: (names) => {
advertised.activated.activate(names);
if (advertised.flushPromotions()) {
wire = advertised
.computeAdvertised(toolset.dynamicRunner.currentDefinitions())
.map((d) => d.name);
}
},
});
if (search.kind !== "string") throw new Error("expected string tool");

// Discovery: the card names list_teams with its description.
// Discovery: the card names list_teams; search loads the top ranked
// hits onto the next infer's tail.
const card = await search.handler(
{ query: "linear teams" },
new AbortController().signal,
);
expect(card).toContain("mcp__linear__list_teams");
expect(card).toContain("List Linear teams");

// Search alone promotes nothing: no Linear schema on the wire.
expect(wire).not.toContain("mcp__linear__list_teams");
expect(wire).not.toContain("mcp__linear__save_issue");
expect(wire).toContain("mcp__linear__list_teams");

const run = (name: string, args: Record<string, unknown> = {}) =>
toolset.dynamicRunner.run(
Expand Down
6 changes: 3 additions & 3 deletions src/agent/prompts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ export function buildHarnessFacts(
"- Attached images are native multimodal input; inspect them directly unless file-level forensics are requested.",
...(dynamicTools
? [
"- Core tools plus the advertised catalog are resident. Use tool_search to find MCP/plugin tools; the top two cards include the input schema. Call a listed name and it joins the tool list — unused matches stay off it.",
"- Core tools plus the advertised catalog are resident. Use tool_search to find MCP/plugin tools; top matches include the input schema and join the tool list next turn (default 5).",
"- Use search_agents before dispatching named specialists or teams (ids and descriptions by default; include_body=true for the loaded system prompt).",
"- The user may send follow-up messages while workers run; they are queued. Enter delivers at the next parent tool.boundary; Alt+Enter on session-idle. A long parent tool holds that boundary. Update your plan, spawn or adjust workers, and keep the operator informed.",
]
Expand Down Expand Up @@ -163,7 +163,7 @@ const GUIDELINE_SUB_BLOCKS: Record<
...(ctx.subAgent
? []
: [
"- tool_search before assuming a plugin or MCP tool exists (top cards include the schema; call a listed name). use_skill to load a skill body.",
"- tool_search before assuming a plugin or MCP tool exists (top matches join the list next turn). use_skill to load a skill body.",
]),
],
askVsProceed: (ctx) => [
Expand Down Expand Up @@ -329,7 +329,7 @@ export function buildEnvironmentContext(env: EnvironmentInfo): string {
if (env.topLevel) lines.push(`Top level: ${env.topLevel}`);
if (env.mcpServers !== undefined && env.mcpServers.length > 0) {
lines.push(
`MCP: ${env.mcpServers.join(", ")} — tool_search finds their tools (top hits include schema); call a listed name to use it.`,
`MCP: ${env.mcpServers.join(", ")} — tool_search finds their tools (top matches join the list next turn); call a listed name to use it.`,
);
}
lines.push(`Memory file: ${env.cwd}/${SETTINGS_DIR_NAME}/MEMORY.md`);
Expand Down
94 changes: 71 additions & 23 deletions src/agent/tool-search.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
TOOL_SEARCH_MAX_RESULTS,
TOOL_SEARCH_LIMIT_MAX,
TOOL_SEARCH_SCHEMA_MAX,
TOOL_SEARCH_SCHEMA_CARDS,
toolSearchDefinition,
type ToolAvailability,
} from "./tool-search.js";
Expand Down Expand Up @@ -470,6 +471,32 @@ describe("createToolSearchTool", () => {
expect(listed.every((name) => catalogNames.includes(name))).toBe(true);
});

test("search loads the top ranked names when a promoter is wired", async () => {
const catalog: ToolDefinition[] = Array.from({ length: 8 }, (_, i) => ({
name: `mcp__linear__op_${i}`,
description: "Linear issue operation",
inputSchema: {
type: "object",
properties: { [`field_${i}`]: { type: "string" } },
required: [`field_${i}`],
},
}));
const loaded: string[] = [];
const tool = createToolSearchTool({
search: () => catalog.map((d) => d.name),
lookup: (name) => catalog.find((d) => d.name === name),
promote: (names) => {
loaded.push(...names);
},
});
const out = await call(tool, { query: "linear" });
expect(loaded).toEqual(
catalog.slice(0, TOOL_SEARCH_SCHEMA_CARDS).map((d) => d.name),
);
expect(out).toContain("mcp__linear__op_7");
expect(out).not.toContain("field_7");
});

test("search cards include the input schema so a first call is formable", async () => {
const tool = createToolSearchTool({
search: (q) => index.search(q),
Expand All @@ -484,8 +511,8 @@ describe("createToolSearchTool", () => {
);
});

test("only the top two ranked cards include input schema", async () => {
const catalog: ToolDefinition[] = Array.from({ length: 5 }, (_, i) => ({
test("only the top ranked cards include input schema", async () => {
const catalog: ToolDefinition[] = Array.from({ length: 8 }, (_, i) => ({
name: `mcp__linear__op_${i}`,
description: "Linear issue operation",
inputSchema: {
Expand All @@ -501,20 +528,15 @@ describe("createToolSearchTool", () => {
const out = await call(tool, { query: "linear" });
expect(listedToolNames(out)).toEqual(catalog.map((d) => d.name));
expect(out).toContain("field_0");
expect(out).toContain("field_1");
expect(out).not.toContain("field_2");
expect(out).not.toContain("field_3");
expect(out).not.toContain("field_4");
expect(out).toContain("mcp__linear__op_2");
expect(out).toContain("mcp__linear__op_4");
expect(out).toContain("field_4");
expect(out).not.toContain("field_5");
expect(out).not.toContain("field_7");
expect(out).toContain("mcp__linear__op_7");
});

test("tool_search description does not claim matches load on the next turn", () => {
expect(toolSearchDefinition.description).not.toMatch(
/load on the next turn/i,
);
test("tool_search description says top matches join the list next turn", () => {
expect(toolSearchDefinition.description).toMatch(/next turn/i);
expect(toolSearchDefinition.description).toMatch(/schema/i);
expect(toolSearchDefinition.description).toMatch(/call a listed name/i);
});

test("an oversized input schema is capped on the card", async () => {
Expand Down Expand Up @@ -819,6 +841,10 @@ describe("promote-on-execute", () => {
search: (q, limit) => wideIndex.search(q, limit),
lookup: (name) =>
runner.currentDefinitions().find((d) => d.name === name),
promote: (names) => {
advertised.activated.activate(names);
advertised.flushPromotions();
},
});
return { runner, advertised, promoted, search };
}
Expand All @@ -842,21 +868,27 @@ describe("promote-on-execute", () => {
.map((d) => d.name);
}

test("a tool_search call does not change the advertised set", async () => {
test("a tool_search call loads ranked hits onto the advertised tail", async () => {
const { advertised, promoted, search, runner } = wirePromoteOnExecute();
const before = advertisedNames(advertised, runner);
if (search.kind !== "string") throw new Error("expected string tool");
const out = await search.handler(
{ query: "linear issue", limit: 3 },
new AbortController().signal,
);
expect(listedToolNames(out).length).toBe(3);
const listed = listedToolNames(out);
expect(listed.length).toBe(3);
expect(promoted).toEqual([]);
expect(advertised.activated.list()).toEqual([]);
expect(advertisedNames(advertised, runner)).toEqual(before);
const loaded = listed.slice(0, TOOL_SEARCH_SCHEMA_CARDS);
const rest = listed.slice(TOOL_SEARCH_SCHEMA_CARDS);
expect(advertised.activated.list()).toEqual(loaded);
const after = advertisedNames(advertised, runner);
for (const name of loaded) expect(after).toContain(name);
for (const name of rest) expect(after).not.toContain(name);
expect(before.every((name) => after.includes(name))).toBe(true);
});

test("a subsequent call to a searched name promotes only that name", async () => {
test("a call to an already-loaded search hit does not re-promote", async () => {
const { advertised, promoted, search, runner } = wirePromoteOnExecute();
if (search.kind !== "string") throw new Error("expected string tool");
const listed = listedToolNames(
Expand All @@ -866,19 +898,35 @@ describe("promote-on-execute", () => {
),
);
expect(listed).toHaveLength(3);
expect(promoted).toEqual([]);
const called = listed[0];
if (called === undefined) throw new Error("expected a listed tool");
const others = listed.slice(1);
const result = await dispatch(runner, called);
expect(result.content).toBe(`ran:${called}`);
expect(result.isError).toBeUndefined();
expect(promoted).toEqual([]);
expect(advertisedNames(advertised, runner)).toContain(called);
});

test("a call to a search hit past the loaded set still promote-on-execute", async () => {
const { advertised, promoted, search, runner } = wirePromoteOnExecute();
if (search.kind !== "string") throw new Error("expected string tool");
const listed = listedToolNames(
await search.handler(
{ query: "linear issue", limit: 8 },
new AbortController().signal,
),
);
const called = listed[TOOL_SEARCH_SCHEMA_CARDS];
if (called === undefined)
throw new Error("expected a hit past the load cap");
const loaded = listed.slice(0, TOOL_SEARCH_SCHEMA_CARDS);
const result = await dispatch(runner, called);
expect(result.content).toBe(`ran:${called}`);
expect(result.isError).toBeUndefined();
expect(promoted).toEqual([called]);
const names = advertisedNames(advertised, runner);
expect(names).toContain(called);
for (const other of others) {
expect(names).not.toContain(other);
}
for (const name of loaded) expect(names).toContain(name);
});

test("a never-searched undeclared name still promote-on-execute", async () => {
Expand Down
22 changes: 14 additions & 8 deletions src/agent/tool-search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@ import {
import { canonicalToolName } from "./canonical-tool-name.js";

// Tools whose full schema is always advertised to the model. Everything else is
// registered but discovered on demand via tool_search, which returns name +
// description cards only. A found tool's schema joins the wire when the model
// actually calls it (promote-on-execute). Shared by the system prompt
// registered but discovered on demand via tool_search, which returns ranked
// cards and loads the top TOOL_SEARCH_SCHEMA_CARDS onto the next infer's
// advertised tail (Codex/Pi deferred). Remaining hits stay name + description
// until the model calls them (promote-on-execute). Shared by the system prompt
// and the advertised-set gate so the two never drift.
//
// `present` is deliberately absent: most sessions never render a view, and at
Expand Down Expand Up @@ -252,7 +253,7 @@ export function createActivatedToolTracker(): ActivatedToolTracker {
export const toolSearchDefinition: ToolDefinition = {
name: "tool_search",
description:
"Find tools (MCP servers, integrations) and skills by capability. Top two matches include input schema — call a listed name to use it (it then joins the tool list). Unused matches stay off the list. Load a skill body with use_skill.",
"Find tools (MCP servers, integrations) and skills by capability. Top matches include input schema and join the tool list on the next turn (default 5). Unused matches stay off the list. Load a skill body with use_skill.",
inputSchema: {
type: "object",
properties: {
Expand Down Expand Up @@ -283,7 +284,7 @@ export const TOOL_SEARCH_LIMIT_MAX = 20;
export const TOOL_SEARCH_SCORE_RATIO = 0.75;
export const TOOL_SEARCH_DESC_MAX = 160;
export const TOOL_SEARCH_SCHEMA_MAX = 600;
const TOOL_SEARCH_SCHEMA_CARDS = 2;
export const TOOL_SEARCH_SCHEMA_CARDS = 5;

export function createToolIndex(
getDefs: () => readonly ToolDefinition[],
Expand Down Expand Up @@ -335,6 +336,9 @@ export interface ToolSearchDeps {
// Skill matches as "- name: description" lines; rendered beside tool matches.
searchSkills?: (query: string) => string[];
lookup: (name: string) => ToolDefinition | undefined;
// Load the top ranked names onto the next infer's advertised tail.
// Omitted in unit tests that only assert card text.
promote?: (names: string[]) => void;
// Resolves to the remaining in-flight MCP handshake count after waiting up
// to `timeoutMs`. The toolset bounds its own wait; the handler re-races
// below so even a stuck dependency can never hang the call. Omitted callers
Expand Down Expand Up @@ -521,9 +525,11 @@ export function createToolSearchTool(deps: ToolSearchDeps): AgentTool {
return `No tools or skills matched "${query}". Try different keywords describing the capability.`;
}
// Ranked cards: top TOOL_SEARCH_SCHEMA_CARDS include a compact input
// schema so the first call is formable; the rest are name + description.
// Search does not open the call gate or grow the tools array;
// promote-on-execute declares a name when the model actually calls it.
// schema and are flushed onto the next infer's advertised tail
// (Codex/Pi deferred). The rest are name + description; promote-on-execute
// still declares a called name that was not in that prefix.
const load = names.slice(0, TOOL_SEARCH_SCHEMA_CARDS);
if (load.length > 0) deps.promote?.(load);
const blocks = names.map((name, i) =>
renderToolCard(deps.lookup(name), name, i < TOOL_SEARCH_SCHEMA_CARDS),
);
Expand Down
9 changes: 6 additions & 3 deletions src/agent/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -786,10 +786,11 @@ export async function createAgentToolset(
}),
];

// tool_search ranks over the live runner (set just below). Matches stay
// cards-only; the session promoter (wired once advertise/reload exists)
// declares a name when the model actually calls it.
// tool_search ranks over the live runner (set just below). The top ranked
// hits load onto the next infer via the session promoter (wired once
// advertise/reload exists); remaining cards stay name+description until call.
const runnerHolder: { current?: DynamicToolRunner } = {};
const promoterHolder: { current?: (names: string[]) => void } = {};
const toolIndex = createToolIndex(
() => runnerHolder.current?.currentDefinitions() ?? [],
advertisedBuiltIns,
Expand All @@ -810,6 +811,7 @@ export async function createAgentToolset(
runnerHolder.current
?.currentDefinitions()
.find((d) => d.name === name),
promote: (names) => promoterHolder.current?.(names),
// Misses wait briefly for in-flight MCP handshakes (bounded, so hung
// OAuth cannot hang the call) and re-search before answering. Reads the
// connection map live — declared below, populated by the time any
Expand Down Expand Up @@ -1610,6 +1612,7 @@ export async function createAgentToolset(
mcpServersSource = source;
},
setToolPromoter: (promote) => {
promoterHolder.current = promote;
dynamicRunner.setOnUndeclaredCall((name) => promote([name]));
},
skills,
Expand Down
Loading
Loading