diff --git a/docs/operations/monitoring.md b/docs/operations/monitoring.md index b778428a5d..5546cd9fd8 100644 --- a/docs/operations/monitoring.md +++ b/docs/operations/monitoring.md @@ -420,3 +420,84 @@ way. Bucket fill itself stays per host. - [Telemetry](../features/observability/telemetry.md): the metric and span catalogue - [Cost and usage](../features/observability/cost-and-usage.md) and [audit](../features/observability/audit.md) + +## Loom invocation observability + +Loom observes actual HTTP operations and MCP tool calls through its existing +OpenTelemetry exporters. Each call observation has `loom.call.root=true`, a +unique `loom.call.attempt_id`, the operation, transport, outcome, and duration +in milliseconds. Provider and retrieval stages remain children of that call. + +Transport and origin answer different questions. `loom.call.transport` is +`http` or `mcp`. `loom.call.origin` is the caller-reported `tool`, `hook`, +`worker`, or `unknown`. Absent or invalid origin stays unknown; MCP alone does +not prove that an agent selected the call. `caller_type` can separately name +`agent`, `plugin`, `sdk`, `cli`, or `worker`. Worker passes carry worker origin +but do not count as received requests, and the Langfuse worker filters still +apply. Origin and correlation metadata grant no permissions. + +HTTP callers can supply these headers. Each value is independently optional; +identifiers accept at most 128 ASCII letters, digits, underscores, dots, +colons, or hyphens. Invalid identifiers are omitted. + +| Header | Meaning | +| --------------------------------------------------------------------------------- | ------------------------------------------------------------------- | +| `X-Loom-Call-Origin` | Declared invocation origin | +| `X-Loom-Caller-Type` | Declared caller type | +| `X-Loom-Hook-Name`, `X-Loom-Hook-Event` | Known hook entrypoint and event | +| `X-Loom-Conversation-Id`, `X-Loom-Prompt-Id` | Optional supplied conversation and turn | +| `X-Loom-Tool-Call-Id` | Caller-supplied tool-call correlation; uniqueness is caller-defined | +| `X-Loom-Benchmark-Run-Id`, `X-Loom-Benchmark-Case-Id`, `X-Loom-Benchmark-Variant` | Evaluator correlation | +| `traceparent`, `tracestate` | Optional W3C parent context | + +MCP callers supply the same snake-case fields in `params._meta.loom`, with +`origin` for the invocation origin; W3C context goes directly in `_meta`. +Per-call metadata takes precedence over request headers. For example: + +```json +{ + "name": "loom_op", + "arguments": { "op": "help" }, + "_meta": { + "loom": { + "origin": "tool", + "caller_type": "agent", + "tool_call_id": "call-example", + "benchmark_run_id": "run-example", + "benchmark_case_id": "case-example", + "benchmark_variant": "with-memory" + } + } +} +``` + +The hook transport declares hook origin even without a prompt ID. Known hook +entrypoints also identify detached requests without borrowing another turn's +identity. Optional supplied IDs never change stored session scope, tenant +identity, authorization, or capture consent. Raw prompts, results and +credentials are not trace metadata. + +Call observations cover HTTP operation refusals and MCP catalog, protocol, +and authorization-screening outcomes after the HTTP bearer gate. Invalid +JSON, unsupported HTTP protocol headers, whole-batch limits and bearer +refusals can precede MCP call observations. Private-work diagnostics are +suppressed by the existing export boundary. Trace sampling and span filters +still apply, so missing observations do not prove zero calls. + +Count only observations marked `loom.call.root`, excluding HTTP envelopes, +batch entries and nested stages. Received retries have distinct attempt IDs. +A host tool-call ID can correlate several separate Loom requests from hook +fanout. Grouping attempts requires a caller guarantee that the supplied ID +identifies one logical Loom request within the caller's scope and remains +stable across its retries. Without that guarantee the logical count is unknown. +Structured benchmark metadata is filterable; it does not imply arbitrary +grouping or an answer-quality gain. + +Sum call durations for accumulated call time. Concurrent requests overlap, +so this sum differs from elapsed wall time. A host prompt lifecycle is not +required for Loom invocation tracing. Valid supplied parent context can link +Loom calls to an external trace; absent context leaves an independent request. +Sum actual generation and embedding usage once on those observations, without +copying cost to the call parent. Missing usage or price stays unknown. Loom +provider usage is separate from external agent-host billing. Output length is +not measured model usage. diff --git a/integrations/claude-code-plugin/.claude-plugin/plugin.json b/integrations/claude-code-plugin/.claude-plugin/plugin.json index cb2d6376b5..e61111bbb8 100644 --- a/integrations/claude-code-plugin/.claude-plugin/plugin.json +++ b/integrations/claude-code-plugin/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "loom", "displayName": "ClickHouse Loom", - "version": "1.2.45", + "version": "1.2.48", "description": "Shared long-term memory and code intelligence for Claude Code, self-hosted on ClickHouse. MCP tools to save and recall decisions, preferences and corrections across sessions and teammates; lifecycle hooks that capture each session with no model call; a code index that answers where-is, who-calls and blast-radius questions with line-accurate units instead of whole-file reads.", "author": { "name": "ClickHouse" diff --git a/integrations/claude-code-plugin/README.md b/integrations/claude-code-plugin/README.md index f7a7b76373..bc50b3bc3d 100644 --- a/integrations/claude-code-plugin/README.md +++ b/integrations/claude-code-plugin/README.md @@ -511,3 +511,18 @@ Captures POST to the `observe.record` wire op and crystallization to `observe.crystallize`. A failed POST is swallowed — the primer (`context.pack`) and the safety model work regardless. See [Claude Code](https://clickhouse.github.io/loom/features/integrations/claude-code/). + +## Invocation attribution + +Needs the next server release for invocation-correlation headers. Hook requests +declare hook origin and their entrypoint when known, including detached work. +The event can independently supply a conversation ID, `prompt_id` or `turn_id`, +tool-call ID and hook event. Missing IDs stay absent; conflicting prompt and +turn IDs are omitted. There is no shared latest-prompt binding. Trace headers +carry no prompt text, and identifiers pass through normal secret redaction. + +This labels requests the plugin sends. The host's separate MCP client may +supply its own per-call metadata; absent origin stays unknown. Neither host +prompt lifecycle tracing nor external host billing is required. See +[Loom invocation observability](../../docs/operations/monitoring.md#loom-invocation-observability) +for the optional benchmark fields, sampling limits and accounting rules. diff --git a/integrations/claude-code-plugin/lib/loom-client.mjs b/integrations/claude-code-plugin/lib/loom-client.mjs index 01b9c6668a..e23aa2748d 100644 --- a/integrations/claude-code-plugin/lib/loom-client.mjs +++ b/integrations/claude-code-plugin/lib/loom-client.mjs @@ -31,6 +31,7 @@ import { fileURLToPath } from "node:url"; // than that, and a shorter grace would kill a fire-and-forget post before it // left a slow link. import { DEFAULT_BOUND_MS, boundMs, rttCachePath } from "./hook-bounds.mjs"; +import { entrypointHeaders, promptHeaders } from "./prompt-context.mjs"; import { readJsonFile, tmpStatePath, writeJsonFile } from "./tmp-state.mjs"; let _rttCache; @@ -717,6 +718,9 @@ export function truncate(text, max = 2000) { return s.slice(0, max - 14).trimEnd() + "\n…[truncated]"; } +const _hookEntryHeaders = entrypointHeaders(process.argv[1], fileURLToPath(new URL("../hooks/", import.meta.url))); +let _hookPromptHeaders = { ..._hookEntryHeaders }; + // --- transport ---------------------------------------------------------------- // Every request the plugin makes goes through requestJson: one bound, one // header set, one error shape. Nothing here throws or rejects. @@ -736,6 +740,7 @@ export async function requestJson( headers: { ...(body === undefined ? {} : { "content-type": "application/json" }), ...(token ? { authorization: `Bearer ${token}` } : {}), + ..._hookPromptHeaders, ...headers, }, ...(body === undefined ? {} : { body: JSON.stringify(body) }), @@ -1088,8 +1093,15 @@ export function readStdinJson() { if (settled) return; settled = true; try { - resolve(normalizePayload(raw.trim() ? JSON.parse(raw) : {})); + const payload = normalizePayload(raw.trim() ? JSON.parse(raw) : {}); + const candidate = { ..._hookEntryHeaders, ...promptHeaders(payload) }; + // Identifiers still pass through redaction; no prompt or tool content is sent here. + _hookPromptHeaders = Object.fromEntries( + Object.entries(candidate).filter(([, value]) => redactSecrets(value) === value), + ); + resolve(payload); } catch { + _hookPromptHeaders = { ..._hookEntryHeaders }; resolve({}); } }; diff --git a/integrations/claude-code-plugin/lib/prompt-context.mjs b/integrations/claude-code-plugin/lib/prompt-context.mjs new file mode 100644 index 0000000000..1d8293458f --- /dev/null +++ b/integrations/claude-code-plugin/lib/prompt-context.mjs @@ -0,0 +1,24 @@ +import { basename, dirname, resolve } from "node:path"; + +// Caller metadata describes hook requests even when the host supplies no prompt identity. +const ID = /^[a-zA-Z0-9_.:-]{1,128}$/; +const valid = (value) => typeof value === "string" && ID.test(value) && !/[\r\n]/.test(value); + +export function promptHeaders(payload = {}) { + const headers = { "x-loom-call-origin": "hook", "x-loom-caller-type": "plugin" }; + const { session_id: session, prompt_id: prompt, turn_id: turn } = payload; + if (valid(session)) headers["x-loom-conversation-id"] = session; + if (!(prompt && turn && prompt !== turn) && valid(prompt || turn)) { + headers["x-loom-prompt-id"] = prompt || turn; + } + if (valid(payload.tool_use_id)) headers["x-loom-tool-call-id"] = payload.tool_use_id; + if (valid(payload.hook_event_name)) headers["x-loom-hook-event"] = payload.hook_event_name; + return headers; +} + +export function entrypointHeaders(entrypoint, hookDirectory) { + if (typeof entrypoint !== "string" || dirname(resolve(entrypoint)) !== resolve(hookDirectory)) return {}; + const name = basename(entrypoint, ".mjs"); + if (!/^loom-[a-z-]+$/.test(name)) return {}; + return { "x-loom-call-origin": "hook", "x-loom-caller-type": "plugin", "x-loom-hook-name": name }; +} diff --git a/integrations/claude-code-plugin/lib/usage.mjs b/integrations/claude-code-plugin/lib/usage.mjs index e850f24353..e3fe0c85b2 100644 --- a/integrations/claude-code-plugin/lib/usage.mjs +++ b/integrations/claude-code-plugin/lib/usage.mjs @@ -29,6 +29,14 @@ export function extractTurns(objs, seen = new Set(), { cap = 200 } = {}) { const u = msg.usage; const id = String(msg.id || ""); if (!u || !id || next.has(id)) continue; + // Missing counters are unknown; a later transcript row may complete them. + if ( + !Number.isSafeInteger(u.input_tokens) || + u.input_tokens < 0 || + !Number.isSafeInteger(u.output_tokens) || + u.output_tokens < 0 + ) + continue; const model = String(msg.model || ""); if (!model) continue; const cc = u.cache_creation || {}; diff --git a/integrations/claude-code-plugin/tests/prompt-context.test.mjs b/integrations/claude-code-plugin/tests/prompt-context.test.mjs new file mode 100644 index 0000000000..d96c63b3bc --- /dev/null +++ b/integrations/claude-code-plugin/tests/prompt-context.test.mjs @@ -0,0 +1,104 @@ +import "./temp-home.mjs"; +import assert from "node:assert/strict"; +import test from "node:test"; +import { spawnSync } from "node:child_process"; +import { promptHeaders } from "../lib/prompt-context.mjs"; + +test("host prompt IDs bind hook calls without capturing prompt text", () => { + const claude = promptHeaders({ session_id: "conversation-a", prompt_id: "prompt-a", prompt: "private text" }); + const codex = promptHeaders({ session_id: "conversation-a", turn_id: "prompt-a" }); + assert.equal(claude["x-loom-prompt-id"], "prompt-a"); + assert.equal(codex["x-loom-prompt-id"], "prompt-a"); + assert.equal(claude["x-loom-conversation-id"], "conversation-a"); + assert.equal(claude["x-loom-call-origin"], "hook"); + assert.equal(JSON.stringify(claude).includes("private text"), false); +}); + +test("optional IDs are independent and ambiguous prompt IDs are dropped", () => { + assert.equal(promptHeaders({ session_id: "conversation-a" })["x-loom-conversation-id"], "conversation-a"); + assert.equal(promptHeaders({ prompt_id: "prompt-a" })["x-loom-prompt-id"], "prompt-a"); + assert.equal(promptHeaders({ session_id: "c", prompt_id: "a", turn_id: "b" })["x-loom-prompt-id"], undefined); + assert.equal(promptHeaders({ session_id: "c", prompt_id: "bad\r\nheader" })["x-loom-prompt-id"], undefined); + assert.equal(promptHeaders({ session_id: "c", turn_id: "x".repeat(129) })["x-loom-prompt-id"], undefined); +}); + +test("identifiers reject trailing line breaks without losing hook origin", () => { + for (const suffix of ["\n", "\r", "\r\n"]) { + const badPrompt = promptHeaders({ session_id: "c", turn_id: `p${suffix}` }); + const badSession = promptHeaders({ session_id: `c${suffix}`, turn_id: "p" }); + assert.equal(badPrompt["x-loom-prompt-id"], undefined); + assert.equal(badSession["x-loom-conversation-id"], undefined); + assert.equal(badPrompt["x-loom-call-origin"], "hook"); + const headers = promptHeaders({ session_id: "c", turn_id: "p", tool_use_id: `call${suffix}` }); + assert.equal(headers["x-loom-tool-call-id"], undefined); + } +}); + +test("simultaneous turns and tool IDs stay separate", () => { + const a = promptHeaders({ session_id: "c", turn_id: "a", tool_use_id: "call-a", hook_event_name: "PreToolUse" }); + const b = promptHeaders({ session_id: "c", turn_id: "b", tool_use_id: "call-b", hook_event_name: "PostToolUse" }); + assert.equal(a["x-loom-prompt-id"], "a"); + assert.equal(b["x-loom-prompt-id"], "b"); + assert.equal(a["x-loom-tool-call-id"], "call-a"); + assert.equal(b["x-loom-tool-call-id"], "call-b"); + assert.equal(b["x-loom-hook-event"], "PostToolUse"); +}); + +test("the shared HTTP transport carries only the event's explicit prompt context", () => { + const url = new URL("../lib/loom-client.mjs", import.meta.url).href; + const script = ` + const { readStdinJson, requestJson } = await import(${JSON.stringify(url)}); + await readStdinJson(); + await requestJson("http://127.0.0.1:1/v1/memory.get", { + body: { memory_key: "example" }, + fetchImpl: async (url, options) => { + console.log(JSON.stringify({ headers: options.headers, body: options.body })); + return { status: 200, json: async () => ({}) }; + } + }); + `; + const env = { + ...process.env, + LOOM_CONFIG_FILE: "/nonexistent/loom-config.json", + LOOM_API_KEY_FILE: "/nonexistent/loom-token", + LOOM_API_KEY: "test-secret-must-not-leak", + CLAUDE_PLUGIN_OPTION_TOKEN: "", + }; + const capture = (payload) => { + const child = spawnSync(process.execPath, ["--input-type=module", "-e", script], { + input: JSON.stringify(payload), + encoding: "utf8", + timeout: 5000, + env, + }); + assert.equal(child.status, 0, child.stderr); + return JSON.parse(child.stdout); + }; + const a = capture({ session_id: "c", turn_id: "a", hook_event_name: "UserPromptSubmit", prompt: "private prompt" }); + const b = capture({ session_id: "c", turn_id: "b" }); + const absent = capture({ session_id: "c" }); + const secret = capture({ session_id: "c", turn_id: env.LOOM_API_KEY }); + assert.equal(a.headers["x-loom-prompt-id"], "a"); + assert.equal(b.headers["x-loom-prompt-id"], "b"); + assert.equal(absent.headers["x-loom-prompt-id"], undefined); + assert.equal(secret.headers["x-loom-prompt-id"], undefined); + assert.deepEqual(JSON.parse(a.body), { memory_key: "example" }); + assert.equal(JSON.stringify(a).includes("private prompt"), false); +}); + +test("invocation origin survives a hook event without a host prompt", () => { + const headers = promptHeaders({ session_id: "session-a", hook_event_name: "SessionStart" }); + assert.equal(headers["x-loom-call-origin"], "hook"); + assert.equal(headers["x-loom-hook-event"], "SessionStart"); + assert.equal(headers["x-loom-prompt-id"], undefined); + assert.equal(headers["x-loom-conversation-id"], "session-a"); +}); + +test("detached hooks declare their real entrypoint without inheriting a prompt", async () => { + const { entrypointHeaders } = await import("../lib/prompt-context.mjs"); + const headers = entrypointHeaders("/tmp/plugin/hooks/loom-flush.mjs", "/tmp/plugin/hooks"); + assert.equal(headers["x-loom-call-origin"], "hook"); + assert.equal(headers["x-loom-hook-name"], "loom-flush"); + assert.equal(headers["x-loom-prompt-id"], undefined); + assert.deepEqual(entrypointHeaders("/tmp/other/loom-flush.mjs", "/tmp/plugin/hooks"), {}); +}); diff --git a/integrations/claude-code-plugin/tests/usage.test.mjs b/integrations/claude-code-plugin/tests/usage.test.mjs index f75546c11d..c7861dfb13 100644 --- a/integrations/claude-code-plugin/tests/usage.test.mjs +++ b/integrations/claude-code-plugin/tests/usage.test.mjs @@ -3,6 +3,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { parseLines } from "../lib/transcript.mjs"; import { extractTurns, recordBody } from "../lib/usage.mjs"; +import { extractTurns as extractCodexTurns } from "../../codex-plugin/lib/usage.mjs"; const LINE = (id, extra = {}) => JSON.stringify({ @@ -73,3 +74,63 @@ test("recordBody carries the identity tuple and the consumer", () => { assert.equal(b.consumer, "claude-code"); assert.equal(b.turns.length, 1); }); + +for (const usage of [ + {}, + { input_tokens: 9 }, + { output_tokens: 2 }, + { input_tokens: -1, output_tokens: 2 }, + { input_tokens: 1.5, output_tokens: 2 }, + { input_tokens: "9", output_tokens: 2 }, +]) { + test(`unmeasured token counters do not create a billing row: ${JSON.stringify(usage)}`, () => { + const { turns, next } = extractTurns([{ type: "assistant", message: { id: "unknown", model: "model", usage } }]); + assert.deepEqual(turns, []); + assert.equal(next.has("unknown"), false); + }); +} + +test("measured zero token counters remain a recorded row", () => { + const { turns, next } = extractTurns([ + { + type: "assistant", + message: { id: "zero", model: "model", usage: { input_tokens: 0, output_tokens: 0 } }, + }, + ]); + assert.equal(turns.length, 1); + assert.equal(turns[0].input_tokens, 0); + assert.equal(turns[0].output_tokens, 0); + assert.equal(next.has("zero"), true); +}); + +test("an incomplete usage record does not hide a later measured record", () => { + const message = { id: "eventual", model: "model" }; + const { turns } = extractTurns([ + { type: "assistant", message: { ...message, usage: {} } }, + { type: "assistant", message: { ...message, usage: { input_tokens: 9, output_tokens: 2 } } }, + ]); + assert.equal(turns.length, 1); + assert.equal(turns[0].input_tokens, 9); + assert.equal(turns[0].output_tokens, 2); +}); + +test("Codex usage extraction preserves unknown counts, measured zero and message deduplication", () => { + const row = (id, usage) => ({ type: "assistant", message: { id, model: "model", usage } }); + const { turns, next } = extractCodexTurns([ + row("unknown", { input_tokens: 9 }), + row("eventual", {}), + row("zero", { input_tokens: 0, output_tokens: 0 }), + row("eventual", { input_tokens: 9, output_tokens: 2 }), + row("eventual", { input_tokens: 9, output_tokens: 2 }), + ]); + assert.deepEqual( + turns.map(({ message_id, input_tokens, output_tokens }) => [message_id, input_tokens, output_tokens]), + [ + ["zero", 0, 0], + ["eventual", 9, 2], + ], + ); + assert.equal(next.has("unknown"), false); + assert.equal(next.size, 2); + assert.deepEqual(extractCodexTurns([row("eventual", { input_tokens: 9, output_tokens: 2 })], next).turns, []); +}); diff --git a/integrations/codex-plugin/.codex-plugin/plugin.json b/integrations/codex-plugin/.codex-plugin/plugin.json index 0c0aac0de5..4573ec24d0 100644 --- a/integrations/codex-plugin/.codex-plugin/plugin.json +++ b/integrations/codex-plugin/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "loom", - "version": "1.2.45", + "version": "1.2.48", "description": "Shared long-term memory and code intelligence for Codex, self-hosted on ClickHouse. MCP tools to save and recall decisions, preferences and corrections across sessions and teammates; lifecycle hooks that capture each session with no model call; a code index that answers where-is, who-calls and blast-radius questions with line-accurate units instead of whole-file reads.", "author": { "name": "ClickHouse" diff --git a/integrations/codex-plugin/lib/loom-client.mjs b/integrations/codex-plugin/lib/loom-client.mjs index 01b9c6668a..e23aa2748d 100644 --- a/integrations/codex-plugin/lib/loom-client.mjs +++ b/integrations/codex-plugin/lib/loom-client.mjs @@ -31,6 +31,7 @@ import { fileURLToPath } from "node:url"; // than that, and a shorter grace would kill a fire-and-forget post before it // left a slow link. import { DEFAULT_BOUND_MS, boundMs, rttCachePath } from "./hook-bounds.mjs"; +import { entrypointHeaders, promptHeaders } from "./prompt-context.mjs"; import { readJsonFile, tmpStatePath, writeJsonFile } from "./tmp-state.mjs"; let _rttCache; @@ -717,6 +718,9 @@ export function truncate(text, max = 2000) { return s.slice(0, max - 14).trimEnd() + "\n…[truncated]"; } +const _hookEntryHeaders = entrypointHeaders(process.argv[1], fileURLToPath(new URL("../hooks/", import.meta.url))); +let _hookPromptHeaders = { ..._hookEntryHeaders }; + // --- transport ---------------------------------------------------------------- // Every request the plugin makes goes through requestJson: one bound, one // header set, one error shape. Nothing here throws or rejects. @@ -736,6 +740,7 @@ export async function requestJson( headers: { ...(body === undefined ? {} : { "content-type": "application/json" }), ...(token ? { authorization: `Bearer ${token}` } : {}), + ..._hookPromptHeaders, ...headers, }, ...(body === undefined ? {} : { body: JSON.stringify(body) }), @@ -1088,8 +1093,15 @@ export function readStdinJson() { if (settled) return; settled = true; try { - resolve(normalizePayload(raw.trim() ? JSON.parse(raw) : {})); + const payload = normalizePayload(raw.trim() ? JSON.parse(raw) : {}); + const candidate = { ..._hookEntryHeaders, ...promptHeaders(payload) }; + // Identifiers still pass through redaction; no prompt or tool content is sent here. + _hookPromptHeaders = Object.fromEntries( + Object.entries(candidate).filter(([, value]) => redactSecrets(value) === value), + ); + resolve(payload); } catch { + _hookPromptHeaders = { ..._hookEntryHeaders }; resolve({}); } }; diff --git a/integrations/codex-plugin/lib/prompt-context.mjs b/integrations/codex-plugin/lib/prompt-context.mjs new file mode 100644 index 0000000000..1d8293458f --- /dev/null +++ b/integrations/codex-plugin/lib/prompt-context.mjs @@ -0,0 +1,24 @@ +import { basename, dirname, resolve } from "node:path"; + +// Caller metadata describes hook requests even when the host supplies no prompt identity. +const ID = /^[a-zA-Z0-9_.:-]{1,128}$/; +const valid = (value) => typeof value === "string" && ID.test(value) && !/[\r\n]/.test(value); + +export function promptHeaders(payload = {}) { + const headers = { "x-loom-call-origin": "hook", "x-loom-caller-type": "plugin" }; + const { session_id: session, prompt_id: prompt, turn_id: turn } = payload; + if (valid(session)) headers["x-loom-conversation-id"] = session; + if (!(prompt && turn && prompt !== turn) && valid(prompt || turn)) { + headers["x-loom-prompt-id"] = prompt || turn; + } + if (valid(payload.tool_use_id)) headers["x-loom-tool-call-id"] = payload.tool_use_id; + if (valid(payload.hook_event_name)) headers["x-loom-hook-event"] = payload.hook_event_name; + return headers; +} + +export function entrypointHeaders(entrypoint, hookDirectory) { + if (typeof entrypoint !== "string" || dirname(resolve(entrypoint)) !== resolve(hookDirectory)) return {}; + const name = basename(entrypoint, ".mjs"); + if (!/^loom-[a-z-]+$/.test(name)) return {}; + return { "x-loom-call-origin": "hook", "x-loom-caller-type": "plugin", "x-loom-hook-name": name }; +} diff --git a/integrations/codex-plugin/lib/usage.mjs b/integrations/codex-plugin/lib/usage.mjs index e850f24353..e3fe0c85b2 100644 --- a/integrations/codex-plugin/lib/usage.mjs +++ b/integrations/codex-plugin/lib/usage.mjs @@ -29,6 +29,14 @@ export function extractTurns(objs, seen = new Set(), { cap = 200 } = {}) { const u = msg.usage; const id = String(msg.id || ""); if (!u || !id || next.has(id)) continue; + // Missing counters are unknown; a later transcript row may complete them. + if ( + !Number.isSafeInteger(u.input_tokens) || + u.input_tokens < 0 || + !Number.isSafeInteger(u.output_tokens) || + u.output_tokens < 0 + ) + continue; const model = String(msg.model || ""); if (!model) continue; const cc = u.cache_creation || {}; diff --git a/server/observability/genai.py b/server/observability/genai.py index b4682c8e5b..76858d7324 100644 --- a/server/observability/genai.py +++ b/server/observability/genai.py @@ -387,8 +387,11 @@ def stamp_identity(span: Any) -> None: `loom.org`, so a span carrying the user without the org could be erased for another org's user of the same id. """ + from server.observability import prompt_context + ident = _identity.get() if not ident: + prompt_context.stamp(span) return for key, attr in ( ("operation", "loom.operation"), @@ -419,6 +422,7 @@ def stamp_identity(span: Any) -> None: # sees first, and a batch that reaches it before the root would produce # a correct trace with an empty name. span.set_attribute("langfuse.trace.name", ident["operation"]) + prompt_context.stamp(span) @contextlib.contextmanager diff --git a/server/observability/otel.py b/server/observability/otel.py index b1398d5664..3439508fa8 100644 --- a/server/observability/otel.py +++ b/server/observability/otel.py @@ -547,14 +547,18 @@ def worker_pass_span(name: str) -> Any: @contextlib.contextmanager def _cm() -> Iterator[Any]: - with tracer.start_as_current_span(f"worker.{name}.pass") as span: - try: - span.set_attribute("loom.worker", name) - span.set_attribute("loom.operation", f"worker.{name}") - span.set_attribute("langfuse.trace.name", f"worker.{name}") - except Exception: # noqa: BLE001 — span attrs must not fail the pass - pass - yield span + from server.observability import prompt_context + + with prompt_context.bind({"origin": "worker", "caller_type": "worker"}): + with tracer.start_as_current_span(f"worker.{name}.pass") as span: + try: + span.set_attribute("loom.worker", name) + span.set_attribute("loom.operation", f"worker.{name}") + span.set_attribute("langfuse.trace.name", f"worker.{name}") + prompt_context.stamp(span) + except Exception: # noqa: BLE001 — span attrs must not fail the pass + pass + yield span return _cm() except Exception: # noqa: BLE001 — tracing must never fail the pass @@ -607,6 +611,8 @@ def set_loom_attributes( span = trace.get_current_span() if span is None or not span.is_recording(): return + from server.observability import prompt_context + span.set_attribute("loom.operation", op_name) span.set_attribute("loom.org", org) span.set_attribute("loom.namespace", namespace) @@ -631,6 +637,7 @@ def set_loom_attributes( # correct trace with an EMPTY name). The loom operation is both stable # and more useful than the HTTP route. span.set_attribute("langfuse.trace.name", op_name) + prompt_context.stamp(span) except Exception: # noqa: BLE001 — trace attrs must never fail a request pass diff --git a/server/observability/prompt_context.py b/server/observability/prompt_context.py new file mode 100644 index 0000000000..966e6b6b08 --- /dev/null +++ b/server/observability/prompt_context.py @@ -0,0 +1,271 @@ +"""Bounded caller metadata for actual Loom invocations, independent of authorization.""" + +from __future__ import annotations + +import contextlib +import contextvars +import re +import sys +import time +import uuid +from collections.abc import Iterator, Mapping +from typing import Any + +_ID = re.compile(r"[a-zA-Z0-9_.:-]{1,128}\Z") +_current: contextvars.ContextVar[dict[str, str] | None] = contextvars.ContextVar( + "loom_prompt_context", default=None +) +_FIELDS = ( + "conversation_id", + "prompt_id", + "tool_call_id", + "hook_event", + "hook_name", + "caller_type", + "benchmark_run_id", + "benchmark_case_id", + "benchmark_variant", +) +_ORIGINS = {"tool", "hook", "worker", "unknown"} +_CALLERS = {"plugin", "sdk", "cli", "agent", "worker"} + + +def _identifier(value: Any) -> str: + return value if isinstance(value, str) and _ID.fullmatch(value) else "" + + +def parse(value: Any) -> dict[str, str]: + source = value if isinstance(value, Mapping) else {} + parsed = {key: _identifier(source.get(key)) for key in _FIELDS} + if parsed["caller_type"] not in _CALLERS: + parsed.pop("caller_type") + origin = source.get("origin") + parsed["origin"] = origin if isinstance(origin, str) and origin in _ORIGINS else "unknown" + return {key: item for key, item in parsed.items() if item} + + +def from_headers(headers: Mapping[str, str]) -> dict[str, str]: + values = {key: headers.get("x-loom-" + key.replace("_", "-")) for key in _FIELDS} + values["origin"] = headers.get("x-loom-call-origin") + return parse(values) + + +@contextlib.contextmanager +def bind(value: Any, *, transport: str = "") -> Iterator[None]: + values = parse(value) + if transport: + values["transport"] = transport + token = _current.set(values) + try: + yield + finally: + _current.reset(token) + + +def stamp(span: Any) -> None: + values = _current.get() or {} + if values.get("conversation_id"): + span.set_attribute("langfuse.session.id", values["conversation_id"]) + for key, value in values.items(): + if key.startswith("benchmark_"): + attr = "loom.benchmark." + key.removeprefix("benchmark_") + elif key in {"origin", "transport", "caller_type", "hook_event", "hook_name"}: + attr = "loom.call." + key + else: + attr = "loom.prompt." + key + span.set_attribute(attr, value) + span.set_attribute("langfuse.trace.metadata.loom_" + key, value) + + +def authenticated(span: Any, auth: Any, operation: str, request_id: str = "") -> None: + """Keep early refusals discoverable without trusting a requested namespace.""" + if span is None or auth is None: + return + try: + from server.observability.otel import set_loom_attributes + + set_loom_attributes( + op_name=operation, + org=auth.org, + namespace="-", + agent="-", + user_id=auth.subject, + request_id=request_id, + capture_level="off", + ) + except Exception: # noqa: BLE001 — optional identity stamping must not fail a call + pass + + +@contextlib.contextmanager +def mcp_call( + params: Mapping[str, Any], + *, + headers: Mapping[str, str] | None = None, + operation: str = "tools/call", +) -> Iterator[Any]: + meta = params.get("_meta") + meta = meta if isinstance(meta, dict) else {} + values = from_headers(headers or {}) + supplied = meta.get("loom") + if isinstance(supplied, Mapping): + values.update(supplied) + carrier = dict(headers or {}) + carrier.update(meta) + with bind(values, transport="mcp"): + with _call_span("mcp.tool", carrier, operation) as span: + yield span + + +@contextlib.contextmanager +def http_call(headers: Mapping[str, str], operation: str) -> Iterator[Any]: + with bind(from_headers(headers), transport="http"): + with _call_span("loom.http", headers, operation) as span: + yield span + + +@contextlib.contextmanager +def _call_span(name: str, carrier: Mapping[str, Any], operation: str) -> Iterator[Any]: + from server.core.diagnostics import codes_only + from server.observability import genai + + identity = genai._identity.set({}) + try: + if codes_only(): + yield None + return + with _optional_span(name, carrier) as span: + started = time.perf_counter() + _attributes(span, {"loom.operation": operation, "langfuse.trace.name": operation}) + try: + yield span + except BaseException as error: + status = getattr(error, "status_code", 0) + detail = getattr(error, "detail", None) + code = ( + detail.get("code") + if isinstance(detail, Mapping) + else getattr(error, "code", None) + ) + result = ( + "timeout" + if isinstance(error, TimeoutError) or _timed_out(code, status) + else ("refusal" if isinstance(status, int) and 400 <= status < 500 else "error") + ) + _outcome(span, result) + raise + finally: + _attributes(span, {"loom.call.duration_ms": (time.perf_counter() - started) * 1000}) + finally: + genai._identity.reset(identity) + + +@contextlib.contextmanager +def _optional_span(name: str, meta: Mapping[str, Any]) -> Iterator[Any]: + try: + from opentelemetry import trace + from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + + carrier = { + key: meta[key] + for key in ("traceparent", "tracestate") + if isinstance(meta.get(key), str) and len(meta[key]) <= 512 + } + remote = TraceContextTextMapPropagator().extract(carrier) + context = remote if trace.get_current_span(remote).get_span_context().is_valid else None + manager = trace.get_tracer("loom.handlers").start_as_current_span( + name, + context=context, + record_exception=False, + set_status_on_exception=False, + attributes={ + "loom.call.root": True, + "loom.call.attempt_id": str(uuid.uuid4()), + "langfuse.observation.type": "tool", + }, + ) + span = manager.__enter__() + except Exception: # noqa: BLE001 — optional tracing must not fail a call + yield None + return + try: + try: + if span is not None: + stamp(span) + except Exception: # noqa: BLE001 — caller metadata must not fail a call + pass + yield span + finally: + try: + manager.__exit__(*sys.exc_info()) + except Exception: # noqa: BLE001 — closing telemetry must not fail a call + pass + + +def _attributes(span: Any, values: dict[str, Any]) -> None: + if span is None: + return + try: + for key, value in values.items(): + span.set_attribute(key, value) + except Exception: # noqa: BLE001 — metadata must not change a response + pass + + +def _outcome(span: Any, value: str) -> None: + if span is None: + return + try: + from opentelemetry.trace import StatusCode + + _attributes( + span, + { + "loom.call.outcome": value, + "loom.call.success": value == "success", + "langfuse.observation.metadata.loom_outcome": value, + }, + ) + if value != "success": + span.set_status(StatusCode.ERROR) + except Exception: # noqa: BLE001 — outcome telemetry must not fail a call + pass + + +def _timed_out(code: Any, status: Any) -> bool: + return code in ("BACKEND_TIMEOUT", "EMBED_TIMEOUT", "TIMEOUT") or status in (408, 504) + + +def outcome( + span: Any, response: Mapping[str, Any], *, protocol_refusals: tuple[int, ...] = () +) -> None: + result = response.get("result") + error = response.get("error") + if isinstance(error, dict): + code = error.get("code") + detail = error.get("data") + detail = detail if isinstance(detail, dict) else {} + value = ( + "refusal" + if isinstance(code, int) and code in (-32600, -32601, -32602, *protocol_refusals) + else "error" + ) + elif isinstance(result, dict) and result.get("isError"): + detail = result.get("structuredContent") + detail = detail if isinstance(detail, dict) else {} + value = ( + "refusal" + if detail.get("category") in ("input", "auth", "capacity", "configuration") + else "error" + ) + else: + detail, value = {}, "success" + if _timed_out(detail.get("code"), detail.get("status")): + value = "timeout" + _outcome(span, value) + + +def http_outcome(span: Any, status: int | None, request_id: str = "") -> None: + status = 200 if status is None else status + _attributes(span, {"loom.request_id": request_id} if request_id else {}) + _outcome(span, "success" if status < 400 else "refusal" if status < 500 else "error") diff --git a/server/transport/http.py b/server/transport/http.py index 20286aa4e5..f1773aab52 100644 --- a/server/transport/http.py +++ b/server/transport/http.py @@ -752,7 +752,7 @@ async def _finish_success( def _make_endpoint(op: Operation) -> Callable[[Request, Response], Awaitable[Any]]: - async def endpoint(request: Request, response: Response) -> Any: + async def endpoint(request: Request, response: Response, span: Any) -> Any: # The phases run in this order and each refusal raises its own HTTP # error: token, session protection, body, validation, authorization, policy, rate limit, # idempotency replay, then the handler. Nothing after a gate runs @@ -769,8 +769,10 @@ async def endpoint(request: Request, response: Response) -> Any: # Bound before the body is read, as MCP binds it before validation: # a refusal's ledger row reads the token's org and pins from it. from server.auth.context import set_current_auth + from server.observability.prompt_context import authenticated set_current_auth(auth) + authenticated(span, auth, op.name, request_id) bind_claimed_class(auth) await gates.session_content(op, auth) body = await _read_body(op, request, request_id, auth) @@ -792,10 +794,10 @@ async def endpoint(request: Request, response: Response) -> Any: payload = gates.with_target(op, payload, target) payload = await _bind_context_and_policy(op, request, call, target, payload) _rate_limit(op, response, call, target) + await _start_telemetry(op, response, call) replay = await _idempotency_gate(op, request, response, call, target, payload) if replay.hit is not None: return replay.hit.body - await _start_telemetry(op, response, call) with required_call_exit( op_name=op.name, @@ -811,7 +813,17 @@ async def endpoint(request: Request, response: Response) -> Any: op, call, payload, result, elapsed_ms, (replay.scope, replay.body_hash) ) - return endpoint + async def with_prompt(request: Request, response: Response) -> Any: + from server.observability import prompt_context + + with prompt_context.http_call(request.headers, getattr(op, "name", "http")) as span: + result = await endpoint(request, response, span) + prompt_context.http_outcome( + span, response.status_code, response.headers.get("X-Request-Id", "") + ) + return result + + return with_prompt def _success_headers( diff --git a/server/transport/mcp.py b/server/transport/mcp.py index 70946a13d8..4d0059ca5e 100644 --- a/server/transport/mcp.py +++ b/server/transport/mcp.py @@ -32,7 +32,8 @@ from __future__ import annotations import logging -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Mapping +from functools import partial from typing import Any from fastapi import APIRouter, FastAPI, Request, Response @@ -57,6 +58,7 @@ ) from server.transport.mcp_call import _call_tool from server.transport.mcp_config import ( + _LOOM_ACTIVATE_TOOL_NAME, _TRANSPORT_ERROR_ID, INVALID_PARAMS, INVALID_REQUEST, @@ -67,6 +69,7 @@ METHOD_NOT_FOUND, PARSE_ERROR, SERVER_INFO, + UNSUPPORTED_PROTOCOL_VERSION, _err, _ok, cache_hints, @@ -123,6 +126,49 @@ def _listing_params(request: Any) -> dict[str, str | None]: } +def _call_operation(params: dict[str, Any]) -> str: + from server.operations.registry import OPERATIONS_BY_NAME + from server.transport.mcp_op import CATALOG_ALIASES, TOOLS_BY_NAME, resolve_op + + name = params.get("name") + if name == _LOOM_ACTIVATE_TOOL_NAME: + return _LOOM_ACTIVATE_TOOL_NAME + op = TOOLS_BY_NAME.get(name) if isinstance(name, str) else None + if name == "loom_op": + arguments = params.get("arguments") or {} + if not isinstance(arguments, dict): + return "loom_op" + target = str(arguments.get("op") or "").strip() + if target.lower() in CATALOG_ALIASES: + return "loom_op.help" + op = resolve_op(target, OPERATIONS_BY_NAME) + return op.name if op else "loom_op" + return op.name if op else "tools/call" + + +async def _observe_call( + payload: Any, + *, + headers: Mapping[str, str], + dispatch: Callable[[Any], Awaitable[dict[str, Any] | None]], + auth: Any = None, +) -> dict[str, Any] | None: + from server.observability import prompt_context + + if not isinstance(payload, dict) or payload.get("method") != "tools/call": + return await dispatch(payload) + params = payload.get("params") + params = params if isinstance(params, dict) else {} + with prompt_context.mcp_call( + params, headers=headers, operation=_call_operation(params) + ) as span: + prompt_context.authenticated(span, auth, _call_operation(params)) + result = await dispatch(payload) + if result is not None: + prompt_context.outcome(span, result, protocol_refusals=(UNSUPPORTED_PROTOCOL_VERSION,)) + return result + + async def _handle_request( payload: dict[str, Any], auth: Any = None, @@ -130,6 +176,22 @@ async def _handle_request( mcp_session_id: str = "", schema_profile: str | None = None, toolset: str | None = None, +) -> dict[str, Any] | None: + async def dispatch(req: Any) -> dict[str, Any] | None: + return await _dispatch_request( + req, auth, client_name, mcp_session_id, schema_profile, toolset + ) + + return await _observe_call(payload, headers={}, dispatch=dispatch, auth=auth) + + +async def _dispatch_request( + payload: dict[str, Any], + auth: Any = None, + client_name: str | None = None, + mcp_session_id: str = "", + schema_profile: str | None = None, + toolset: str | None = None, ) -> dict[str, Any] | None: """Process one JSON-RPC request. Returns response or None (notification). @@ -226,19 +288,17 @@ async def _handle_request( if method == "tools/call": from server.transport.lean_results import single_channel - return single_channel( - classify_tool_errors( - await _call_tool( - req_id, - params, - auth, - client_name, - mcp_session_id=mcp_session_id, - toolset=toolset, - ) - ), - _client_name_to_agent(client_name), + result = classify_tool_errors( + await _call_tool( + req_id, + params, + auth, + client_name, + mcp_session_id=mcp_session_id, + toolset=toolset, + ) ) + return single_channel(result, _client_name_to_agent(client_name)) if method == "prompts/list": return _ok( @@ -409,7 +469,7 @@ async def mcp_endpoint(request: Request, response: Response) -> Any: message=f"missing bearer token ({gated} requires auth in mcp_auth_mode={mode!r})", ) - async def _dispatch(req: Any) -> dict[str, Any] | None: + async def _dispatch_one(req: Any) -> dict[str, Any] | None: nonlocal session_id if not isinstance(req, dict): return await _handle_request(req) # Invalid Request, per entry @@ -424,11 +484,7 @@ async def _dispatch(req: Any) -> dict[str, Any] | None: if not session_id: session_id = new_session_id(_initialize_client_name(req)) response.headers["Mcp-Session-Id"] = session_id - params = req.get("params") - if isinstance(params, dict): - client_info = params.get("clientInfo") - if isinstance(client_info, dict): - _remember_client_for_session(session_id, client_info.get("name")) + _remember_client_for_session(session_id, _initialize_client_name(req)) client_name = _resolve_client_name(req, session_id) # One read of the selectors, so routing, the screen and the call agree. listing = _listing_params(request) @@ -445,7 +501,7 @@ async def _dispatch(req: Any) -> dict[str, Any] | None: ) if screened is not None: return classify_tool_errors(screened) - return await _handle_request( + return await _dispatch_request( req, auth, client_name, @@ -453,6 +509,10 @@ async def _dispatch(req: Any) -> dict[str, Any] | None: **listing, ) + _dispatch = partial( + _observe_call, headers=request.headers, dispatch=_dispatch_one, auth=auth + ) + if isinstance(payload, list): responses = await _dispatch_batch(payload, _dispatch) return responses if responses else _accepted(response) diff --git a/server/transport/mcp_call.py b/server/transport/mcp_call.py index b7cc26445a..475ee4af39 100644 --- a/server/transport/mcp_call.py +++ b/server/transport/mcp_call.py @@ -247,9 +247,9 @@ async def _call_tool( """One MCP `tools/call`, in phases (each below this function, in order): resolve the tool → session-content admission → bind the caller's identity → stamp the session run's id → argument refusal + validation → authorization → admission (traffic - class, policy, rate limit) → the session run begins → idempotency replay - (only for an admitted caller) → open the trace → span identity - (inline) → the handler → success telemetry → the response. Every phase + class, policy, rate limit) → the session run begins → stamp admitted trace + identity → idempotency replay (only for an admitted caller) → the handler → + success telemetry → the response. Every phase that can end the call returns its response and this function returns it at once, so no gate runs after a refusal and none runs after the handler.""" @@ -317,21 +317,8 @@ async def _call_tool( await _bind_session_run(op, defaults, mcp_session_id, run_caller) - # The replay cache is read only for an admitted caller. - replay, idem_scope, idem_body_hash = await _idempotency_gate(call, name, client_idem_key) - if replay is not None: - return replay # IDEMPOTENT_CONFLICT refusal, or the cached result replayed - - # Scope the span and publish the identity ContextVar BEFORE the handler runs. - # The two `set_loom_attributes` calls after `op.handler(...)` (in - # `_handler_failed` / `_record_success_telemetry`) happen once it - # returns, which is fine for span attributes but useless for the provider - # spans: here `set_loom_attributes` sets the identity, and every `llm.*` / - # `embed.*` span of this request is created INSIDE the handler. - # Set later, MCP generations would carry no loom.org/namespace/operation - # and no Langfuse session or user. In a JSON-RPC batch the ContextVar is - # task-scoped, so entry N's provider spans would carry entry N-1's tenant. - # Setting it here, once per entry before any LLM call, prevents both. + # Replays need the admitted tenant's tags for trace discovery and erasure. + # Provider children also read this identity before the handler returns. try: from server.observability import capture_consent as _consent from server.observability import otel as _otel_pre @@ -354,6 +341,11 @@ async def _call_tool( # caches and refuse. genai.set_identity(operation=op.name, capture_level="off") + # The replay cache is read only for an admitted caller. + replay, idem_scope, idem_body_hash = await _idempotency_gate(call, name, client_idem_key) + if replay is not None: + return replay # IDEMPOTENT_CONFLICT refusal, or the cached result replayed + with required_call_exit( op_name=op.name, identity=ledger_identity(arguments), diff --git a/server/transport/mcp_op.py b/server/transport/mcp_op.py index 0316679ac3..2ca1a66353 100644 --- a/server/transport/mcp_op.py +++ b/server/transport/mcp_op.py @@ -28,7 +28,7 @@ import difflib import re -from collections.abc import Set +from collections.abc import Mapping, Set from typing import Any from server.operations.base import Operation @@ -40,7 +40,7 @@ LOOM_OP_TOOL_NAME = "loom_op" # The code for a name no op (or tool) answers to, direct or through loom_op. UNKNOWN_OP = "UNKNOWN_OP" -_HELP = {"", "help", "ops", "list", "catalog", "?"} +CATALOG_ALIASES = {"", "help", "ops", "list", "catalog", "?"} _CATALOG_LIMIT = 15 @@ -342,6 +342,12 @@ def _result(req_id: Any, body: dict[str, Any], *, error: bool = False) -> dict[s ) +def resolve_op(name: str, operations: Mapping[str, Operation]) -> Operation | None: + """Resolve dotted names and the first-underscore alias accepted by `loom_op`.""" + name = name.strip() + return operations.get(name) or operations.get(name.replace("_", ".", 1)) + + def route(req: Any, *, toolset: str | None, auth: Any) -> tuple[Any, dict[str, Any] | None]: """`(req, None)` unchanged unless it is a `loom_op` call; the rewritten `tools/call` for a real op; or `(req, response)` for help / unknown ops. @@ -359,11 +365,11 @@ def route(req: Any, *, toolset: str | None, auth: Any) -> tuple[Any, dict[str, A if assistant_connection(toolset): return req, None # `_call_tool` refuses `loom_op` on the assistant toolset name = str(args.get("op") or "").strip() - if name.lower() in _HELP: + if name.lower() in CATALOG_ALIASES: return req, _result(req.get("id"), catalog(str(args.get("query") or ""), toolset=toolset)) # a hidden op is not here, so its name gets hidden_refusal by_name = {op.name: op for op in ALL_OPERATIONS if not is_hidden(op.name)} - op = by_name.get(name) or by_name.get(name.replace("_", ".", 1)) + op = resolve_op(name, by_name) writes = routes_writes(toolset) if op is None and (hidden := hidden_refusal(name, toolset)) is not None: emit_token_refusal(name, auth, error_code="OP_NOT_ON_MCP") diff --git a/tests/unit/session_content_senders.json b/tests/unit/session_content_senders.json index a88a417f63..259eec190d 100644 --- a/tests/unit/session_content_senders.json +++ b/tests/unit/session_content_senders.json @@ -375,7 +375,7 @@ "integrations/claude-code-plugin/lib/collections-line.mjs": "d50ec2cbae9ef0058fcf8e26ea024a79c67e57014c39447f58b54a6837654183", "integrations/claude-code-plugin/lib/first-session.mjs": "e9183d950899f3ef78bdd17ba677d25e9ff5baffddc97ca282fecc5a80a92651", "integrations/claude-code-plugin/lib/flush-job.mjs": "943dbf075ad27a9e8ac9c491dc1e523d0a690ec8fc9acc71e597a6b1cadf5a01", - "integrations/claude-code-plugin/lib/loom-client.mjs": "d1cc7c8bb6674dfb6d2c76045e6e8c6cbce2fb806996fce8aafba5f917002cb6", + "integrations/claude-code-plugin/lib/loom-client.mjs": "7527ce145616516f36c28bcaab778b5c8db861de09ea54b5ac4f79cc060dfa02", "integrations/claude-code-plugin/lib/namespace-health.mjs": "02db4a35195b32675c3d929350ddeca8a8b991c08e46b206fb38586efbd5cc7e", "integrations/claude-code-plugin/lib/prompt-pack.mjs": "8730ea72d40e6022a6e314f1f6703dd4f38f10c4e719886f319d88f7c3d15abf", "integrations/claude-code-plugin/lib/repo-scope.mjs": "127a7daefbe4cdaf014a58351f1b3cec068220985ac940a2952f161a8c881108", @@ -390,7 +390,7 @@ "integrations/codex-plugin/lib/collections-line.mjs": "d50ec2cbae9ef0058fcf8e26ea024a79c67e57014c39447f58b54a6837654183", "integrations/codex-plugin/lib/first-session.mjs": "e9183d950899f3ef78bdd17ba677d25e9ff5baffddc97ca282fecc5a80a92651", "integrations/codex-plugin/lib/flush-job.mjs": "943dbf075ad27a9e8ac9c491dc1e523d0a690ec8fc9acc71e597a6b1cadf5a01", - "integrations/codex-plugin/lib/loom-client.mjs": "d1cc7c8bb6674dfb6d2c76045e6e8c6cbce2fb806996fce8aafba5f917002cb6", + "integrations/codex-plugin/lib/loom-client.mjs": "7527ce145616516f36c28bcaab778b5c8db861de09ea54b5ac4f79cc060dfa02", "integrations/codex-plugin/lib/namespace-health.mjs": "02db4a35195b32675c3d929350ddeca8a8b991c08e46b206fb38586efbd5cc7e", "integrations/codex-plugin/lib/prompt-pack.mjs": "8730ea72d40e6022a6e314f1f6703dd4f38f10c4e719886f319d88f7c3d15abf", "integrations/codex-plugin/lib/repo-scope.mjs": "127a7daefbe4cdaf014a58351f1b3cec068220985ac940a2952f161a8c881108", diff --git a/tests/unit/test_mcp_malformed_requests.py b/tests/unit/test_mcp_malformed_requests.py index 868adaec23..7d279005fe 100644 --- a/tests/unit/test_mcp_malformed_requests.py +++ b/tests/unit/test_mcp_malformed_requests.py @@ -114,13 +114,13 @@ def test_a_batch_past_its_cap_or_empty_is_refused_before_any_entry_runs( client = _client(monkeypatch, auth=auth) ran: list[Any] = [] - handle = mcp_mod._handle_request + dispatch = mcp_mod._dispatch_request async def counted(*args: Any, **kwargs: Any) -> Any: ran.append(1) - return await handle(*args, **kwargs) + return await dispatch(*args, **kwargs) - monkeypatch.setattr(mcp_mod, "_handle_request", counted) + monkeypatch.setattr(mcp_mod, "_dispatch_request", counted) ping = {"jsonrpc": "2.0", "id": 1, "method": "ping"} for body in ([ping] * (MAX_BATCH_ENTRIES + 1), []): r = client.post("/mcp", json=body, headers=_headers(auth)) diff --git a/tests/unit/test_mirror_publication_batch.py b/tests/unit/test_mirror_publication_batch.py index 1aab33b7ea..b1e0a5921b 100644 --- a/tests/unit/test_mirror_publication_batch.py +++ b/tests/unit/test_mirror_publication_batch.py @@ -331,7 +331,7 @@ async def dispatch(req, *_args, **_kwargs): monkeypatch.setattr(mcp, "_effective_auth_mode", lambda: "open") monkeypatch.setattr(mcp, "_verify_bearer", AsyncMock(return_value=None)) - monkeypatch.setattr(mcp, "_handle_request", dispatch) + monkeypatch.setattr(mcp, "_dispatch_request", dispatch) payload = request(1, "ping") if notification: payload.pop("id") diff --git a/tests/unit/test_prompt_context.py b/tests/unit/test_prompt_context.py new file mode 100644 index 0000000000..d7e6a68238 --- /dev/null +++ b/tests/unit/test_prompt_context.py @@ -0,0 +1,764 @@ +"""Explicit prompt correlation never changes stored identity or fills missing causality.""" + +from __future__ import annotations + +import asyncio +from contextlib import contextmanager + +import pytest +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from server.observability import genai, prompt_context +from tests.unit.test_http_idempotency_after_auth import client as client +from tests.unit.test_mcp_idempotency_replay_conflict import gated as gated +from tests.unit.test_mcp_idempotency_replay_conflict import probe as probe + + +@pytest.fixture +def spans(monkeypatch): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + monkeypatch.setattr(trace, "get_tracer", provider.get_tracer) + identity_token = genai._identity.set({}) + yield exporter + genai._identity.reset(identity_token) + provider.shutdown() + + +def metadata(prompt="prompt-a"): + return { + "conversation_id": "conversation", + "prompt_id": prompt, + "origin": "model", + "tool_call_id": "call-a", + } + + +def test_mcp_restores_explicit_parent_and_stamps_provider_children(spans): + meta = { + "traceparent": "00-11111111111111111111111111111111-2222222222222222-01", + "loom": metadata(), + } + with prompt_context.mcp_call({"_meta": meta}) as call: + genai.set_identity( + org="org", namespace="ns", operation="memory.get", session_id="stored-session" + ) + genai.stamp_identity(call) + with trace.get_tracer("test").start_as_current_span("embed.provider") as child: + genai.stamp_identity(child) + prompt_context.outcome(call, {"result": {"isError": False}}) + child, root = spans.get_finished_spans() + assert root.context.trace_id == int("1" * 32, 16) + assert root.parent.span_id == int("2" * 16, 16) + assert child.parent.span_id == root.context.span_id + assert child.attributes["loom.prompt.prompt_id"] == "prompt-a" + assert child.attributes["langfuse.session.id"] == "conversation" + assert child.attributes["loom.org"] == "org" + assert root.attributes["loom.call.root"] is True + assert root.attributes["loom.call.success"] is True + assert not any(key.startswith("gen_ai.usage") for key in root.attributes) + + +def test_errors_and_retries_have_one_span_per_attempt(spans): + for failed in (True, False): + with prompt_context.mcp_call({"_meta": {"loom": metadata()}}) as span: + genai.set_identity(org="org", namespace="ns") + genai.stamp_identity(span) + prompt_context.outcome(span, {"result": {"isError": failed}}) + first, second = spans.get_finished_spans() + assert first.context.span_id != second.context.span_id + assert first.attributes["loom.call.success"] is False + assert first.status.status_code is trace.StatusCode.ERROR + assert second.attributes["loom.call.success"] is True + assert ( + first.attributes["loom.prompt.tool_call_id"] + == second.attributes["loom.prompt.tool_call_id"] + ) + + +@pytest.mark.asyncio +async def test_concurrent_turns_and_unattributed_call_do_not_share_context(spans): + entered = asyncio.Event() + release = asyncio.Event() + + async def first(): + with prompt_context.bind(metadata("a")): + entered.set() + await release.wait() + with trace.get_tracer("test").start_as_current_span("a") as span: + prompt_context.stamp(span) + + async def second(): + await entered.wait() + with prompt_context.bind(metadata("b")): + with trace.get_tracer("test").start_as_current_span("b") as span: + prompt_context.stamp(span) + release.set() + + await asyncio.gather(first(), second()) + with prompt_context.mcp_call({"_meta": {"traceparent": "malformed"}}): + pass + by_name = {span.name: span for span in spans.get_finished_spans()} + assert by_name["a"].attributes["loom.prompt.prompt_id"] == "a" + assert by_name["b"].attributes["loom.prompt.prompt_id"] == "b" + assert "loom.prompt.prompt_id" not in by_name["mcp.tool"].attributes + + +def test_optional_ids_are_independent_and_reject_content(): + assert prompt_context.from_headers({"x-loom-prompt-id": "p"}) == { + "origin": "unknown", + "prompt_id": "p", + } + assert prompt_context.parse({"conversation_id": "s", "prompt_id": "line\nbreak"}) == { + "conversation_id": "s", + "origin": "unknown", + } + assert prompt_context.parse({"conversation_id": "s", "prompt_id": "x" * 129}) == { + "conversation_id": "s", + "origin": "unknown", + } + parsed = prompt_context.from_headers( + {"x-loom-conversation-id": "s", "x-loom-prompt-id": "p", "x-loom-call-origin": "hook"} + ) + assert parsed == {"conversation_id": "s", "prompt_id": "p", "origin": "hook"} + + +def test_broken_optional_tracer_does_not_prevent_the_call(monkeypatch): + class BrokenTracer: + @contextmanager + def start_as_current_span(self, *args, **kwargs): + raise RuntimeError("tracing unavailable") + yield # The SDK starts a span when entering its context manager. + + monkeypatch.setattr(trace, "get_tracer", lambda *args: BrokenTracer()) + reached = [] + with prompt_context.mcp_call({"_meta": {"loom": metadata()}}) as span: + reached.append("handler") + assert span is None + assert reached == ["handler"] + + +@pytest.mark.parametrize("handler_fails", [False, True]) +def test_broken_tracer_exit_preserves_the_handler_outcome(monkeypatch, handler_fails): + class BrokenTracer: + @contextmanager + def start_as_current_span(self, *args, **kwargs): + try: + yield None + finally: + raise RuntimeError("tracing unavailable") + + monkeypatch.setattr(trace, "get_tracer", lambda *args: BrokenTracer()) + error = ValueError("handler failed") + + def call(): + with prompt_context.mcp_call({}): + if handler_fails: + raise error + return "result" + + if handler_fails: + with pytest.raises(ValueError) as raised: + call() + assert raised.value is error + else: + assert call() == "result" + + +@pytest.mark.asyncio +async def test_dispatch_error_has_prompt_context_and_does_not_leak_to_the_next_call( + monkeypatch, spans +): + from server.transport import mcp + + async def refuse(*args, **kwargs): + genai.set_identity(org="org", namespace="ns", operation="memory.get") + genai.stamp_identity(trace.get_current_span()) + return {"jsonrpc": "2.0", "id": 1, "error": {"code": -32602, "message": "invalid"}} + + monkeypatch.setattr(mcp, "_call_tool", refuse) + result = await mcp._handle_request( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "loom_memory_get", "_meta": {"loom": metadata()}}, + } + ) + assert "error" in result + root = spans.get_finished_spans()[0] + assert root.attributes["loom.prompt.prompt_id"] == "prompt-a" + assert root.attributes["loom.call.success"] is False + with trace.get_tracer("test").start_as_current_span("after") as after: + genai.stamp_identity(after) + assert "loom.prompt.prompt_id" not in spans.get_finished_spans()[-1].attributes + + +@pytest.mark.asyncio +async def test_http_prompt_binding_resets_after_a_refusal(monkeypatch, spans): + from fastapi import HTTPException, Request, Response + + from server.transport import http + + monkeypatch.setattr(http, "_begin_request", lambda *args: "request") + + def refuse(request): + with trace.get_tracer("test").start_as_current_span("http-refusal") as span: + prompt_context.stamp(span) + raise HTTPException(status_code=401, detail="missing token") + + monkeypatch.setattr(http, "_extract_auth", refuse) + endpoint = http._make_endpoint(None) + request = Request( + { + "type": "http", + "headers": [ + (b"x-loom-conversation-id", b"conversation"), + (b"x-loom-prompt-id", b"prompt-a"), + ], + } + ) + with pytest.raises(HTTPException) as raised: + await endpoint(request, Response()) + assert raised.value.status_code == 401 + assert spans.get_finished_spans()[0].attributes["loom.prompt.prompt_id"] == "prompt-a" + with trace.get_tracer("test").start_as_current_span("after-http") as span: + prompt_context.stamp(span) + assert "loom.prompt.prompt_id" not in spans.get_finished_spans()[-1].attributes + + +def test_invocation_origin_and_benchmark_ids_do_not_require_prompt_ids(): + parsed = prompt_context.from_headers( + { + "x-loom-call-origin": "hook", + "x-loom-hook-event": "SessionStart", + "x-loom-benchmark-run-id": "run-7", + "x-loom-benchmark-case-id": "case-2", + "x-loom-benchmark-variant": "with-memory", + "x-loom-caller-type": "plugin", + } + ) + assert parsed == { + "origin": "hook", + "hook_event": "SessionStart", + "caller_type": "plugin", + "benchmark_run_id": "run-7", + "benchmark_case_id": "case-2", + "benchmark_variant": "with-memory", + } + assert prompt_context.parse({"origin": "guess", "prompt_id": "p"}) == { + "origin": "unknown", + "prompt_id": "p", + } + assert prompt_context.parse({"origin": "tool", "benchmark_case_id": "private\ntext"}) == { + "origin": "tool" + } + + +def test_invocation_metadata_is_structured_and_does_not_copy_usage(spans): + with prompt_context.mcp_call( + { + "_meta": { + "loom": { + "origin": "tool", + "tool_call_id": "stable-1", + "benchmark_run_id": "bench-1", + } + } + } + ) as span: + genai.set_identity(org="org", namespace="ns", operation="memory.get") + genai.stamp_identity(span) + prompt_context.outcome(span, {"result": {"isError": False}}) + root = spans.get_finished_spans()[0] + assert root.attributes["loom.call.origin"] == "tool" + assert root.attributes["langfuse.trace.metadata.loom_benchmark_run_id"] == "bench-1" + assert root.attributes["loom.call.transport"] == "mcp" + assert root.attributes["loom.call.outcome"] == "success" + assert root.attributes["loom.call.duration_ms"] >= 0 + assert not any(k.startswith("gen_ai.usage") for k in root.attributes) + assert not any("bench-1" in t for t in root.attributes["langfuse.trace.tags"]) + + +@pytest.mark.asyncio +async def test_invocation_mcp_protocol_refusal_has_one_attempt_span(spans): + from server.transport import mcp + + result = await mcp._handle_request( + {"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": []} + ) + assert "error" in result + roots = [s for s in spans.get_finished_spans() if s.attributes.get("loom.call.root")] + assert len(roots) == 1 + assert roots[0].attributes["loom.call.outcome"] == "refusal" + assert roots[0].attributes["loom.call.origin"] == "unknown" + + +@pytest.mark.asyncio +async def test_invocation_http_early_refusal_has_operation_and_outcome(monkeypatch, spans): + from types import SimpleNamespace + + from fastapi import HTTPException, Request, Response + + from server.transport import http + + monkeypatch.setattr(http, "_begin_request", lambda *args: "request-safe") + + def refuse(request): + raise HTTPException(status_code=401, detail="must-not-export-this-secret") + + monkeypatch.setattr(http, "_extract_auth", refuse) + endpoint = http._make_endpoint(SimpleNamespace(name="memory.get")) + request = Request( + { + "type": "http", + "headers": [(b"x-loom-call-origin", b"hook"), (b"x-loom-hook-event", b"SessionStart")], + } + ) + with pytest.raises(HTTPException): + await endpoint(request, Response()) + roots = [s for s in spans.get_finished_spans() if s.attributes.get("loom.call.root")] + assert len(roots) == 1 + attrs = roots[0].attributes + assert attrs["loom.operation"] == "memory.get" + assert attrs["loom.call.outcome"] == "refusal" + assert attrs["loom.call.origin"] == "hook" + assert "must-not-export-this-secret" not in str(attrs) + + +@pytest.mark.parametrize("origin", [None, [], {}, 3, "automatic-guess"]) +def test_invocation_invalid_origin_remains_unknown(origin): + assert prompt_context.parse({"origin": origin}) == {"origin": "unknown"} + + +@pytest.mark.parametrize( + "response, expected", + [ + ( + { + "result": { + "isError": True, + "structuredContent": {"category": "auth", "code": "AUTH_INSUFFICIENT"}, + } + }, + "refusal", + ), + ( + {"error": {"code": -32603, "data": {"code": "BACKEND_TIMEOUT", "status": 504}}}, + "timeout", + ), + ({"error": {"code": -32603, "message": "private failure"}}, "error"), + ], +) +def test_invocation_outcomes_export_no_error_text(spans, response, expected): + with prompt_context.mcp_call({}) as span: + prompt_context.outcome(span, response) + root = spans.get_finished_spans()[0] + assert root.attributes["loom.call.outcome"] == expected + assert "private failure" not in str(root.attributes) + assert not root.events + + +def test_invocation_retry_identity_is_distinct_from_received_attempts(spans): + for _unused in range(2): + with prompt_context.mcp_call( + {"_meta": {"loom": {"origin": "tool", "tool_call_id": "stable"}}} + ) as span: + prompt_context.outcome(span, {"result": {}}) + first, second = spans.get_finished_spans() + assert first.attributes["loom.call.attempt_id"] != second.attributes["loom.call.attempt_id"] + assert ( + first.attributes["loom.prompt.tool_call_id"] + == second.attributes["loom.prompt.tool_call_id"] + == "stable" + ) + + +def test_invocation_exception_is_safe_and_identity_restores(spans): + genai.set_identity(org="outer", namespace="outside") + with pytest.raises(TimeoutError): + with prompt_context.mcp_call({}): + assert not genai._identity.get() + raise TimeoutError("private SQL or credential") + root = spans.get_finished_spans()[0] + assert root.attributes["loom.call.outcome"] == "timeout" + assert root.attributes["loom.call.origin"] == "unknown" + assert "loom.org" not in root.attributes + assert not root.events + assert genai._identity.get()["org"] == "outer" + + +def test_invocation_private_export_boundary_still_drops_call_metadata(monkeypatch): + from server.core.diagnostics import private_work + from server.observability import otel_langfuse + + exporter = InMemorySpanExporter() + provider = TracerProvider() + monkeypatch.setattr(otel_langfuse.settings, "langfuse_sample_rate", 1.0) + provider.add_span_processor( + otel_langfuse._langfuse_processor_class()( + SimpleSpanProcessor(exporter), otel_langfuse._LangfuseSpanFilter() + ) + ) + monkeypatch.setattr(trace, "get_tracer", provider.get_tracer) + with prompt_context.mcp_call({"_meta": {"loom": {"origin": "tool"}}}) as span: + with private_work(): + with trace.get_tracer("test").start_as_current_span("private.child"): + pass + span.end() + assert exporter.get_finished_spans() == () + provider.shutdown() + + +def test_invocation_catalog_and_unknown_tool_each_have_one_received_attempt(monkeypatch, spans): + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from server.settings import settings + from server.transport import mcp + + monkeypatch.setattr(settings, "identity_provider", "single-user") + app = FastAPI() + mcp.attach(app) + with TestClient(app) as client: + catalog = client.post( + "/mcp", + json={ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "loom_op", + "arguments": {"op": "help"}, + "_meta": {"loom": {"origin": "tool", "benchmark_case_id": "catalog"}}, + }, + }, + ) + unknown = client.post( + "/mcp", + json={ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "not-a-real-tool", "arguments": {}}, + }, + ) + assert catalog.status_code == unknown.status_code == 200 + assert not catalog.json()["result"]["isError"] + assert unknown.json()["result"]["isError"] + roots = [s for s in spans.get_finished_spans() if s.attributes.get("loom.call.root")] + assert len(roots) == 2 + assert roots[0].attributes["loom.operation"] == "loom_op.help" + assert roots[0].attributes["loom.benchmark.case_id"] == "catalog" + assert roots[0].attributes["loom.call.outcome"] == "success" + assert roots[1].attributes["loom.call.outcome"] == "refusal" + + +def test_invocation_worker_origin_is_scoped_without_counting_a_received_call(spans): + from server.observability import otel + + with otel.worker_pass_span("example") as root: + with trace.get_tracer("test").start_as_current_span("worker.child") as child: + genai.stamp_identity(child) + child, root = spans.get_finished_spans() + assert child.attributes["loom.call.origin"] == "worker" + assert child.attributes["loom.call.caller_type"] == "worker" + assert not root.attributes.get("loom.call.root") + with trace.get_tracer("test").start_as_current_span("outside") as span: + prompt_context.stamp(span) + assert "loom.call.origin" not in spans.get_finished_spans()[-1].attributes + + +def test_invocation_http_default_response_status_is_success(spans): + with prompt_context.http_call({}, "memory.get") as span: + prompt_context.http_outcome(span, None, "request-example") + root = spans.get_finished_spans()[0] + assert root.attributes["loom.call.outcome"] == "success" + assert root.attributes["loom.request_id"] == "request-example" + + +@pytest.mark.parametrize("target", ["memory.set", "memory_set", " memory.set "]) +def test_review_operation_refusal_retains_normalized_target(monkeypatch, spans, target): + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from server.settings import settings + from server.transport import mcp + + monkeypatch.setattr(settings, "identity_provider", "single-user") + app = FastAPI() + mcp.attach(app) + with TestClient(app) as client: + response = client.post( + "/mcp?toolset=readonly", + json={ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "loom_op", "arguments": {"op": target}}, + }, + ) + assert response.json()["result"]["isError"] is True + roots = [s for s in spans.get_finished_spans() if s.attributes.get("loom.call.root")] + assert len(roots) == 1 + assert roots[0].attributes["loom.operation"] == "memory.set" + assert roots[0].attributes["loom.call.outcome"] == "refusal" + + +@pytest.mark.asyncio +async def test_review_operation_activation_retains_operation(monkeypatch, spans): + from server.transport import mcp, mcp_call + + async def primer(*args): + return "primer" + + monkeypatch.setattr(mcp_call, "_render_primer_and_emit", primer) + response = await mcp._handle_request( + {"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "loom_activate"}} + ) + assert response["result"]["isError"] is False + roots = [s for s in spans.get_finished_spans() if s.attributes.get("loom.call.root")] + assert len(roots) == 1 + assert roots[0].attributes["loom.operation"] == "loom_activate" + assert roots[0].attributes["loom.call.outcome"] == "success" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("version", ["2026-07-28", "2025-11-25"]) +async def test_review_protocol_version_refusal(monkeypatch, spans, version): + from server.transport import mcp + from server.transport.mcp_config import META_PROTOCOL_VERSION, UNSUPPORTED_PROTOCOL_VERSION + + response = await mcp._handle_request( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "not-a-real-tool", + "_meta": {META_PROTOCOL_VERSION: version}, + }, + } + ) + if version == "2026-07-28": + assert response["error"]["code"] == UNSUPPORTED_PROTOCOL_VERSION + else: + assert response["result"]["isError"] is True + roots = [s for s in spans.get_finished_spans() if s.attributes.get("loom.call.root")] + assert len(roots) == 1 + assert roots[0].attributes["loom.call.outcome"] == "refusal" + + +@pytest.mark.parametrize( + "arguments, operation, failed", + [ + ({"op": "help"}, "loom_op.help", False), + ({"op": " Catalog "}, "loom_op.help", False), + (None, "loom_op.help", False), + ({"op": "definitely_missing"}, "loom_op", True), + ({"op": 23}, "loom_op", True), + ("invalid", "loom_op", True), + ], +) +def test_review_operation_label_distinguishes_help_from_invalid_targets( + monkeypatch, spans, arguments, operation, failed +): + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from server.settings import settings + from server.transport import mcp + + monkeypatch.setattr(settings, "identity_provider", "single-user") + app = FastAPI() + mcp.attach(app) + with TestClient(app) as client: + response = client.post( + "/mcp", + json={ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "loom_op", "arguments": arguments}, + }, + ) + body = response.json() + assert bool(body.get("error") or body.get("result", {}).get("isError")) is failed + roots = [s for s in spans.get_finished_spans() if s.attributes.get("loom.call.root")] + assert len(roots) == 1 + assert roots[0].attributes["loom.operation"] == operation + assert roots[0].attributes["loom.call.outcome"] == ("refusal" if failed else "success") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "kind, expected", [("embed", "timeout"), ("http", "timeout"), ("busy", "error")] +) +async def test_review_http_timeout_conversion_preserves_classification( + monkeypatch, spans, kind, expected +): + from contextlib import nullcontext + from types import SimpleNamespace + + from fastapi import HTTPException + + from server.core.errors import EmbedTimeoutError + from server.transport import http + + code = "UPSTREAM_UNAVAILABLE" if kind == "busy" else "EMBED_TIMEOUT" + + async def handler(payload): + if kind == "embed": + raise EmbedTimeoutError("private provider failure") + raise HTTPException(503, detail={"code": code, "message": "private provider failure"}) + + monkeypatch.setattr(http.gates, "tenant_slot", lambda *args: nullcontext()) + monkeypatch.setattr(http.llm_free, "serving", lambda *args: nullcontext()) + monkeypatch.setattr(http, "_emit_event", lambda *args, **kwargs: None) + monkeypatch.setattr(http, "_record_failure", lambda *args, **kwargs: None) + op = SimpleNamespace(name="memory.search", handler=handler, idempotent=True) + call = SimpleNamespace( + request_id="request", raw_body={}, identity={"namespace": "ns"}, auth=None, target="ns" + ) + with pytest.raises(HTTPException) as raised: + with prompt_context.http_call({}, op.name): + await http._run_handler(op, None, call) + assert raised.value.status_code == 503 + assert raised.value.detail["code"] == code + root = spans.get_finished_spans()[0] + assert root.attributes["loom.call.outcome"] == expected + assert "private provider failure" not in str(root.attributes) + assert not root.events + + +def _assert_authenticated_replay_roots(roots): + assert len(roots) == 2 + for root in roots: + assert root.attributes["loom.org"] == "o" + assert root.attributes["loom.namespace"] == "n" + assert root.attributes["langfuse.user.id"] == "alice" + assert {"org:o", "namespace:n"} <= set(root.attributes["langfuse.trace.tags"]) + assert root.attributes["loom.request_id"] + assert root.attributes["loom.prompt.tool_call_id"] == "logical-replay" + assert root.attributes["loom.call.outcome"] == "success" + assert ( + roots[0].attributes["loom.call.attempt_id"] != roots[1].attributes["loom.call.attempt_id"] + ) + assert roots[0].attributes["loom.request_id"] != roots[1].attributes["loom.request_id"] + + +@pytest.mark.parametrize("transport", ["http", "mcp"]) +def test_review_authenticated_early_refusal_uses_token_identity( + monkeypatch, client, spans, transport +): + from server.auth import bearer, jwt + from server.transport import mcp + + http_client, rows, ran = client + monkeypatch.setattr(jwt, "authenticate_token", bearer.authenticate_token) + headers = {"Authorization": "Bearer alice", "X-Loom-Tool-Call-Id": "logical-refusal"} + if transport == "http": + result = http_client.post( + "/v1/memory.unlock", json={"org": "forged", "namespace": "forged"}, headers=headers + ) + assert result.status_code == 422 + else: + mcp.attach(http_client.app) + result = http_client.post( + "/mcp", + headers=headers, + json={ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "does_not_exist", + "arguments": {"org": "forged", "namespace": "forged"}, + }, + }, + ) + assert result.json().get("error") or result.json().get("result", {}).get("isError") + assert not rows and not ran + roots = [s for s in spans.get_finished_spans() if s.attributes.get("loom.call.root")] + assert len(roots) == 1 + attributes = roots[0].attributes + assert attributes["loom.org"] == "o" + assert attributes["langfuse.user.id"] == "alice" + assert "org:o" in attributes["langfuse.trace.tags"] + assert attributes["loom.namespace"] == "-" + assert "forged" not in str(attributes) + assert attributes["loom.prompt.tool_call_id"] == "logical-refusal" + assert attributes["loom.call.outcome"] == "refusal" + + +def test_review_authenticated_http_replay_retains_identity(client, spans): + from tests.unit.test_http_idempotency_after_auth import BODY + + http_client, rows, ran = client + headers = { + "Authorization": "Bearer alice", + "Idempotency-Key": "identity-replay", + "X-Loom-Call-Origin": "tool", + "X-Loom-Tool-Call-Id": "logical-replay", + } + first = http_client.post("/v1/memory.unlock", json=BODY, headers=headers) + replay = http_client.post("/v1/memory.unlock", json=BODY, headers=headers) + assert first.status_code == replay.status_code == 200 + assert replay.headers["X-Idempotent-Replay"] == "true" + assert ran == ["k"] and len(rows) == 1 + _assert_authenticated_replay_roots( + [s for s in spans.get_finished_spans() if s.attributes.get("loom.call.root")] + ) + + +@pytest.mark.asyncio +async def test_review_authenticated_mcp_replay_retains_identity(monkeypatch, gated, spans): + from dataclasses import replace + + from pydantic import BaseModel + + from server.transport import mcp, mcp_call + from tests.unit.test_mcp_idempotency_replay_conflict import TOOL, _principal + + class ScopedInput(BaseModel): + text: str + org: str + namespace: str + agent: str + + op = replace(mcp_call.TOOLS_BY_NAME[TOOL], input_model=ScopedInput) + monkeypatch.setitem(mcp_call.TOOLS_BY_NAME, TOOL, op) + results = [] + for req_id in (1, 2): + results.append( + await mcp._handle_request( + { + "jsonrpc": "2.0", + "id": req_id, + "method": "tools/call", + "params": { + "name": TOOL, + "arguments": { + "text": "safe fixture", + "org": "o", + "namespace": "n", + "agent": "a", + "_idempotency_key": "identity-replay", + }, + "_meta": {"loom": {"origin": "tool", "tool_call_id": "logical-replay"}}, + }, + }, + auth=_principal("alice"), + ) + ) + assert all(not r["result"]["isError"] for r in results) + assert results[1]["result"]["_meta"] == {"idempotentReplay": True} + store, ran = gated + assert ran == ["safe fixture"] and len(store.rows) == 1 + _assert_authenticated_replay_roots( + [s for s in spans.get_finished_spans() if s.attributes.get("loom.call.root")] + )