Skip to content

Commit 8d879e9

Browse files
TheGreatAxiosTheGreatAxios
committed
perf(session): cut per-turn store walks, re-parses and stringifies
A 200-turn simulation dropped 139 to 124 fs ops per turn (access probes 2697 to 3) and wall median ~13%. commit() now surveys the worktree once instead of stat-probing every candidate segment; a per-store turns cache with per-segment stat fingerprints plus explicit mutation-site invalidation loads each segment at most once, so load/ readLiveTurns/prompt-write parse once per content instead of three times. contentPrefixLength/promptEqualsLiveTurns memoize turn JSON instead of stringifying full history; the well-formedness prefix scan is a single forward pass instead of O(E^2); readAt reads a commit-time segment manifest and parallel blob reads, degrading to the ref-walk when the manifest is absent or torn; attachment blob reads dedupe; aged-image markers prefilter before regex; anthropic cache prompt aggregates one forward scan; compaction identities are cached per tool_call block. Invalidation is strictly wider than the reads it replaced: mutation sites and failed-commit restores drop the caches; corrupt manifests degrade to the walk.
1 parent 1004268 commit 8d879e9

8 files changed

Lines changed: 1012 additions & 224 deletions

‎scripts/context-store-perf.ts‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
#!/usr/bin/env bun
2+
/**
3+
* CL-10666 before/after measurement: session-store per-turn I/O.
4+
*
5+
* One machine, one scenario: a fresh session dir plus N simulated reactor
6+
* cycles (load, writePrompt, writeTurns, commit) through the optimized
7+
* context store. Every node fs.promises call the store stack makes is
8+
* counted (split by whether the path is under .git), along with wall time,
9+
* so the per-turn walks, re-parses, and re-stringified comparisons become an
10+
* honest delta between two builds on the same host.
11+
*
12+
* Usage:
13+
* bun scripts/context-store-perf.ts [turnCount]
14+
* bun scripts/context-store-perf.ts [turnCount] --json
15+
*/
16+
17+
import fs from "node:fs";
18+
import path from "node:path";
19+
import type { ConversationTurn } from "@intx/types/runtime";
20+
import { createOptimizedContextStore } from "../src/session/optimized-context-store.js";
21+
22+
const COUNTED_OPS = [
23+
"access",
24+
"copyFile",
25+
"lstat",
26+
"mkdir",
27+
"open",
28+
"opendir",
29+
"readFile",
30+
"readdir",
31+
"readlink",
32+
"rename",
33+
"rmdir",
34+
"stat",
35+
"truncate",
36+
"unlink",
37+
"writeFile",
38+
] as const;
39+
40+
const counts: Record<string, number> = {};
41+
const gitPaths: Record<string, number> = {};
42+
43+
function bump(table: Record<string, number>, op: string): void {
44+
table[op] = (table[op] ?? 0) + 1;
45+
}
46+
47+
function installCounters(): void {
48+
const target = fs.promises as unknown as Record<
49+
string,
50+
(...args: unknown[]) => unknown
51+
>;
52+
for (const op of COUNTED_OPS) {
53+
const original = target[op];
54+
if (typeof original !== "function") continue;
55+
target[op] = function counted(this: unknown, ...args: unknown[]) {
56+
const first = args[0];
57+
if (typeof first === "string" && first.includes(`/.git/`)) {
58+
bump(gitPaths, op);
59+
}
60+
bump(counts, op);
61+
return original.apply(this, args);
62+
};
63+
}
64+
}
65+
66+
function turn(text: string): ConversationTurn {
67+
return { role: "user", content: [{ type: "text", text }], timestamp: 1 };
68+
}
69+
70+
function assistantTurn(text: string): ConversationTurn {
71+
return {
72+
role: "assistant",
73+
content: [{ type: "text", text }],
74+
timestamp: 2,
75+
};
76+
}
77+
78+
function turnBody(index: number): string {
79+
// ~1.5 KB of filler plus a marker line, close to a small unit of context.
80+
return (
81+
`turn ${index}: ` +
82+
"boundary-check padding. ".repeat(60) +
83+
`end-of-turn ${index}`
84+
);
85+
}
86+
87+
async function runScenario(dir: string, turnCount: number): Promise<number> {
88+
const store = await createOptimizedContextStore(dir);
89+
const history: ConversationTurn[] = [];
90+
for (let index = 0; index < turnCount; index++) {
91+
history.push(turn(turnBody(index)));
92+
if (index % 4 === 1) history.push(assistantTurn(`reply to ${index}`));
93+
await store.load();
94+
await store.writePrompt(history);
95+
await store.writeTurns(history);
96+
await store.commit({ message: `checkpoint ${index}` });
97+
}
98+
const fresh = await createOptimizedContextStore(dir);
99+
return (await fresh.load()).turns.length;
100+
}
101+
102+
function sortedEntries(table: Record<string, number>): [string, number][] {
103+
return Object.entries(table).sort((a, b) => b[1] - a[1]);
104+
}
105+
106+
async function main(argv: string[]): Promise<number> {
107+
const arg = argv[2];
108+
const turnCount =
109+
arg !== undefined && !arg.startsWith("-") ? Number(arg) : 40;
110+
if (!Number.isFinite(turnCount) || turnCount < 1) {
111+
console.error(
112+
"usage: bun scripts/context-store-perf.ts [turnCount] [--json]",
113+
);
114+
return 1;
115+
}
116+
installCounters();
117+
118+
const dir = await fs.promises.mkdtemp(
119+
path.join(process.cwd(), "context-store-perf-"),
120+
);
121+
const started = performance.now();
122+
const loadedTurns = await runScenario(dir, turnCount);
123+
const wallMs = Math.round(performance.now() - started);
124+
125+
const totalOps = Object.values(counts).reduce((sum, n) => sum + n, 0);
126+
const gitOps = Object.values(gitPaths).reduce((sum, n) => sum + n, 0);
127+
128+
const asJson = argv.includes("--json");
129+
if (asJson) {
130+
console.log(
131+
JSON.stringify(
132+
{
133+
turnCount,
134+
wallMs,
135+
loadedTurns,
136+
totalFsOps: totalOps,
137+
gitPathFsOps: gitOps,
138+
fsOps: counts,
139+
gitPathFsOpsByName: gitPaths,
140+
perTurn: {
141+
wallMs: +(wallMs / turnCount).toFixed(2),
142+
fsOps: +(totalOps / turnCount).toFixed(2),
143+
},
144+
},
145+
null,
146+
2,
147+
),
148+
);
149+
} else {
150+
console.log(
151+
`scenario: ${turnCount} turns, one store instance (fresh dir); verified ${loadedTurns} turns at the end`,
152+
);
153+
console.log(
154+
`wall: ${wallMs} ms (${(wallMs / turnCount).toFixed(2)} ms/turn)`,
155+
);
156+
console.log(
157+
`fs ops: ${totalOps} total (${(totalOps / turnCount).toFixed(2)}/turn), of which ${gitOps} under .git`,
158+
);
159+
for (const [op, n] of sortedEntries(counts)) {
160+
console.log(` ${op}: ${n}`);
161+
}
162+
for (const [op, n] of sortedEntries(gitPaths)) {
163+
console.log(` (.git) ${op}: ${n}`);
164+
}
165+
}
166+
await fs.promises.rm(dir, { recursive: true, force: true });
167+
return 0;
168+
}
169+
170+
process.exit(await main(process.argv));

‎src/session/anthropic-cache-prompt.ts‎

Lines changed: 27 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -42,42 +42,37 @@ function passthrough(
4242
};
4343
}
4444

45-
function newestAssistantTextIndex(turns: readonly ConversationTurn[]): number {
46-
for (let i = turns.length - 1; i >= 0; i--) {
47-
const turn = turns[i];
48-
if (turn === undefined || turn.role !== "assistant") continue;
49-
if (
50-
turn.content.some(
51-
(block) => block.type === "text" && block.text.length > 0,
52-
)
53-
) {
54-
return i;
55-
}
56-
}
57-
return -1;
58-
}
59-
60-
// Tool results before the newest assistant text are already consumed.
61-
// When the model has not written that text yet, the newest result turn is
62-
// the unconsumed suffix and stays whole.
63-
function stubBoundary(turns: readonly ConversationTurn[]): number {
64-
const assistantText = newestAssistantTextIndex(turns);
65-
if (assistantText >= 0) return assistantText;
66-
for (let i = turns.length - 1; i >= 0; i--) {
67-
const turn = turns[i];
68-
if (turn?.content.some((block) => block.type === "tool_result")) return i;
69-
}
70-
return turns.length;
71-
}
72-
73-
function toolNames(turns: readonly ConversationTurn[]): Map<string, string> {
45+
// One forward pass collects everything shrinkPrompt needs: every tool_call
46+
// id→name (a result's call turn can sit anywhere, so names span the whole
47+
// transcript), the newest assistant text index, and the last tool_result
48+
// index. The boundary is that text index when one exists, else the last
49+
// result turn (the unconsumed suffix stays whole when the model has not
50+
// written text yet), else the whole history.
51+
function stubSetup(turns: readonly ConversationTurn[]): {
52+
names: Map<string, string>;
53+
boundary: number;
54+
} {
7455
const names = new Map<string, string>();
75-
for (const turn of turns) {
56+
let newestAssistantText = -1;
57+
let lastToolResult = -1;
58+
for (let i = 0; i < turns.length; i++) {
59+
const turn = turns[i];
60+
if (turn === undefined) continue;
61+
let hasText = false;
7662
for (const block of turn.content) {
7763
if (block.type === "tool_call") names.set(block.id, block.name);
64+
else if (block.type === "tool_result") lastToolResult = i;
65+
else if (block.type === "text" && block.text.length > 0) hasText = true;
7866
}
67+
if (hasText && turn.role === "assistant") newestAssistantText = i;
7968
}
80-
return names;
69+
const boundary =
70+
newestAssistantText >= 0
71+
? newestAssistantText
72+
: lastToolResult >= 0
73+
? lastToolResult
74+
: turns.length;
75+
return { names, boundary };
8176
}
8277

8378
function stubText(name: string | undefined): string {
@@ -124,8 +119,7 @@ function shrinkPrompt(turns: ConversationTurn[]): {
124119
output: ConversationTurn[];
125120
stubbed: number;
126121
} {
127-
const boundary = stubBoundary(turns);
128-
const names = toolNames(turns);
122+
const { names, boundary } = stubSetup(turns);
129123
let stubbed = 0;
130124
let changed = false;
131125
const output = turns.map((turn, index) => {

‎src/session/attachment-store.test.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,4 +146,61 @@ describe("ageImageBlocks / rehydrateAttachmentImages", () => {
146146
expect(aged.blobs).toHaveLength(1);
147147
expect(recorded).toEqual([]);
148148
});
149+
150+
test("identical markers across turns read the blob once (CL-10666)", async () => {
151+
const text = formatAgedImageMarker({
152+
uri: attachmentUri("img-shared"),
153+
mimeType: "image/png",
154+
});
155+
const markerTurn: ConversationTurn = {
156+
role: "user",
157+
content: [{ type: "text", text }],
158+
timestamp: 1,
159+
};
160+
const blob = new TextEncoder().encode(PNG_B64);
161+
const reads: string[] = [];
162+
const readBlob = async (key: string) => {
163+
reads.push(key);
164+
return blob;
165+
};
166+
167+
const out = await rehydrateAttachmentImages(
168+
[markerTurn, markerTurn, markerTurn],
169+
readBlob,
170+
);
171+
expect(reads).toEqual(["img-shared"]);
172+
for (const turn of out) {
173+
expect(turn.content).toEqual([
174+
{
175+
type: "image",
176+
source: { kind: "base64", mimeType: "image/png", data: PNG_B64 },
177+
},
178+
]);
179+
}
180+
expect(reads).toHaveLength(1);
181+
});
182+
183+
test("a missing shared blob keeps every copy of its marker from one read", async () => {
184+
const text = formatAgedImageMarker({
185+
uri: attachmentUri("img-gone"),
186+
mimeType: "image/png",
187+
});
188+
const markerTurn: ConversationTurn = {
189+
role: "user",
190+
content: [{ type: "text", text }],
191+
timestamp: 1,
192+
};
193+
const reads: string[] = [];
194+
const out = await rehydrateAttachmentImages(
195+
[markerTurn, markerTurn],
196+
async (key) => {
197+
reads.push(key);
198+
throw new Error(`Blob not found: ${key}`);
199+
},
200+
);
201+
expect(reads).toHaveLength(1);
202+
for (const turn of out) {
203+
expect(turn.content).toEqual([{ type: "text", text }]);
204+
}
205+
});
149206
});

‎src/session/attachment-store.ts‎

Lines changed: 30 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -81,11 +81,27 @@ export async function ageImageBlocks(
8181
return { turn: { ...turn, content }, blobs };
8282
}
8383

84-
/** Restore aged markers to image blocks via a blob reader; missing blobs leave the marker text. */
84+
/** Restore aged markers to image blocks via a blob reader; missing blobs leave
85+
* the marker text. The same attachment id only reads its blob once per
86+
* restore: compaction copies (excerpted spines, folded handoffs) repeat the
87+
* identical marker, and each copy then deserializes from the one read. */
8588
export async function rehydrateAttachmentImages(
8689
turns: readonly ConversationTurn[],
8790
readBlob: (key: string) => Promise<Uint8Array>,
8891
): Promise<ConversationTurn[]> {
92+
const blobReads = new Map<string, Promise<Uint8Array | null>>();
93+
const blobFor = (key: string): Promise<Uint8Array | null> => {
94+
const shared = blobReads.get(key);
95+
if (shared !== undefined) return shared;
96+
// One failure is memoized for the id too: missing blobs keep their
97+
// markers, and re-serving that outcome avoids a retry storm per copy.
98+
const sharedRead = readBlob(key).then(
99+
(bytes) => bytes,
100+
() => null,
101+
);
102+
blobReads.set(key, sharedRead);
103+
return sharedRead;
104+
};
89105
const out: ConversationTurn[] = [];
90106
for (const turn of turns) {
91107
let changed = false;
@@ -100,21 +116,21 @@ export async function rehydrateAttachmentImages(
100116
content.push(block);
101117
continue;
102118
}
103-
try {
104-
const bytes = await readBlob(marker.id);
105-
const data = new TextDecoder().decode(bytes);
106-
content.push({
107-
type: "image",
108-
source: {
109-
kind: "base64",
110-
mimeType: marker.mimeType,
111-
data,
112-
},
113-
});
114-
changed = true;
115-
} catch {
119+
const bytes = await blobFor(marker.id);
120+
if (bytes === null) {
116121
content.push(block);
122+
continue;
117123
}
124+
const data = new TextDecoder().decode(bytes);
125+
content.push({
126+
type: "image",
127+
source: {
128+
kind: "base64",
129+
mimeType: marker.mimeType,
130+
data,
131+
},
132+
});
133+
changed = true;
118134
}
119135
out.push(changed ? { ...turn, content } : turn);
120136
}

‎src/session/attachment-uri.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,12 @@ export interface AgedImageMarker {
3636
export function parseAgedImageMarker(
3737
text: string,
3838
): AgedImageMarker | undefined {
39+
// Prefilter: markers are the only shape that can start this way, and text
40+
// blocks here include whole tool outputs, so skip the regex on anything
41+
// that cannot possibly match.
42+
if (!text.startsWith("[image attachment aged: attachment:///")) {
43+
return undefined;
44+
}
3945
// [image attachment aged: attachment:///ID mimeType=TYPE — ...]
4046
const match = text.match(
4147
/^\[image attachment aged: (attachment:\/\/\/[^\s]+) mimeType=([^\s]+) —/,

0 commit comments

Comments
 (0)