Merge pull request #1509 from corbitsdev/cl-10810-add-a-perf-regression-gate-and-frozen-baseline-for-the

ci(session): gate session-store fs.promises counts against a baseline

Author: TheGreatAxios

.github/workflows/ci.yml

@@ -197,6 +197,42 @@ jobs:
       - name: Test
         run: bun run check:projects-dir-guard
 
+  # scripts/context-store-perf.ts and scripts/turn-persistence-census.ts
+  # both patch fs.promises only, so synchronous fs is invisible. The job
+  # fails when a counted op moves past the tolerance in
+  # scripts/check-session-store-perf-baseline.ts. It does not see sync fs,
+  # callback fs, or Bun.file. .git mkdir and writeFile have slack for
+  # object-id fanout; every other counted op is exact. Wall time is not gated.
+  session-store-perf:
+    name: session-store perf baseline
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: "1.3.14"
+
+      - name: Cache dependencies
+        uses: actions/cache@v4
+        with:
+          path: node_modules
+          key: bun-${{ hashFiles('bun.lock') }}
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Compare to the frozen baseline
+        run: |
+          bun scripts/context-store-perf.ts 200 --json > "$RUNNER_TEMP/context-store-perf.json"
+          bun scripts/turn-persistence-census.ts 200 --flow=coalesced --json > "$RUNNER_TEMP/turn-persistence-census.json"
+          bun scripts/check-session-store-perf-baseline.ts \
+            scripts/results/session-store-perf-baseline.json \
+            "$RUNNER_TEMP/context-store-perf.json" \
+            "$RUNNER_TEMP/turn-persistence-census.json"
+
   # protect-main still requires the pre-restructure check names. These jobs
   # exist only to publish those contexts after the real work succeeds.
   prettier:
@@ -218,7 +254,7 @@ jobs:
       - run: "true"
 
   build-and-test:
-    needs: [build, test]
+    needs: [build, test, session-store-perf]
     runs-on: ubuntu-latest
     steps:
       - run: "true"

scripts/check-session-store-perf-baseline.test.ts

@@ -0,0 +1,111 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, test } from "bun:test";
+import {
+  COUNTED_OP_TOLERANCE,
+  GIT_FANOUT_TOLERANCE,
+  type BaselineFile,
+  type CensusMeasurement,
+  type PerfMeasurement,
+  gateSessionStoreBaseline,
+  gateSessionStoreBaselineJson,
+} from "./check-session-store-perf-baseline.js";
+
+const repoRoot = join(import.meta.dir, "..");
+const baselinePath = join(
+  repoRoot,
+  "scripts",
+  "results",
+  "session-store-perf-baseline.json",
+);
+const baselineRaw = readFileSync(baselinePath, "utf8");
+const ci = readFileSync(
+  join(repoRoot, ".github", "workflows", "ci.yml"),
+  "utf8",
+);
+
+function parsedBaseline(): {
+  baseline: BaselineFile;
+  perf: PerfMeasurement;
+  census: CensusMeasurement;
+} {
+  const failures = gateSessionStoreBaselineJson(
+    baselineRaw,
+    JSON.stringify(JSON.parse(baselineRaw).contextStorePerf),
+    JSON.stringify(JSON.parse(baselineRaw).turnPersistenceCensus),
+  );
+  expect(failures).toEqual([]);
+  const baseline = JSON.parse(baselineRaw) as BaselineFile;
+  return {
+    baseline,
+    perf: baseline.contextStorePerf,
+    census: baseline.turnPersistenceCensus,
+  };
+}
+
+describe("session-store perf baseline", () => {
+  test("checked-in snapshot matches the gate, ignoring wall time", () => {
+    expect(COUNTED_OP_TOLERANCE).toBe(0);
+    expect(GIT_FANOUT_TOLERANCE).toBe(64);
+    const { baseline, perf, census } = parsedBaseline();
+    const noisy = {
+      ...perf,
+      wallMs: 1,
+      perTurn: { wallMs: 99, fsOps: 0 },
+    };
+    expect(gateSessionStoreBaseline(baseline, noisy, census)).toEqual([]);
+  });
+
+  test("a per-turn access outside .git fails; git fanout within slack passes", () => {
+    const { baseline, perf, census } = parsedBaseline();
+    const extraAccess: PerfMeasurement = {
+      ...perf,
+      fsOps: { ...perf.fsOps, access: (perf.fsOps.access ?? 0) + 1 },
+    };
+    expect(gateSessionStoreBaseline(baseline, extraAccess, census)).not.toEqual(
+      [],
+    );
+
+    const fanout = GIT_FANOUT_TOLERANCE;
+    const within: PerfMeasurement = {
+      ...perf,
+      fsOps: {
+        ...perf.fsOps,
+        mkdir: (perf.fsOps.mkdir ?? 0) + fanout,
+        writeFile: (perf.fsOps.writeFile ?? 0) + fanout,
+      },
+      gitPathFsOpsByName: {
+        ...perf.gitPathFsOpsByName,
+        mkdir: (perf.gitPathFsOpsByName.mkdir ?? 0) + fanout,
+        writeFile: (perf.gitPathFsOpsByName.writeFile ?? 0) + fanout,
+      },
+    };
+    expect(gateSessionStoreBaseline(baseline, within, census)).toEqual([]);
+
+    const past: PerfMeasurement = {
+      ...perf,
+      fsOps: {
+        ...perf.fsOps,
+        mkdir: (perf.fsOps.mkdir ?? 0) + fanout + 1,
+      },
+      gitPathFsOpsByName: {
+        ...perf.gitPathFsOpsByName,
+        mkdir: (perf.gitPathFsOpsByName.mkdir ?? 0) + fanout + 1,
+      },
+    };
+    const failures = gateSessionStoreBaseline(baseline, past, census);
+    expect(failures.some((line) => line.includes(".git mkdir"))).toBe(true);
+  });
+
+  test("CI runs both scripts and states the fs.promises-only blind spot", () => {
+    expect(ci).toContain("bun scripts/context-store-perf.ts 200 --json");
+    expect(ci).toContain(
+      "bun scripts/turn-persistence-census.ts 200 --flow=coalesced --json",
+    );
+    expect(ci).toContain(
+      "both patch fs.promises only, so synchronous fs is invisible",
+    );
+    expect(ci).not.toContain("synchronous fs is counted");
+    expect(ci).not.toContain("every filesystem");
+  });
+});