From ec26e55ae105dbe76cb05c61705df18e81362d63 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sun, 30 Aug 2026 22:52:43 -0700 Subject: [PATCH 1/4] Add tests for killing E2E_REQUIRED as a caller flag --- .../test/tool-package-freshness.test.ts | 33 +++++++ scripts/ci-jobs.test.ts | 4 +- scripts/e2e/db-gate.test.ts | 93 ++++++++++++++++--- test/isolation/setup.test.ts | 36 +++++-- 4 files changed, 140 insertions(+), 26 deletions(-) diff --git a/scripts/checks/test/tool-package-freshness.test.ts b/scripts/checks/test/tool-package-freshness.test.ts index 95af0dbea3..a871897043 100644 --- a/scripts/checks/test/tool-package-freshness.test.ts +++ b/scripts/checks/test/tool-package-freshness.test.ts @@ -1,8 +1,12 @@ import { describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { auditFreshness, packagesWithChangedSource, + pullRequestBaseSha, readToolPackageNames, } from "../tool-package-freshness"; @@ -97,3 +101,32 @@ describe("scope", () => { expect(names).toEqual(["github-tools", "memory-tools"]); }); }); + +describe("pullRequestBaseSha", () => { + test("reads pull_request.base.sha from GITHUB_EVENT_PATH", () => { + const dir = mkdtempSync(path.join(tmpdir(), "base-ref-")); + const eventPath = path.join(dir, "event.json"); + writeFileSync( + eventPath, + JSON.stringify({ pull_request: { base: { sha: "abc123def" } } }), + ); + expect(pullRequestBaseSha({ GITHUB_EVENT_PATH: eventPath })).toBe( + "abc123def", + ); + rmSync(dir, { recursive: true, force: true }); + }); + + test("returns undefined when GITHUB_EVENT_PATH is unset", () => { + expect(pullRequestBaseSha({})).toBeUndefined(); + }); + + test("returns undefined when the event is not a pull_request", () => { + const dir = mkdtempSync(path.join(tmpdir(), "base-ref-")); + const eventPath = path.join(dir, "event.json"); + writeFileSync(eventPath, JSON.stringify({ ref: "refs/heads/main" })); + expect( + pullRequestBaseSha({ GITHUB_EVENT_PATH: eventPath }), + ).toBeUndefined(); + rmSync(dir, { recursive: true, force: true }); + }); +}); diff --git a/scripts/ci-jobs.test.ts b/scripts/ci-jobs.test.ts index c7ea9c3eb9..58d8e5e0ed 100644 --- a/scripts/ci-jobs.test.ts +++ b/scripts/ci-jobs.test.ts @@ -33,6 +33,9 @@ test("CI splits e2e, isolation, and db-suites onto their own Postgres jobs", asy const yaml = await readFile(join(ROOT, ".github/workflows/ci.yml"), "utf8"); const jobs = jobBodies(yaml); + expect(yaml).not.toContain("E2E_REQUIRED"); + expect(yaml).not.toContain("CHECK_BASE_REF"); + expect(jobs.has("walking-skeleton")).toBe(false); for (const name of DB_JOBS) { const body = jobs.get(name); @@ -44,7 +47,6 @@ test("CI splits e2e, isolation, and db-suites onto their own Postgres jobs", asy } const dbSuites = jobs.get("db-suites") ?? ""; - expect(dbSuites).toContain("E2E_REQUIRED:"); expect(dbSuites).toContain("bun test apps/hub/test"); expect(dbSuites).toContain("grep -rl DATABASE_URL"); diff --git a/scripts/e2e/db-gate.test.ts b/scripts/e2e/db-gate.test.ts index 3b698113ab..edf77fbed3 100644 --- a/scripts/e2e/db-gate.test.ts +++ b/scripts/e2e/db-gate.test.ts @@ -1,44 +1,107 @@ // Unit coverage for dbGate: a missing database must never skip in -// total silence. Without E2E_REQUIRED it returns describe.skip (the -// suite still skips quietly in local dev); with E2E_REQUIRED=1 (this -// repo's convention, see harness.ts's e2eDatabaseUrl) it throws -// instead, so CI can never report green on a suite that never ran. +// total silence. Locally (CI unset) it returns describe.skip and the +// summary names `docker-compose.test.yml`. Under CI=true it throws +// instead, so a miswired pipeline cannot report green on a suite that +// never ran. GitHub's unit/structural jobs have no Postgres and are +// not required; they still skip with the same compose hint. import { afterEach, describe, expect, test } from "bun:test"; -import { dbGate } from "./db-gate.ts"; +import { databaseIsRequired, dbGate } from "./db-gate.ts"; -const saved = process.env["E2E_REQUIRED"]; +const gatedKeys = ["CI", "GITHUB_JOB", "E2E_REQUIRED"] as const; +const saved = new Map(gatedKeys.map((key) => [key, process.env[key]])); + +function unset(key: (typeof gatedKeys)[number]): void { + delete process.env[key]; +} afterEach(() => { - if (saved === undefined) delete process.env["E2E_REQUIRED"]; - else process.env["E2E_REQUIRED"] = saved; + for (const key of gatedKeys) { + const value = saved.get(key); + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } +}); + +describe("databaseIsRequired", () => { + test("is false when CI is unset, even if E2E_REQUIRED=1", () => { + unset("CI"); + unset("GITHUB_JOB"); + process.env["E2E_REQUIRED"] = "1"; + expect(databaseIsRequired()).toBe(false); + }); + + test("is true when CI=true and the job is not a unit/structural job", () => { + process.env["CI"] = "true"; + unset("GITHUB_JOB"); + expect(databaseIsRequired()).toBe(true); + process.env["GITHUB_JOB"] = "e2e"; + expect(databaseIsRequired()).toBe(true); + process.env["GITHUB_JOB"] = "isolation"; + expect(databaseIsRequired()).toBe(true); + process.env["GITHUB_JOB"] = "db-suites"; + expect(databaseIsRequired()).toBe(true); + }); + + test("is false on GitHub jobs that never provision Postgres", () => { + process.env["CI"] = "true"; + process.env["GITHUB_JOB"] = "build-test"; + expect(databaseIsRequired()).toBe(false); + process.env["GITHUB_JOB"] = "structural"; + expect(databaseIsRequired()).toBe(false); + process.env["GITHUB_JOB"] = "lint"; + expect(databaseIsRequired()).toBe(false); + process.env["GITHUB_JOB"] = "typecheck"; + expect(databaseIsRequired()).toBe(false); + }); }); describe("dbGate", () => { test("returns describe when a database URL is configured", () => { - delete process.env["E2E_REQUIRED"]; + process.env["CI"] = "true"; + unset("GITHUB_JOB"); expect(dbGate("postgres://localhost:5432/workbench", "example")).toBe( describe, ); }); - test("returns describe.skip when no database is configured and E2E_REQUIRED is unset", () => { - delete process.env["E2E_REQUIRED"]; + test("returns describe.skip when no database is configured and CI is unset", () => { + unset("CI"); + unset("GITHUB_JOB"); expect(dbGate(undefined, "example")).toBe(describe.skip); expect(dbGate("", "example")).toBe(describe.skip); }); - test("throws when E2E_REQUIRED=1 but no database is configured", () => { + test("does not treat E2E_REQUIRED as a required-ness signal", () => { + unset("CI"); + unset("GITHUB_JOB"); process.env["E2E_REQUIRED"] = "1"; + expect(dbGate(undefined, "example")).toBe(describe.skip); + }); + + test("throws when CI=true but no database is configured", () => { + process.env["CI"] = "true"; + unset("GITHUB_JOB"); + expect(() => dbGate(undefined, "example-suite")).toThrow(/example-suite/); expect(() => dbGate(undefined, "example-suite")).toThrow( - /E2E_REQUIRED=1.*example-suite/, + /docker compose -f docker-compose\.test\.yml up -d/, + ); + expect(() => dbGate("", "example-suite")).toThrow( + /postgres:\/\/postgres:postgres@localhost:5432\/workbench/, ); }); - test("does not throw under E2E_REQUIRED=1 once a database is configured", () => { - process.env["E2E_REQUIRED"] = "1"; + test("does not throw under CI=true once a database is configured", () => { + process.env["CI"] = "true"; + unset("GITHUB_JOB"); expect(dbGate("postgres://localhost:5432/workbench", "example")).toBe( describe, ); }); + + test("skips on the GitHub unit job even when CI=true", () => { + process.env["CI"] = "true"; + process.env["GITHUB_JOB"] = "build-test"; + expect(dbGate(undefined, "example")).toBe(describe.skip); + }); }); diff --git a/test/isolation/setup.test.ts b/test/isolation/setup.test.ts index bb796cbaef..0fb2309fbf 100644 --- a/test/isolation/setup.test.ts +++ b/test/isolation/setup.test.ts @@ -1,8 +1,8 @@ // Unit coverage for the isolation suite's database gate: a missing // DATABASE_URL/ISOLATION_DATABASE_URL must only ever skip quietly in -// local dev, never in CI. E2E_REQUIRED=1 is this repo's one convention -// for that (see scripts/e2e/harness.ts's e2eDatabaseUrl), so this test -// proves resolveDatabaseUrl honors it identically. +// local dev, never in CI. dbGate's CI=true convention is this repo's +// one signal for that, so this test proves resolveDatabaseUrl honors +// it identically. import { afterEach, describe, expect, test } from "bun:test"; import { resolveDatabaseUrl } from "./setup.ts"; @@ -10,6 +10,8 @@ import { resolveDatabaseUrl } from "./setup.ts"; const gatedKeys = [ "DATABASE_URL", "ISOLATION_DATABASE_URL", + "CI", + "GITHUB_JOB", "E2E_REQUIRED", ] as const; const saved = new Map(gatedKeys.map((key) => [key, process.env[key]])); @@ -28,32 +30,46 @@ afterEach(() => { }); describe("resolveDatabaseUrl", () => { - test("returns undefined when no database is configured and E2E_REQUIRED is unset", () => { + test("returns undefined when no database is configured and CI is unset", () => { unset("DATABASE_URL"); unset("ISOLATION_DATABASE_URL"); + unset("CI"); + unset("GITHUB_JOB"); unset("E2E_REQUIRED"); expect(resolveDatabaseUrl()).toBeUndefined(); }); - test("throws when E2E_REQUIRED=1 but no database is configured", () => { + test("throws when CI=true but no database is configured", () => { unset("DATABASE_URL"); unset("ISOLATION_DATABASE_URL"); - process.env["E2E_REQUIRED"] = "1"; - expect(() => resolveDatabaseUrl()).toThrow(/E2E_REQUIRED=1/); + process.env["CI"] = "true"; + unset("GITHUB_JOB"); + expect(() => resolveDatabaseUrl()).toThrow( + /docker compose -f docker-compose\.test\.yml up -d/, + ); }); test("ISOLATION_DATABASE_URL wins over DATABASE_URL", () => { process.env["DATABASE_URL"] = "postgres://localhost:5432/other"; process.env["ISOLATION_DATABASE_URL"] = "postgres://localhost:5432/isolation"; - unset("E2E_REQUIRED"); + unset("CI"); expect(resolveDatabaseUrl()).toBe("postgres://localhost:5432/isolation"); }); - test("E2E_REQUIRED=1 does not throw once a database is configured", () => { + test("CI=true does not throw once a database is configured", () => { process.env["DATABASE_URL"] = "postgres://localhost:5432/workbench"; unset("ISOLATION_DATABASE_URL"); - process.env["E2E_REQUIRED"] = "1"; + process.env["CI"] = "true"; + unset("GITHUB_JOB"); expect(resolveDatabaseUrl()).toBe("postgres://localhost:5432/workbench"); }); + + test("does not treat E2E_REQUIRED as a required-ness signal", () => { + unset("DATABASE_URL"); + unset("ISOLATION_DATABASE_URL"); + unset("CI"); + process.env["E2E_REQUIRED"] = "1"; + expect(resolveDatabaseUrl()).toBeUndefined(); + }); }); From 2cdfd6950d23a1fd33cb5fac68a949c02ab43f3c Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sun, 30 Aug 2026 22:52:50 -0700 Subject: [PATCH 2/4] Kill E2E_REQUIRED; infer DB required-ness from CI --- .github/workflows/ci.yml | 15 ++-- apps/hub/test/rate-limit.test.ts | 10 +-- packages/chat/test/migrations.test.ts | 2 +- packages/evals/src/store/store.test.ts | 11 +-- .../webhook-triggers/test/migrations.test.ts | 2 +- scripts/checks/db-gate.ts | 4 +- scripts/checks/report-error.ts | 11 +-- scripts/checks/tool-package-freshness.ts | 63 ++++++++++++++--- scripts/e2e/channel-reconnect.test.ts | 2 +- scripts/e2e/chat.test.ts | 2 +- scripts/e2e/db-gate.ts | 68 ++++++++++++++++--- scripts/e2e/db-setup.test.ts | 18 ++--- scripts/e2e/greeting-delivery.test.ts | 2 +- scripts/e2e/harness.ts | 14 ++-- scripts/e2e/heartbeat.test.ts | 2 +- scripts/e2e/local-rip.test.ts | 2 +- scripts/e2e/play.ts | 2 +- scripts/e2e/routine-repeat.test.ts | 2 +- scripts/e2e/routine-trigger-input.test.ts | 2 +- scripts/e2e/smoke-auth.test.ts | 2 +- scripts/e2e/smoke-chat.test.ts | 2 +- scripts/e2e/smoke-library.test.ts | 2 +- scripts/e2e/smoke-onboarding.test.ts | 2 +- scripts/e2e/smoke-webhook.test.ts | 2 +- scripts/e2e/walking-skeleton.test.ts | 2 +- scripts/e2e/workbench-digest.test.ts | 2 +- scripts/evals-run.ts | 19 ++---- test/isolation/setup.ts | 17 ++--- 28 files changed, 168 insertions(+), 116 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54b22519b6..4a524516ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,8 +75,6 @@ jobs: - run: bun run check:react-ui-pin - run: bun run check:tool-package-pins - run: bun run check:tool-package-freshness - env: - CHECK_BASE_REF: ${{ github.event.pull_request.base.sha }} e2e: runs-on: ubuntu-latest @@ -117,8 +115,7 @@ jobs: # CI stays green. Fail loudly instead: the env file must exist # with a usable DATABASE_URL and SESSION_SECRET, Postgres must be # reachable, and git (the workflow-asset publication path) must be - # present. E2E_REQUIRED=1 below turns any remaining skip into a - # hard failure. + # present. CI=true turns any remaining skip into a hard failure. - name: Assert the e2e test env is wired run: | test -f .env @@ -130,7 +127,6 @@ jobs: - name: Run the walking skeleton run: bun run test:e2e env: - E2E_REQUIRED: "1" DATABASE_URL: postgres://postgres:postgres@localhost:5432/workbench isolation: @@ -175,7 +171,6 @@ jobs: - name: Run the two-org isolation suite run: bun test test/isolation env: - E2E_REQUIRED: "1" DATABASE_URL: postgres://postgres:postgres@localhost:5432/workbench db-suites: @@ -230,7 +225,7 @@ jobs: # apps/hub's own DB-backed suites (e.g. the sign-up rate-limit # proof) never run under the plain `checks` job, which has no - # Postgres. E2E_REQUIRED=1 turns a would-be skip here into a hard + # Postgres. CI=true turns a would-be skip here into a hard # failure, so a misconfigured invocation can't pass vacuously. - name: Run the hub's database-backed suites run: | @@ -239,15 +234,14 @@ jobs: set +a bun test apps/hub/test env: - E2E_REQUIRED: "1" DATABASE_URL: postgres://postgres:postgres@localhost:5432/workbench # Every DB-gated suite under packages/* and apps/hub/src gates on # DATABASE_URL and describe.skip's without one, so build-test (no # Postgres) never runs them. Find them by the same signal they # gate on rather than hardcoding a file list, so a newly added - # suite gets picked up automatically. E2E_REQUIRED=1 turns a - # would-be skip into a hard failure. + # suite gets picked up automatically. CI=true turns a would-be + # skip into a hard failure. - name: Run the database-backed package suites run: | set -a @@ -255,5 +249,4 @@ jobs: set +a bun --env-file="${GITHUB_WORKSPACE}/.env" test $(grep -rl DATABASE_URL --include='*.test.ts' packages apps | grep -v apps/hub/test) env: - E2E_REQUIRED: "1" DATABASE_URL: postgres://postgres:postgres@localhost:5432/workbench diff --git a/apps/hub/test/rate-limit.test.ts b/apps/hub/test/rate-limit.test.ts index b00483014d..16de8b5dd0 100644 --- a/apps/hub/test/rate-limit.test.ts +++ b/apps/hub/test/rate-limit.test.ts @@ -31,14 +31,8 @@ if (!databaseUrl) { // Without a database there is nothing meaningful to assert; skipping // loudly beats a vacuous pass. Set DATABASE_URL or // ISOLATION_DATABASE_URL to run this against a real Postgres. In CI, - // E2E_REQUIRED=1 turns that skip into a loud failure so this proof can - // never silently vanish from the pipeline (mirrors scripts/e2e/harness.ts). - if (process.env["E2E_REQUIRED"] === "1") { - throw new Error( - "E2E_REQUIRED=1 but DATABASE_URL is not set; the sign-up rate-limit " + - "suite would be skipped. Set DATABASE_URL to a reachable Postgres.", - ); - } + // dbGate's CI=true convention turns that skip into a loud failure + // (resolveDatabaseUrl throws) so this proof can never silently vanish. test.skip("sign-up rate limiting (set DATABASE_URL or ISOLATION_DATABASE_URL to run)", () => {}); } else { await prepareDatabase(databaseUrl); diff --git a/packages/chat/test/migrations.test.ts b/packages/chat/test/migrations.test.ts index 2a67ddf530..61d25bd454 100644 --- a/packages/chat/test/migrations.test.ts +++ b/packages/chat/test/migrations.test.ts @@ -1,6 +1,6 @@ // DB-gated: skipped when no DATABASE_URL is reachable (a fresh // checkout still runs the unit gates), and turned into a loud failure -// by E2E_REQUIRED=1 so the suite can never silently vanish from CI — +// by CI=true so the suite can never silently vanish from CI — // mirroring scripts/e2e/harness.ts's e2eDatabaseUrl/baseUrlToE2eUrl. // Runs against its own scratch database, never the developer's or the // walking-skeleton suite's, so a failure here can never corrupt either. diff --git a/packages/evals/src/store/store.test.ts b/packages/evals/src/store/store.test.ts index ac04cfd05e..9e3863d380 100644 --- a/packages/evals/src/store/store.test.ts +++ b/packages/evals/src/store/store.test.ts @@ -1,24 +1,19 @@ // Round-trips EvalRunResult through a real, uniquely-named scratch // Postgres database — never the developer's own DATABASE_URL. Skips // without DATABASE_URL (like every other real-Postgres suite in this -// repo); E2E_REQUIRED=1 turns that skip into a loud CI failure. +// repo); CI=true turns that skip into a loud CI failure. import { afterAll, describe, expect, test } from "bun:test"; import postgres from "postgres"; import { applyEvalsMigrations } from "./migrations.ts"; import { createPostgresEvalRunStore } from "./pg-store.ts"; import type { EvalRunResult } from "../types.ts"; +import { assertDatabaseConfigured } from "../../../../scripts/e2e/db-gate.ts"; function scratchDatabaseUrl(): string | undefined { const base = process.env["DATABASE_URL"]; if (base === undefined || base === "") { - if (process.env["E2E_REQUIRED"] === "1") { - throw new Error( - "E2E_REQUIRED=1 but DATABASE_URL is not set; the evals store " + - "suite would be skipped. Set DATABASE_URL to a reachable " + - "Postgres.", - ); - } + assertDatabaseConfigured(undefined, "evals store suite"); return undefined; } const url = new URL(base); diff --git a/packages/webhook-triggers/test/migrations.test.ts b/packages/webhook-triggers/test/migrations.test.ts index 73d95f10f4..7f01295b41 100644 --- a/packages/webhook-triggers/test/migrations.test.ts +++ b/packages/webhook-triggers/test/migrations.test.ts @@ -1,6 +1,6 @@ // DB-gated: skipped when no DATABASE_URL is reachable (a fresh // checkout still runs the unit gates), and turned into a loud failure -// by E2E_REQUIRED=1 so the suite can never silently vanish from CI — +// by CI=true so the suite can never silently vanish from CI — // mirroring `@corbits/chat`'s own migrations test. Runs against its // own scratch database, never the developer's or the walking-skeleton // suite's, so a failure here can never corrupt either. diff --git a/scripts/checks/db-gate.ts b/scripts/checks/db-gate.ts index a0d75351e5..9341d84fca 100644 --- a/scripts/checks/db-gate.ts +++ b/scripts/checks/db-gate.ts @@ -1,7 +1,7 @@ // check:db-gate — every DB-gated suite must skip through dbGate // (scripts/e2e/db-gate.ts), never a hand-rolled // `databaseUrl ? describe.skip : describe` ternary. A hand-rolled gate -// skips in total silence and never honors E2E_REQUIRED=1 — exactly +// skips in total silence and never honors CI=true — exactly // the gap CL-7279 closed, one file at a time, until the next merge // added a new hand-rolled copy the sweep never saw. This check makes // the invariant self-enforcing instead of a one-time sweep. @@ -49,7 +49,7 @@ export function auditDbGate( report.violations.push( `${relPath}: hand-rolls a describeIfDb gate instead of using dbGate ` + `from scripts/e2e/db-gate.ts. A hand-rolled gate skips silently ` + - `and never honors E2E_REQUIRED=1 — replace it with ` + + `and never honors CI=true — replace it with ` + `\`const describeIfDb = dbGate(databaseUrl, import.meta.path);\`.`, ); } diff --git a/scripts/checks/report-error.ts b/scripts/checks/report-error.ts index fd83ae6712..98916c0dc3 100644 --- a/scripts/checks/report-error.ts +++ b/scripts/checks/report-error.ts @@ -56,9 +56,10 @@ // // This follows check:tool-package-freshness's own precedent for scoping a // new invariant to what changed rather than retroactively flagging -// pre-existing state: CI passes CHECK_BASE_REF; locally this falls back to -// the merge base with origin/main; with neither available the touched-file -// half of the gate no-ops (new-vs-baseline enforcement still applies). +// pre-existing state: on a GitHub pull_request job the base SHA comes +// from the event payload; locally this falls back to the merge base with +// origin/main; with neither available the touched-file half of the gate +// no-ops (new-vs-baseline enforcement still applies). // // Regenerate the baseline after fixing (or newly opting out) entries: // bun run scripts/checks/report-error.ts --write-baseline @@ -547,7 +548,7 @@ async function main(): Promise { ? parseBaseline(await baselineFile.text()) : new Set(); - const baseRef = resolveBaseRef(root, process.env["CHECK_BASE_REF"]); + const baseRef = resolveBaseRef(root); const changedLines = baseRef === undefined ? undefined @@ -561,7 +562,7 @@ async function main(): Promise { }); if (baseRef === undefined) { report.notes.push( - "no base ref (no origin/main, no CHECK_BASE_REF); skipping the " + + "no base ref (no origin/main, no pull_request.base.sha); skipping the " + "touched-line ratchet — CI supplies the base ref for the " + "authoritative run. New-vs-baseline enforcement still applies.", ); diff --git a/scripts/checks/tool-package-freshness.ts b/scripts/checks/tool-package-freshness.ts index 7eba091bec..f98c1f27fe 100644 --- a/scripts/checks/tool-package-freshness.ts +++ b/scripts/checks/tool-package-freshness.ts @@ -16,6 +16,7 @@ // history rather than a snapshot of the working tree, which is why it is a // separate check. import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; import path from "node:path"; import { emptyReport, @@ -59,15 +60,59 @@ function git(root: string, args: readonly string[]): string | undefined { } /** - * The commit this change branched from. CI supplies the base ref - * explicitly; locally the merge base with `origin/main` answers the same - * question a reviewer would ask. + * The PR base SHA GitHub Actions writes into GITHUB_EVENT_PATH. + * Absent (local runs, push events, unreadable payload) means fall + * through to merge-base with origin/main. */ -export function resolveBaseRef( - root: string, - explicit: string | undefined, +export function pullRequestBaseSha( + env: NodeJS.ProcessEnv = process.env, ): string | undefined { - if (explicit !== undefined && explicit.length > 0) return explicit; + const eventPath = env["GITHUB_EVENT_PATH"]; + if (eventPath === undefined || eventPath === "") return undefined; + let raw: string; + try { + raw = readFileSync(eventPath, "utf8"); + } catch { + return undefined; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return undefined; + } + if ( + typeof parsed !== "object" || + parsed === null || + !("pull_request" in parsed) + ) { + return undefined; + } + const pullRequest = parsed.pull_request; + if ( + typeof pullRequest !== "object" || + pullRequest === null || + !("base" in pullRequest) + ) { + return undefined; + } + const base = pullRequest.base; + if (typeof base !== "object" || base === null || !("sha" in base)) { + return undefined; + } + const sha = base.sha; + return typeof sha === "string" && sha.length > 0 ? sha : undefined; +} + +/** + * The commit this change branched from. On a GitHub pull_request job + * that is the PR base SHA from the event payload; locally (and on + * push) the merge base with origin/main answers the same question a + * reviewer would ask. + */ +export function resolveBaseRef(root: string): string | undefined { + const fromEvent = pullRequestBaseSha(); + if (fromEvent !== undefined) return fromEvent; return git(root, ["merge-base", "HEAD", "origin/main"]); } @@ -138,11 +183,11 @@ async function versionAtHead( async function main(): Promise { const root = rootFromArgs(Bun.argv.slice(2)); - const baseRef = resolveBaseRef(root, process.env["CHECK_BASE_REF"]); + const baseRef = resolveBaseRef(root); if (baseRef === undefined) { const report = emptyReport(); report.notes.push( - "no base ref (no origin/main, no CHECK_BASE_REF); skipping — CI " + + "no base ref (no origin/main, no pull_request.base.sha); skipping — CI " + "supplies the base ref for the authoritative run", ); reportAndExit("check:tool-package-freshness", report); diff --git a/scripts/e2e/channel-reconnect.test.ts b/scripts/e2e/channel-reconnect.test.ts index 9b947cacd9..9843404850 100644 --- a/scripts/e2e/channel-reconnect.test.ts +++ b/scripts/e2e/channel-reconnect.test.ts @@ -41,7 +41,7 @@ const databaseUrl = e2eDatabaseUrl(); if (databaseUrl === undefined) { console.warn( "workbench-reconnect e2e: DATABASE_URL is not set; suite skipped. Set " + - "DATABASE_URL (see .env.example) to run it; CI sets E2E_REQUIRED=1 " + + "DATABASE_URL (see .env.example) to run it; start Postgres with `docker compose -f docker-compose.test.yml up -d` " + "so this skip can never pass silently there.", ); } diff --git a/scripts/e2e/chat.test.ts b/scripts/e2e/chat.test.ts index d7802a9b44..6c40c3e0eb 100644 --- a/scripts/e2e/chat.test.ts +++ b/scripts/e2e/chat.test.ts @@ -60,7 +60,7 @@ const databaseUrl = e2eDatabaseUrl(); if (databaseUrl === undefined) { console.warn( "chat e2e: DATABASE_URL is not set; suite skipped. Set DATABASE_URL " + - "(see .env.example) to run it; CI sets E2E_REQUIRED=1 so this skip " + + "(see .env.example) to run it; start Postgres with `docker compose -f docker-compose.test.yml up -d` so this skip " + "can never pass silently there.", ); } diff --git a/scripts/e2e/db-gate.ts b/scripts/e2e/db-gate.ts index 357df27f90..5a2684d18f 100644 --- a/scripts/e2e/db-gate.ts +++ b/scripts/e2e/db-gate.ts @@ -4,11 +4,61 @@ // total silence — a broken CI env would report green the same way a // real pass does. dbGate is the one place that decides what a skip // means: it counts the skip, prints an unmissable summary once the -// run ends, and — under E2E_REQUIRED=1 (this repo's convention for -// turning a DB skip into a hard failure, see harness.ts) — throws -// instead of skipping. +// run ends, and — when CI=true (except GitHub jobs that never +// provision Postgres) — throws instead of skipping. import { afterAll, describe } from "bun:test"; +export const TEST_COMPOSE_FILE = "docker-compose.test.yml"; +export const TEST_COMPOSE_UP = `docker compose -f ${TEST_COMPOSE_FILE} up -d`; +export const TEST_DATABASE_URL = + "postgres://postgres:postgres@localhost:5432/workbench"; + +export const MISSING_DATABASE_HINT = + `Start Postgres with \`${TEST_COMPOSE_UP}\`, then ` + + `DATABASE_URL=${TEST_DATABASE_URL} bun test ...`; + +// GitHub Actions sets CI=true on every job. The unit/structural jobs +// never provision Postgres; DB-gated suites skip there (loudly) and +// run in e2e / isolation / db-suites, which do. Any other CI context +// — including `CI=true bun test` locally — treats a missing +// DATABASE_URL as a hard failure so a miswired pipeline cannot skip +// green. +const CI_JOBS_WITHOUT_POSTGRES = new Set([ + "lint", + "typecheck", + "build-test", + "structural", +]); + +export function databaseIsRequired( + env: NodeJS.ProcessEnv = process.env, +): boolean { + if (env["CI"] !== "true") return false; + const job = env["GITHUB_JOB"]; + if (job !== undefined && job !== "" && CI_JOBS_WITHOUT_POSTGRES.has(job)) { + return false; + } + return true; +} + +export function missingDatabaseError(label: string): Error { + return new Error( + `DATABASE_URL is not set; "${label}" cannot run. ${MISSING_DATABASE_HINT}`, + ); +} + +export function skippedDatabaseWarning(label: string): string { + return `${label}: DATABASE_URL is not set; suite skipped. ${MISSING_DATABASE_HINT}`; +} + +export function assertDatabaseConfigured( + databaseUrl: string | undefined, + label: string, +): void { + if (databaseUrl !== undefined && databaseUrl !== "") return; + if (databaseIsRequired()) throw missingDatabaseError(label); +} + // `bun test` never fires `process.on("exit"/"beforeExit")` handlers, so a // true end-of-run hook does not exist across files — each skipped file // registers its own `afterAll`, printed against the shared, growing list @@ -25,9 +75,9 @@ function printSummary(): void { `SKIPPED ${skipped.length} DB-gated suite(s) so far — no DATABASE_URL. This is NOT a pass.`, ...skipped.map((label) => ` - ${label}`), "", - "To run them: docker compose -f docker-compose.test.yml up -d, then", - "DATABASE_URL=postgres://postgres:postgres@localhost:5432/workbench bun test ...", - "Set E2E_REQUIRED=1 to make a skip like this a hard failure (CI does).", + `To run them: ${TEST_COMPOSE_UP}, then`, + `DATABASE_URL=${TEST_DATABASE_URL} bun test ...`, + "CI=true turns a skip like this into a hard failure.", rule, "", ]; @@ -43,12 +93,8 @@ export function dbGate( label: string, ): typeof describe { if (databaseUrl !== undefined && databaseUrl !== "") return describe; + if (databaseIsRequired()) throw missingDatabaseError(label); skipped.push(label); afterAll(printSummary); - if (process.env["E2E_REQUIRED"] === "1") { - throw new Error( - `E2E_REQUIRED=1 but DATABASE_URL is not set; "${label}" would be skipped.`, - ); - } return describe.skip; } diff --git a/scripts/e2e/db-setup.test.ts b/scripts/e2e/db-setup.test.ts index 42576d8d3b..23dd26b92b 100644 --- a/scripts/e2e/db-setup.test.ts +++ b/scripts/e2e/db-setup.test.ts @@ -9,8 +9,8 @@ // The suite owns a uniquely-named sibling scratch database (created by // setupDatabase on first use, dropped in teardown) so it never touches // the developer's own database. Like the walking skeleton it skips -// without DATABASE_URL, and E2E_REQUIRED=1 turns that skip into a -// loud failure in CI. +// without DATABASE_URL, and CI=true turns that skip into a loud +// failure. import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import { readdir } from "node:fs/promises"; @@ -22,6 +22,7 @@ import { resetSchema, setupDatabase, } from "../db-setup.ts"; +import { assertDatabaseConfigured, skippedDatabaseWarning } from "./db-gate.ts"; const REPO_ROOT = path.resolve(import.meta.dir, "..", ".."); const HUB_DIR = path.join(REPO_ROOT, "apps", "hub"); @@ -36,12 +37,7 @@ const VENDORED_MIGRATIONS_DIR = path.join( function scratchDatabaseUrl(): string | undefined { const base = process.env["DATABASE_URL"]; if (base === undefined || base === "") { - if (process.env["E2E_REQUIRED"] === "1") { - throw new Error( - "E2E_REQUIRED=1 but DATABASE_URL is not set; the db-setup suite " + - "would be skipped. Set DATABASE_URL to a reachable Postgres.", - ); - } + assertDatabaseConfigured(undefined, "db-setup suite"); return undefined; } const url = new URL(base); @@ -56,11 +52,7 @@ function scratchDatabaseUrl(): string | undefined { const databaseUrl = scratchDatabaseUrl(); if (databaseUrl === undefined) { - console.warn( - "db-setup: DATABASE_URL is not set; suite skipped. " + - "Set DATABASE_URL (see .env.example) to run it; " + - "CI sets E2E_REQUIRED=1 so this skip can never pass silently there.", - ); + console.warn(skippedDatabaseWarning("db-setup")); } // Minimal local view of the postgres client, resolved out of the hub's diff --git a/scripts/e2e/greeting-delivery.test.ts b/scripts/e2e/greeting-delivery.test.ts index 80e8948219..89e1737398 100644 --- a/scripts/e2e/greeting-delivery.test.ts +++ b/scripts/e2e/greeting-delivery.test.ts @@ -53,7 +53,7 @@ const databaseUrl = e2eDatabaseUrl(); if (databaseUrl === undefined) { console.warn( "greeting-delivery: DATABASE_URL is not set; suite skipped. Set " + - "DATABASE_URL (see .env.example) to run it; CI sets E2E_REQUIRED=1 " + + "DATABASE_URL (see .env.example) to run it; start Postgres with `docker compose -f docker-compose.test.yml up -d` " + "so this skip can never pass silently there.", ); } diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index 0b40b031da..643b514f16 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -11,6 +11,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { createGitWorkflowPusher } from "../../packages/hub-client/src/index.ts"; import { WORKFLOW_SOURCE_ENTRY } from "../../packages/workflow-source/src/index.ts"; +import { assertDatabaseConfigured } from "./db-gate.ts"; export const REPO_ROOT = path.resolve(import.meta.dir, "..", ".."); const HUB_DIR = path.join(REPO_ROOT, "apps", "hub"); @@ -21,9 +22,9 @@ const SIDECAR_DIR = path.join(REPO_ROOT, "apps", "sidecar"); /** * The suite needs a real Postgres, named by DATABASE_URL. Locally a * missing DATABASE_URL skips the suite (a fresh checkout without a - * database still runs the unit gates); in CI, E2E_REQUIRED=1 turns - * that skip into a loud failure so the suite can never silently - * vanish from the pipeline. + * database still runs the unit gates); in CI, CI=true turns that skip + * into a loud failure so the suite can never silently vanish from the + * pipeline. * * `bun test` workers do not always inherit process.env.DATABASE_URL * even when the shell sourced `.env`. Fall back to the same repo-root @@ -36,12 +37,7 @@ export function e2eDatabaseUrl(): string | undefined { ? fromProcess : databaseUrlFromRepoEnvFile(); if (url !== undefined && url !== "") return baseUrlToE2eUrl(url); - if (process.env["E2E_REQUIRED"] === "1") { - throw new Error( - "E2E_REQUIRED=1 but DATABASE_URL is not set; the walking-skeleton " + - "suite would be skipped. Set DATABASE_URL to a reachable Postgres.", - ); - } + assertDatabaseConfigured(undefined, "walking-skeleton suite"); return undefined; } diff --git a/scripts/e2e/heartbeat.test.ts b/scripts/e2e/heartbeat.test.ts index 8d55adec94..c19a6b6a2c 100644 --- a/scripts/e2e/heartbeat.test.ts +++ b/scripts/e2e/heartbeat.test.ts @@ -42,7 +42,7 @@ if (databaseUrl === undefined) { console.warn( "heartbeat: DATABASE_URL is not set; suite skipped. " + "Set DATABASE_URL (see .env.example) to run it; " + - "CI sets E2E_REQUIRED=1 so this skip can never pass silently there.", + "start Postgres with `docker compose -f docker-compose.test.yml up -d`.", ); } diff --git a/scripts/e2e/local-rip.test.ts b/scripts/e2e/local-rip.test.ts index 8684c92760..3bc0dd80c3 100644 --- a/scripts/e2e/local-rip.test.ts +++ b/scripts/e2e/local-rip.test.ts @@ -90,7 +90,7 @@ const databaseUrl = e2eDatabaseUrl(); if (databaseUrl === undefined) { console.warn( "local-rip: DATABASE_URL is not set; suite skipped. Set DATABASE_URL " + - "(see .env.example) to run it; CI sets E2E_REQUIRED=1 so this skip " + + "(see .env.example) to run it; start Postgres with `docker compose -f docker-compose.test.yml up -d` so this skip " + "can never pass silently there.", ); } diff --git a/scripts/e2e/play.ts b/scripts/e2e/play.ts index 1334757794..c4d0b018c0 100644 --- a/scripts/e2e/play.ts +++ b/scripts/e2e/play.ts @@ -71,7 +71,7 @@ const databaseUrl = e2eDatabaseUrl(); if (databaseUrl === undefined) { console.warn( "greeting-delivery: DATABASE_URL is not set; suite skipped. Set " + - "DATABASE_URL (see .env.example) to run it; CI sets E2E_REQUIRED=1 " + + "DATABASE_URL (see .env.example) to run it; start Postgres with `docker compose -f docker-compose.test.yml up -d` " + "so this skip can never pass silently there.", ); } diff --git a/scripts/e2e/routine-repeat.test.ts b/scripts/e2e/routine-repeat.test.ts index f13a5d0e7a..75a5135f0b 100644 --- a/scripts/e2e/routine-repeat.test.ts +++ b/scripts/e2e/routine-repeat.test.ts @@ -34,7 +34,7 @@ if (databaseUrl === undefined) { console.warn( "routine-repeat: DATABASE_URL is not set; suite skipped. " + "Set DATABASE_URL (see .env.example) to run it; " + - "CI sets E2E_REQUIRED=1 so this skip can never pass silently there.", + "start Postgres with `docker compose -f docker-compose.test.yml up -d`.", ); } diff --git a/scripts/e2e/routine-trigger-input.test.ts b/scripts/e2e/routine-trigger-input.test.ts index bb0eb15698..0dff451e3c 100644 --- a/scripts/e2e/routine-trigger-input.test.ts +++ b/scripts/e2e/routine-trigger-input.test.ts @@ -52,7 +52,7 @@ const databaseUrl = e2eDatabaseUrl(); if (databaseUrl === undefined) { console.warn( "routine-trigger-input: DATABASE_URL is not set; suite skipped. Set " + - "DATABASE_URL (see .env.example) to run it; CI sets E2E_REQUIRED=1 " + + "DATABASE_URL (see .env.example) to run it; start Postgres with `docker compose -f docker-compose.test.yml up -d` " + "so this skip can never pass silently there.", ); } diff --git a/scripts/e2e/smoke-auth.test.ts b/scripts/e2e/smoke-auth.test.ts index 66a8d9efc6..401d8275da 100644 --- a/scripts/e2e/smoke-auth.test.ts +++ b/scripts/e2e/smoke-auth.test.ts @@ -23,7 +23,7 @@ const databaseUrl = e2eDatabaseUrl(); if (databaseUrl === undefined) { console.warn( "smoke-auth: DATABASE_URL is not set; suite skipped. Set DATABASE_URL " + - "(see .env.example) to run it; CI sets E2E_REQUIRED=1 so this skip " + + "(see .env.example) to run it; start Postgres with `docker compose -f docker-compose.test.yml up -d` so this skip " + "can never pass silently there.", ); } diff --git a/scripts/e2e/smoke-chat.test.ts b/scripts/e2e/smoke-chat.test.ts index abc28e48db..98b5983321 100644 --- a/scripts/e2e/smoke-chat.test.ts +++ b/scripts/e2e/smoke-chat.test.ts @@ -32,7 +32,7 @@ const databaseUrl = e2eDatabaseUrl(); if (databaseUrl === undefined) { console.warn( "smoke-chat: DATABASE_URL is not set; suite skipped. Set DATABASE_URL " + - "(see .env.example) to run it; CI sets E2E_REQUIRED=1 so this skip " + + "(see .env.example) to run it; start Postgres with `docker compose -f docker-compose.test.yml up -d` so this skip " + "can never pass silently there.", ); } diff --git a/scripts/e2e/smoke-library.test.ts b/scripts/e2e/smoke-library.test.ts index 755bf08f98..808a49d33e 100644 --- a/scripts/e2e/smoke-library.test.ts +++ b/scripts/e2e/smoke-library.test.ts @@ -29,7 +29,7 @@ const databaseUrl = e2eDatabaseUrl(); if (databaseUrl === undefined) { console.warn( "smoke-library: DATABASE_URL is not set; suite skipped. Set " + - "DATABASE_URL (see .env.example) to run it; CI sets E2E_REQUIRED=1 " + + "DATABASE_URL (see .env.example) to run it; start Postgres with `docker compose -f docker-compose.test.yml up -d` " + "so this skip can never pass silently there.", ); } diff --git a/scripts/e2e/smoke-onboarding.test.ts b/scripts/e2e/smoke-onboarding.test.ts index 9bd5abc224..1c63fe40f8 100644 --- a/scripts/e2e/smoke-onboarding.test.ts +++ b/scripts/e2e/smoke-onboarding.test.ts @@ -27,7 +27,7 @@ const databaseUrl = e2eDatabaseUrl(); if (databaseUrl === undefined) { console.warn( "smoke-onboarding: DATABASE_URL is not set; suite skipped. Set " + - "DATABASE_URL (see .env.example) to run it; CI sets E2E_REQUIRED=1 " + + "DATABASE_URL (see .env.example) to run it; start Postgres with `docker compose -f docker-compose.test.yml up -d` " + "so this skip can never pass silently there.", ); } diff --git a/scripts/e2e/smoke-webhook.test.ts b/scripts/e2e/smoke-webhook.test.ts index f114154b25..794ded269a 100644 --- a/scripts/e2e/smoke-webhook.test.ts +++ b/scripts/e2e/smoke-webhook.test.ts @@ -53,7 +53,7 @@ const databaseUrl = e2eDatabaseUrl(); if (databaseUrl === undefined) { console.warn( "smoke-webhook: DATABASE_URL is not set; suite skipped. Set " + - "DATABASE_URL (see .env.example) to run it; CI sets E2E_REQUIRED=1 " + + "DATABASE_URL (see .env.example) to run it; start Postgres with `docker compose -f docker-compose.test.yml up -d` " + "so this skip can never pass silently there.", ); } diff --git a/scripts/e2e/walking-skeleton.test.ts b/scripts/e2e/walking-skeleton.test.ts index d0799f3ef0..af63a4192a 100644 --- a/scripts/e2e/walking-skeleton.test.ts +++ b/scripts/e2e/walking-skeleton.test.ts @@ -47,7 +47,7 @@ if (databaseUrl === undefined) { console.warn( "walking-skeleton: DATABASE_URL is not set; suite skipped. " + "Set DATABASE_URL (see .env.example) to run it; " + - "CI sets E2E_REQUIRED=1 so this skip can never pass silently there.", + "start Postgres with `docker compose -f docker-compose.test.yml up -d`.", ); } diff --git a/scripts/e2e/workbench-digest.test.ts b/scripts/e2e/workbench-digest.test.ts index 212a738247..1e8795f1de 100644 --- a/scripts/e2e/workbench-digest.test.ts +++ b/scripts/e2e/workbench-digest.test.ts @@ -42,7 +42,7 @@ if (databaseUrl === undefined) { console.warn( "workbench-digest: DATABASE_URL is not set; suite skipped. " + "Set DATABASE_URL (see .env.example) to run it; " + - "CI sets E2E_REQUIRED=1 so this skip can never pass silently there.", + "start Postgres with `docker compose -f docker-compose.test.yml up -d`.", ); } diff --git a/scripts/evals-run.ts b/scripts/evals-run.ts index a3e9ac0c57..28835f55f0 100644 --- a/scripts/evals-run.ts +++ b/scripts/evals-run.ts @@ -18,7 +18,7 @@ // // Needs DATABASE_URL (see .env.example); like every e2e-shaped suite // in this repo, a missing DATABASE_URL skips with a warning locally -// and fails loudly when E2E_REQUIRED=1 (CI). +// and fails loudly when CI=true. import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; @@ -49,6 +49,10 @@ import { startHub, startSidecar, } from "./e2e/harness.ts"; +import { + assertDatabaseConfigured, + skippedDatabaseWarning, +} from "./e2e/db-gate.ts"; function dbConfigFromUrl(databaseUrl: string) { const url = new URL(databaseUrl); @@ -105,12 +109,7 @@ const infra: MyraTargetInfra = { function databaseUrl(): string | undefined { const url = process.env["DATABASE_URL"]; if (url !== undefined && url !== "") return url; - if (process.env["E2E_REQUIRED"] === "1") { - throw new Error( - "E2E_REQUIRED=1 but DATABASE_URL is not set; `bun run eval` would be " + - "skipped. Set DATABASE_URL to a reachable Postgres.", - ); - } + assertDatabaseConfigured(undefined, "bun run eval"); return undefined; } @@ -145,11 +144,7 @@ function assertPlumbingOnly(results: readonly EvalRunResult[]): void { async function main(): Promise { const url = databaseUrl(); if (url === undefined) { - process.stdout.write( - "bun run eval: DATABASE_URL is not set; skipped. Set DATABASE_URL " + - "(see .env.example) to run it; CI sets E2E_REQUIRED=1 so this skip " + - "can never pass silently there.\n", - ); + process.stdout.write(skippedDatabaseWarning("bun run eval") + "\n"); return; } diff --git a/test/isolation/setup.ts b/test/isolation/setup.ts index f70c558e35..3644750fa6 100644 --- a/test/isolation/setup.ts +++ b/test/isolation/setup.ts @@ -16,6 +16,7 @@ import { type HubHandle, type SpawnedApp, } from "../../scripts/e2e/harness.ts"; +import { assertDatabaseConfigured } from "../../scripts/e2e/db-gate.ts"; /** * The minimal surface of the hub the suite needs: a fetch-shaped @@ -46,10 +47,10 @@ export const ISOLATION_SIGNUP_RATE_LIMIT_WINDOW_SECONDS = 60; * The database the suite runs against. The suite never invents a * default: pointing it at a database is an explicit act, because it * creates real rows there. Locally, an unconfigured database skips the - * suite (a fresh checkout still runs the unit gates); E2E_REQUIRED=1 - * turns that same gap into a hard failure, mirroring - * scripts/e2e/harness.ts's e2eDatabaseUrl — this suite guards - * multi-tenant authorization, so it must never vanish from CI silently. + * suite (a fresh checkout still runs the unit gates); CI=true turns + * that same gap into a hard failure, mirroring dbGate — this suite + * guards multi-tenant authorization, so it must never vanish from CI + * silently. */ export function resolveDatabaseUrl(): string | undefined { const isolationUrl = process.env["ISOLATION_DATABASE_URL"]; @@ -58,13 +59,7 @@ export function resolveDatabaseUrl(): string | undefined { ? isolationUrl : process.env["DATABASE_URL"]; if (url !== undefined && url !== "") return url; - if (process.env["E2E_REQUIRED"] === "1") { - throw new Error( - "E2E_REQUIRED=1 but no database is configured for the isolation " + - "suite; set DATABASE_URL or ISOLATION_DATABASE_URL to a reachable " + - "Postgres.", - ); - } + assertDatabaseConfigured(undefined, "isolation suite"); return undefined; } From 67cfaf01add8ebe7647195459db7d0187b7b747a Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sun, 30 Aug 2026 22:52:59 -0700 Subject: [PATCH 3/4] Update docs: remaining test-only CI flags --- AGENTS.md | 32 +++++++++++++++++++++++++++++--- IMPLEMENTATION.md | 7 ++++--- README.md | 13 +++++++------ 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a975a5727c..838dd8a012 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,6 +57,31 @@ behind human approval. logical change per commit; commit messages are written for a public audience. - Worktrees live in `.worktrees/`; branch = `cl--`. +- Remaining test/CI flags (what breaks if unset): + - `DATABASE_URL` — DB-gated suites skip locally with a banner naming + `docker compose -f docker-compose.test.yml up -d`. On CI jobs that + provision Postgres (`e2e`, `isolation`, `db-suites`), `CI=true` + turns a missing URL into a hard failure via `dbGate`. The unit + jobs (`build-test`, `structural`, `lint`, `typecheck`) never + provision Postgres and still skip. + - `HUB_DATA_DIR` — hub boot fails. This is required runtime config, + not a test flag. + - `CI` — set automatically by GitHub Actions (`true`). Not a caller + flag; `dbGate` infers required-ness from it. + - `ISOLATION_DATABASE_URL` — optional override for the isolation + suite; unset falls through to `DATABASE_URL`. + - `E2E_PROVIDER` / `E2E_PROVIDER_API_KEY` / `OLLAMA_BASE_URL` — live + inference e2e (walkthrough, greeting-delivery, play, proof + scripts). Unset stays on the noop/stub path; the suite still runs. + - `EVAL_PROVIDER` / `EVAL_PROVIDER_API_KEY` — live evals. Unset is + plumbing-only (`bun run eval` still runs against the stub). + - `WORKBENCH_CHECK_CONCURRENCY` — override `scripts/run-all.ts` + parallelism. Unset uses a core-count heuristic (all cores on + GitHub Actions, cores-2 locally). + - `E2E_LOG_DIR` — optional directory for hub/sidecar e2e logs. + Unset does not capture them. + - `CHROME_PATH` — browser walkthrough looks for Chrome here first. + Unset falls through to platform defaults and fails if none exist. - Tests are meaningful red/green tests only — no coverage theater. Unit tests for pure modules sit next to the source they cover (`src/**/*.test.ts`); multi-module / DOM / composition suites stay under @@ -109,11 +134,12 @@ scripts/checks/report-error.ts --write-baseline` after fixing (or package's own `package.json` version — `check:tool-package-pins` catches the mismatch. A tool package's `src/` changing without a version bump is the other half of that class, caught separately by - `check:tool-package-freshness`: it diffs against the base ref (CI passes - `CHECK_BASE_REF`; locally it falls back to the merge base with + `check:tool-package-freshness`: it diffs against the base ref (a + GitHub `pull_request` job reads `pull_request.base.sha` from + `GITHUB_EVENT_PATH`; locally it falls back to the merge base with `origin/main`) and flags any tool package whose `src/` differs from that base while its version stayed the same. With no base ref available - (no `origin/main`, no `CHECK_BASE_REF`) it no-ops, deferring to CI as the + (no `origin/main`, no pull-request event) it no-ops, deferring to CI as the authoritative run. - Every package needs a `LICENSE` file (canonical LGPL-2.1-or-later text, copy from any existing `packages/*/LICENSE`) — `check:licenses` fails diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 719e627b56..03e704507b 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -99,9 +99,10 @@ suite needs zero real credentials: `startHub` only forwards an explicit env allowlist, so a real `ANTHROPIC_API_KEY` in the shell never reaches the spawned hub, and every inference source in a test points at the hub's own `noop-inference` endpoint or an unreachable placeholder host — -enforced by `assertNeverRealProvider` in `scripts/e2e/harness.ts`. CI sets -`E2E_REQUIRED=1` so a missing `DATABASE_URL` fails loudly there instead of -silently skipping. +enforced by `assertNeverRealProvider` in `scripts/e2e/harness.ts`. CI infers +required-ness from `CI=true` so a missing `DATABASE_URL` fails loudly there +instead of silently skipping. Start Postgres locally with +`docker compose -f docker-compose.test.yml up -d`. ## Deployment diff --git a/README.md b/README.md index bb7d61e193..ae41937460 100644 --- a/README.md +++ b/README.md @@ -185,9 +185,10 @@ in any order: [Running locally](#running-locally)). Each suite reuses a sibling `_e2e` database that it owns outright — drops and rebuilds its own schema on every run — so it can never touch your working database. -Without `DATABASE_URL` the suite skips with a warning; CI sets -`E2E_REQUIRED=1` so that skip fails loudly there instead of silently -passing. +Without `DATABASE_URL` the suite skips with a warning naming +`docker compose -f docker-compose.test.yml up -d`; CI infers +required-ness from `CI=true` so that skip fails loudly there instead of +silently passing. **Needs zero real credentials.** The whole suite — smoke and non-smoke files alike — runs with no API keys, no paid credentials, and no network @@ -212,9 +213,9 @@ DB-gated tests (migrations, Drizzle-backed stores, route integration tests) that skip via `describeIfDb` when `DATABASE_URL` is unset. A skip is never silent: `scripts/e2e/db-gate.ts`'s `dbGate` — what `describeIfDb` is built on — prints an unmissable summary naming every -skipped suite at the end of the run, and honors the same -`E2E_REQUIRED=1` convention as the e2e suite above, turning a skip into -a hard failure. CI sets `E2E_REQUIRED=1` for exactly this reason. +skipped suite at the end of the run, and infers required-ness from +`CI=true` the same way the e2e suite above does, turning a skip into +a hard failure on CI jobs that provision Postgres. To run these locally against the same Postgres CI uses: From 10493864969054e33930f8d61f9b12a689586d4d Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sun, 30 Aug 2026 23:10:23 -0700 Subject: [PATCH 4/4] Fix db-gate tests to unset env without dynamic delete --- scripts/e2e/db-gate.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/e2e/db-gate.test.ts b/scripts/e2e/db-gate.test.ts index edf77fbed3..0bf569090d 100644 --- a/scripts/e2e/db-gate.test.ts +++ b/scripts/e2e/db-gate.test.ts @@ -11,15 +11,15 @@ import { databaseIsRequired, dbGate } from "./db-gate.ts"; const gatedKeys = ["CI", "GITHUB_JOB", "E2E_REQUIRED"] as const; const saved = new Map(gatedKeys.map((key) => [key, process.env[key]])); +/** Unsets a gated variable for one test. Empty string is not `CI=true` + * and is not a named GITHUB_JOB, same as a variable the shell never set. */ function unset(key: (typeof gatedKeys)[number]): void { - delete process.env[key]; + process.env[key] = ""; } afterEach(() => { for (const key of gatedKeys) { - const value = saved.get(key); - if (value === undefined) delete process.env[key]; - else process.env[key] = value; + process.env[key] = saved.get(key) ?? ""; } });