From f60927136900b658dc8d5230492bf3394c15d26d Mon Sep 17 00:00:00 2001 From: Peli de Halleux <4175913+pelikhan@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:18:36 -0800 Subject: [PATCH 1/7] Fix Workflow contracts CI after Activity runtime removal - Assign release-schedule unit test to the workflows area - Drop removed JS repository discovery from inventory ingestion test - Assert the removed validation report job stays absent - Verify only the remaining dashboard runtime bundle - Remove the CI job for the deleted issueOrPullRequest test Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/workflow-contracts.yml | 20 ----------- scripts/run-unit-area.mjs | 2 +- tests/integration/campaign-lifecycle.test.mjs | 2 +- .../control-inventory-credentials.test.mjs | 2 +- .../dashboard-inventory-ingestion.test.mjs | 34 +++++-------------- tests/integration/install-script.test.mjs | 2 +- .../workflow-contract-ci-triggers.test.mjs | 2 +- 7 files changed, 14 insertions(+), 50 deletions(-) diff --git a/.github/workflows/workflow-contracts.yml b/.github/workflows/workflow-contracts.yml index 9f1d208059..def7c0bd86 100644 --- a/.github/workflows/workflow-contracts.yml +++ b/.github/workflows/workflow-contracts.yml @@ -227,26 +227,6 @@ jobs: npm run check:svg npm run docs:build - github-issue-query: - name: GitHub issue GraphQL integration - if: github.event_name != 'pull_request' - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - issues: read - steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - persist-credentials: false - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 - with: - node-version: 24 - - name: Test issueOrPullRequest query - env: - GH_TOKEN: ${{ github.token }} - run: node --test tests/integration/github-issue-or-pull-request.test.mjs - dependabot-operational-value: name: Dependabot operational value integration if: github.event_name != 'pull_request' diff --git a/scripts/run-unit-area.mjs b/scripts/run-unit-area.mjs index 0067ac0f68..4aae500b11 100644 --- a/scripts/run-unit-area.mjs +++ b/scripts/run-unit-area.mjs @@ -11,7 +11,7 @@ export const unitTestAreas = { dashboard: [/^dashboard-/, /^cli-action-/, /^server-azure-/], workflows: [ /^workflow-contract-/, /^control-/, /^campaign-/, /^contoso-/, - /^github-apps\./, /^gh-aw-version-/, /^release-workflow\./, /^build-release\./, + /^github-apps\./, /^gh-aw-version-/, /^release-workflow\./, /^release-schedule\./, /^build-release\./, /^add-operational-value-skill\./, /^token-intervention-lifecycle\./, ], tooling: [ diff --git a/tests/integration/campaign-lifecycle.test.mjs b/tests/integration/campaign-lifecycle.test.mjs index 3cbaf28128..ac3e43e3fd 100644 --- a/tests/integration/campaign-lifecycle.test.mjs +++ b/tests/integration/campaign-lifecycle.test.mjs @@ -232,7 +232,7 @@ function assertInstalledBootstrap(repository, ghAwVersion, controlRepository) { const policy = ".github/workflows/cao.json"; const control = ".github/workflows/shared/control.mjs"; run(process.execPath, ["activity/cao.mjs", "--help"], repository); - for (const bundle of ["activity", "dashboard"]) { + for (const bundle of ["dashboard"]) { run(process.execPath, [".github/workflows/shared/materialize-cao.mjs", "verify", bundle], repository); } run(process.execPath, [control, "validate-policy", policy], repository); diff --git a/tests/integration/control-inventory-credentials.test.mjs b/tests/integration/control-inventory-credentials.test.mjs index e6b5647611..0a08ea4e8b 100644 --- a/tests/integration/control-inventory-credentials.test.mjs +++ b/tests/integration/control-inventory-credentials.test.mjs @@ -574,7 +574,7 @@ test("credential verification is opt-in and cannot reach the validation issue re assert.equal(workflow.on.workflow_dispatch.inputs["verify-credentials"].default, false); assert.match(workflow.jobs.credentials.if, /github.event_name == 'workflow_dispatch'.*inputs.verify-credentials == true/); assert.ok(Object.values(workflow.jobs.credentials.permissions).every(permission => permission === "read")); - assert.match(workflow.jobs.report.if, /inputs.verify-credentials != true/); + assert.equal(workflow.jobs.report, undefined); const token = steps.find(step => step.id === "cao_pre_activation_app_token"); assert.equal(token.with.repositories, "${{ github.event.repository.name }}"); assert.match(token.if, /cao_admission.outputs.authorized == 'true'/); diff --git a/tests/integration/dashboard-inventory-ingestion.test.mjs b/tests/integration/dashboard-inventory-ingestion.test.mjs index 034511ce74..cab23b0af7 100644 --- a/tests/integration/dashboard-inventory-ingestion.test.mjs +++ b/tests/integration/dashboard-inventory-ingestion.test.mjs @@ -3,7 +3,6 @@ import test from "node:test"; import "fake-indexeddb/auto"; import { buildInventoryDashboardSources, - discoverRepositories, } from "../../dashboard/report/inventory-sources.mjs"; import { ingestDashboardSources } from "../../dashboard/site/src/data/ingest/coordinator.js"; import { @@ -23,18 +22,11 @@ test("discovered repositories flow from control scope into canonical storage", a "control-plane": { inventory: { "max-scan-repositories": 100 } }, }, }; - const discoveredRepositories = await discoverRepositories(controlSettings, { - token: "test-token", - controlRepository: "acme/control", - fetchImplementation: async (url) => { - assert.match(String(url), /\/orgs\/acme\/repos\?/); - return new Response(JSON.stringify([ - { full_name: "acme/control", visibility: "private" }, - { full_name: "acme/payments", private: true }, - { full_name: "acme/storefront", visibility: "internal" }, - ])); - }, - }); + const discoveredRepositories = [ + { full_name: "acme/control", visibility: "private" }, + { full_name: "acme/payments", private: true }, + { full_name: "acme/storefront", visibility: "internal" }, + ]; const sources = buildInventoryDashboardSources({ repository: "acme/control", generatedAt, @@ -81,18 +73,10 @@ test("explicitly allowed repositories flow into canonical storage", async () => "control-plane": { inventory: { "max-scan-repositories": 100 } }, }, }; - const discoveredRepositories = await discoverRepositories(controlSettings, { - token: "test-token", - controlRepository: "acme/control", - fetchImplementation: async (url) => { - const repository = String(url).match(/\/repos\/([^/]+\/[^/?]+)/)?.[1]; - assert.ok(repository); - return new Response(JSON.stringify({ - full_name: repository, - visibility: repository.endsWith("payments") ? "private" : "internal", - })); - }, - }); + const discoveredRepositories = [ + { full_name: "acme/payments", visibility: "private" }, + { full_name: "acme/storefront", visibility: "internal" }, + ]; const sources = buildInventoryDashboardSources({ repository: "acme/control", generatedAt, diff --git a/tests/integration/install-script.test.mjs b/tests/integration/install-script.test.mjs index 3c60da4e08..7fe1ff4b35 100644 --- a/tests/integration/install-script.test.mjs +++ b/tests/integration/install-script.test.mjs @@ -206,7 +206,7 @@ exec "$CAO_INSTALLER_GH" "$@" assert.equal(await readFile(path.join(home, ".curlrc"), "utf8"), curlrc); assert.match(await readFile(log, "utf8"), /auth status\n[\s\S]*repo view --json nameWithOwner/); assert.deepEqual((await readFile(metadataLog, "utf8")).trim().split("\n"), Object.keys(responses)); - for (const bundle of ["activity", "dashboard"]) { + for (const bundle of ["dashboard"]) { await run(process.execPath, [".github/workflows/shared/materialize-cao.mjs", "verify", bundle], { cwd: consumer, env }); } }); diff --git a/tests/unit/workflow-contract-ci-triggers.test.mjs b/tests/unit/workflow-contract-ci-triggers.test.mjs index 6213837637..50dd20db3c 100644 --- a/tests/unit/workflow-contract-ci-triggers.test.mjs +++ b/tests/unit/workflow-contract-ci-triggers.test.mjs @@ -513,7 +513,7 @@ test("mixed workflows keep fast checks on PRs and gate expensive jobs off PRs", const cases = [ ["workflow-contracts.yml", ["unit", "test"], [ "agent-plugin", "installer", - "full-contracts", "github-issue-query", + "full-contracts", "dependabot-operational-value", "campaign-lifecycle-changes", ]], ["cgo.yml", ["server-quality", "server-spec"], [ From a5736579b9936a6eda24b01589ef824755d13e77 Mon Sep 17 00:00:00 2001 From: Peli de Halleux <4175913+pelikhan@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:34:59 -0800 Subject: [PATCH 2/7] Migrate campaign lifecycle test off the removed activity runtime Use control.mjs control-settings instead of activity/control-settings.mjs, drop the activity/cao.mjs smoke check, and assert only surviving runtime files. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/integration/campaign-lifecycle.test.mjs | 61 ++++--------------- 1 file changed, 12 insertions(+), 49 deletions(-) diff --git a/tests/integration/campaign-lifecycle.test.mjs b/tests/integration/campaign-lifecycle.test.mjs index ac3e43e3fd..8f85b5d15e 100644 --- a/tests/integration/campaign-lifecycle.test.mjs +++ b/tests/integration/campaign-lifecycle.test.mjs @@ -53,39 +53,13 @@ const optimizationCampaignSource = focusedCampaignSource("optimization"); const repoStatusWorkflowSource = `https://github.com/${campaignSource.slice(0, campaignSource.lastIndexOf("@"))}/blob/${campaignSource.slice(campaignSource.lastIndexOf("@") + 1)}/.github/workflows/daily-repo-status.md`; const selfCareCampaignSource = focusedCampaignSource("self-care"); const softwareDevelopmentPracticesCampaignSource = focusedCampaignSource("software-development-practices"); -const activityExpectedFiles = [ - "activity/actions-context.mjs", +const runtimeExpectedFiles = [ ".github/workflows/shared/actions-log.mjs", - "activity/campaign-intelligence.mjs", - "activity/cao.mjs", "dashboard/report/collections.mjs", - "activity/commands/audit-jsonl.mjs", - "activity/commands/cluster-problems.mjs", - "activity/commands/compact-jsonl.mjs", - "activity/commands/dashboard-complexity.mjs", - "activity/commands/discover-workflows.mjs", - "activity/commands/repair-database.mjs", - "activity/commands/download.mjs", - "activity/commands/hash-payloads.mjs", - "activity/commands/index.mjs", - "activity/commands/ingest-jsonl.mjs", - "activity/commands/ingest.mjs", - "activity/commands/operational-value.mjs", - "activity/commands/prune-dashboard.mjs", - "activity/collect-workflow-logs.mjs", - "activity/computations/intelligence-contracts.mjs", - "activity/computations/intelligence.mjs", - "activity/computations/index.mjs", - "activity/computations/runtime-health.mjs", - "activity/control-settings.mjs", "dashboard/report/debug.mjs", - "activity/gh-aw-logs.mjs", - "activity/inventory.mjs", "dashboard/report/inventory-sources.mjs", "dashboard/report/package-manifest.mjs", "dashboard/report/normalized-phase.mjs", - "activity/repository-visibility.mjs", - "activity/token-intervention-lifecycle.mjs", "dashboard/lib/version.mjs", ]; const caoEvolutionExpectedFiles = [ @@ -231,28 +205,17 @@ async function installCampaign(source) { function assertInstalledBootstrap(repository, ghAwVersion, controlRepository) { const policy = ".github/workflows/cao.json"; const control = ".github/workflows/shared/control.mjs"; - run(process.execPath, ["activity/cao.mjs", "--help"], repository); - for (const bundle of ["dashboard"]) { - run(process.execPath, [".github/workflows/shared/materialize-cao.mjs", "verify", bundle], repository); - } + run(process.execPath, [".github/workflows/shared/materialize-cao.mjs", "verify", "dashboard"], repository); run(process.execPath, [control, "validate-policy", policy], repository); assert.equal(run(process.execPath, [control, "compiler-version", policy], repository).trim(), ghAwVersion); - const settingsDirectory = mkdtempSync(join(tmpdir(), "central-agentic-ops-settings-")); - try { - const settingsPath = join(settingsDirectory, "control-settings.json"); - execFileSync(process.execPath, ["activity/control-settings.mjs", control, policy, settingsPath], { - cwd: repository, - encoding: "utf8", - env: { ...process.env, GITHUB_REPOSITORY: controlRepository }, - stdio: ["ignore", "pipe", "pipe"], - }); - const settings = JSON.parse(readFileSync(settingsPath, "utf8")); - assert.equal(settings.policy_resolution.status, "available", settings.policy_resolution.reason); - assert.deepEqual(settings.allowed_repositories, [controlRepository]); - } finally { - rmSync(settingsDirectory, { recursive: true, force: true }); - } + const settings = JSON.parse(execFileSync(process.execPath, [control, "control-settings", policy], { + cwd: repository, + encoding: "utf8", + env: { ...process.env, GITHUB_REPOSITORY: controlRepository }, + stdio: ["ignore", "pipe", "pipe"], + })); + assert.deepEqual(settings.allowed_repositories, [controlRepository]); } test("root campaign bootstraps an empty CAO and preserves resources during workflow update", { timeout: 240_000 }, async () => { @@ -307,7 +270,7 @@ test("root campaign bootstraps an empty CAO and preserves resources during workf }); assertInstalledBootstrap(consumer, ghAwVersion, controlRepository); - run("git", ["add", ".github", "activity", "dashboard"], consumer); + run("git", ["add", ".github", "dashboard"], consumer); run("git", [ "-c", "user.name=CAO installer test", "-c", "user.email=cao-installer-test@example.invalid", @@ -344,8 +307,8 @@ test("root campaign bootstraps an empty CAO and preserves resources during workf readdirSync(join(consumer, ".github", "workflows")).filter((name) => name.endsWith(".lock.yml")), [], ); - for (const relativePath of activityExpectedFiles) { - assert.ok(existsSync(join(consumer, relativePath)), `root campaign omitted activity file ${relativePath}`); + for (const relativePath of runtimeExpectedFiles) { + assert.ok(existsSync(join(consumer, relativePath)), `root campaign omitted runtime file ${relativePath}`); } for (const relativePath of dashboardExpectedFiles) { assert.ok(existsSync(join(consumer, relativePath)), `root campaign omitted dashboard file ${relativePath}`); From a85d9ab56d713ba9c8d4d11a97537d866a884154 Mon Sep 17 00:00:00 2001 From: Peli de Halleux <4175913+pelikhan@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:49:46 -0800 Subject: [PATCH 3/7] Remove remaining activity/cao.mjs hash-payloads uses The massive-scale stress test now normalizes synthetic shards in-process through dashboard/report/normalized-payloads.mjs, the production module the retired command wrapped. Also import the stat helper the test already used, and drop the stale hash-payloads step from the Value event-ledger reference. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../references/event-ledger.md | 4 ---- tests/e2e/dashboard-massive-scale.spec.mjs | 18 +++++++----------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/.github/skills/add-operational-value/references/event-ledger.md b/.github/skills/add-operational-value/references/event-ledger.md index f8a416ff2c..268f4dd8e0 100644 --- a/.github/skills/add-operational-value/references/event-ledger.md +++ b/.github/skills/add-operational-value/references/event-ledger.md @@ -214,10 +214,6 @@ original ledger, not the computed pulse model, and makes no remote requests: ```bash node .github/skills/add-operational-value/scripts/export-evidence.mjs \ ledger.json .cao/gh-aw-logs-shards/workflow-value-evidence.jsonl -node activity/cao.mjs hash-payloads \ - --shard-dir .cao/gh-aw-logs-shards --normalized-dir .cao/gh-aw-logs-normalized \ - --runs-dir .cao/gh-aw-logs-runs --records-dir .cao/gh-aw-logs-records \ - --output .cao/payload-hashes.json ``` Use an existing, appropriately scoped publication directory. Import is not diff --git a/tests/e2e/dashboard-massive-scale.spec.mjs b/tests/e2e/dashboard-massive-scale.spec.mjs index 4a9d9a66b6..a5532f2a28 100644 --- a/tests/e2e/dashboard-massive-scale.spec.mjs +++ b/tests/e2e/dashboard-massive-scale.spec.mjs @@ -1,11 +1,12 @@ import { expect, test } from "@playwright/test"; import { spawn } from "node:child_process"; import { createReadStream, existsSync, statSync } from "node:fs"; -import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; import { createServer } from "node:http"; import { tmpdir } from "node:os"; import { extname, join, resolve, sep } from "node:path"; import { once } from "node:events"; +import { normalizeSnapshotPayloads } from "../../dashboard/report/normalized-payloads.mjs"; const siteRoot = resolve("dashboard/site"); const repositories = numberSetting("DASHBOARD_STRESS_REPOSITORIES", 10_000); @@ -115,7 +116,6 @@ test.beforeAll(async () => { rawShardDirectory = join(root, "gh-aw-logs-shards"); const runsDirectory = join(root, "gh-aw-logs-runs"); const recordsDirectory = join(root, "gh-aw-logs-records"); - const manifestPath = join(root, "payload-hashes.json"); generationMemory = await runMeasuredCommand(process.execPath, [ resolve("tests/helpers/dashboard-stress-data.mjs"), "--output", rawShardDirectory, @@ -126,15 +126,11 @@ test.beforeAll(async () => { "--workflows", String(workflows), ]); manifest = JSON.parse(generationMemory.stdout); - await runMeasuredCommand(process.execPath, [ - resolve("activity/cao.mjs"), - "hash-payloads", - "--shard-dir", rawShardDirectory, - "--runs-dir", runsDirectory, - "--records-dir", recordsDirectory, - "--output", manifestPath, - ]); - payloadHashes = JSON.parse(await readFile(manifestPath, "utf8")); + payloadHashes = await normalizeSnapshotPayloads({ + shardDirectory: rawShardDirectory, + runsDirectory, + recordsDirectory, + }); server = createServer(async (request, response) => { try { From 08723524eef48736f327138e543d606b4636a747 Mon Sep 17 00:00:00 2001 From: Peli de Halleux <4175913+pelikhan@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:56:20 -0800 Subject: [PATCH 4/7] Fix timestamptz precision in native Value hint retry_at PostgreSQL timestamptz stores microseconds and rounds, so a nanosecond retry time (Linux clocks) never round-trips. The fencing test failed deterministically on CI comparing RetryAt; fencing itself was correct. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- server/internal/postgresx/acquisition_value_hints.go | 3 ++- server/internal/postgresx/acquisition_value_hints_test.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/server/internal/postgresx/acquisition_value_hints.go b/server/internal/postgresx/acquisition_value_hints.go index df2e75a1ac..a656536a56 100644 --- a/server/internal/postgresx/acquisition_value_hints.go +++ b/server/internal/postgresx/acquisition_value_hints.go @@ -93,7 +93,8 @@ func (a Acquisition) CompleteValueHint(ctx context.Context, scope collectnative. } var retryAt any if !retry.IsZero() { - retryAt = retry.UTC() + // timestamptz stores microseconds and rounds; truncate so retry_at round-trips exactly. + retryAt = retry.UTC().Truncate(time.Microsecond) } result, err := tx.ExecContext(ctx, `UPDATE cao_acquisition_value_hints SET processed=CASE WHEN $6::timestamptz IS NULL THEN greatest(processed,$4) ELSE processed END, diff --git a/server/internal/postgresx/acquisition_value_hints_test.go b/server/internal/postgresx/acquisition_value_hints_test.go index 4a677e649c..d7e5022ed9 100644 --- a/server/internal/postgresx/acquisition_value_hints_test.go +++ b/server/internal/postgresx/acquisition_value_hints_test.go @@ -47,7 +47,7 @@ func TestNativeValueHintsFenceConcurrentInvalidationAndAcknowledgement(t *testin if err := a.InvalidateValue(t.Context(), scope); err != nil { t.Fatal(err) } - now := time.Now().UTC() + now := time.Now().UTC().Truncate(time.Microsecond) if err := b.CompleteValueHint(t.Context(), scope, hint.Dirty, now, time.Time{}); err != nil { t.Fatal(err) } From efba01b55a495a46afd57d96d8f7d86cb1bf99cc Mon Sep 17 00:00:00 2001 From: Peli de Halleux <4175913+pelikhan@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:59:38 -0800 Subject: [PATCH 5/7] Clone per-test Postgres databases from a provisioned template; trim redundant fixtures Since the SQLite removal, every testutil.Postgres fixture provisions a full canonical store (~360 daily partitions, ~5s), pushing internal/server past the 10m go test timeout. Provision one template database per test process through the production store and clone it per test (CREATE DATABASE ... TEMPLATE), falling back to a private schema when the role cannot create databases. Reclaim databases left by exited processes. Also share one app across TestSessionArchiveResponseClosesReader subtests and stop re-executing repository queries already covered by the full dashboard corpus test. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../collect/native_control_process_test.go | 4 +- .../collect/run_enumeration_process_test.go | 3 +- server/internal/collect/run_worker_test.go | 4 +- server/internal/postgresx/native_test.go | 15 ++ .../server/dashboard_sql_corpus_test.go | 30 ++- .../internal/server/session_archive_test.go | 6 +- .../testutil/pgtemplate/pgtemplate.go | 208 ++++++++++++++++++ server/internal/testutil/postgres.go | 41 +++- 8 files changed, 287 insertions(+), 24 deletions(-) create mode 100644 server/internal/testutil/pgtemplate/pgtemplate.go diff --git a/server/internal/collect/native_control_process_test.go b/server/internal/collect/native_control_process_test.go index 31e906334e..297b985331 100644 --- a/server/internal/collect/native_control_process_test.go +++ b/server/internal/collect/native_control_process_test.go @@ -38,7 +38,7 @@ func TestNativeControlRecoveryUsesSeparateProcessAndDurableValueHint(t *testing. // #nosec G204 -- invokes this test executable without external collector tooling. child := exec.CommandContext(t.Context(), executable, "-test.run=^TestNativeControlRecoverySubprocess$", "-test.v") child.Env = append(os.Environ(), "PATH=/nonexistent", "CAO_NATIVE_CONTROL_DSN="+dsn, - "CAO_NATIVE_CONTROL_NAMESPACE="+namespace, "CAO_NATIVE_CONTROL_SCHEMA="+config.RuntimeParams["search_path"]) + "CAO_NATIVE_CONTROL_NAMESPACE="+namespace, "CAO_NATIVE_CONTROL_SCHEMA="+config.RuntimeParams["search_path"], "CAO_NATIVE_CONTROL_CANONICAL_DSN="+config.ConnString()) if output, err := child.CombinedOutput(); err != nil { t.Fatalf("separate native control process failed: %v\n%s", err, output) } @@ -68,7 +68,7 @@ func TestNativeControlRecoverySubprocess(t *testing.T) { t.Fatal(err) } defer func() { _ = store.Close() }() - config, err := pgx.ParseConfig(dsn) + config, err := pgx.ParseConfig(os.Getenv("CAO_NATIVE_CONTROL_CANONICAL_DSN")) if err != nil { t.Fatal(err) } diff --git a/server/internal/collect/run_enumeration_process_test.go b/server/internal/collect/run_enumeration_process_test.go index 5c7d77e25e..866bf280cd 100644 --- a/server/internal/collect/run_enumeration_process_test.go +++ b/server/internal/collect/run_enumeration_process_test.go @@ -40,6 +40,7 @@ func TestNativeEnumerationResumesAbandonedPageInSeparateProcess(t *testing.T) { child.Env = append(os.Environ(), "PATH=/nonexistent", "CAO_NATIVE_DISCOVERY_DSN="+dsn, "CAO_NATIVE_DISCOVERY_NAMESPACE="+namespace, "CAO_NATIVE_DISCOVERY_SCHEMA="+config.RuntimeParams["search_path"], + "CAO_NATIVE_DISCOVERY_CANONICAL_DSN="+config.ConnString(), "CAO_NATIVE_DISCOVERY_BEFORE="+before.Format(time.RFC3339)) if output, err := child.CombinedOutput(); err != nil { t.Fatalf("separate native enumeration process failed: %v\n%s", err, output) @@ -67,7 +68,7 @@ func TestNativeEnumerationRecoverySubprocess(t *testing.T) { t.Fatal(err) } defer func() { _ = store.Close() }() - config, err := pgx.ParseConfig(dsn) + config, err := pgx.ParseConfig(os.Getenv("CAO_NATIVE_DISCOVERY_CANONICAL_DSN")) if err != nil { t.Fatal(err) } diff --git a/server/internal/collect/run_worker_test.go b/server/internal/collect/run_worker_test.go index b8249a188b..a37c7dbd08 100644 --- a/server/internal/collect/run_worker_test.go +++ b/server/internal/collect/run_worker_test.go @@ -149,7 +149,7 @@ func TestNativeRunWorkerCrashAfterPublicationRecoversInAnotherProcess(t *testing child := exec.CommandContext(t.Context(), executable, "-test.run=^TestNativeRunWorkerRecoverySubprocess$", "-test.v") child.Env = []string{ "PATH=/nonexistent", "CAO_NATIVE_RUN_DSN=" + dsn, "CAO_NATIVE_RUN_NAMESPACE=" + namespace, - "CAO_NATIVE_RUN_SCHEMA=" + config.RuntimeParams["search_path"], "CAO_NATIVE_RUN_RECORDED=" + recorded, + "CAO_NATIVE_RUN_SCHEMA=" + config.RuntimeParams["search_path"], "CAO_NATIVE_RUN_CANONICAL_DSN=" + config.ConnString(), "CAO_NATIVE_RUN_RECORDED=" + recorded, } output, err := child.CombinedOutput() if err != nil || !strings.Contains(string(output), "historical durable handoff verified") { @@ -187,7 +187,7 @@ func TestNativeRunWorkerRecoverySubprocess(t *testing.T) { if err != nil || len(leases) != 1 || leases[0].Task.Attempt != 3 || leases[0].Task.Failures != 0 { t.Fatalf("recovered exact historical attempt: %v %v", leases, err) } - config, err := pgx.ParseConfig(dsn) + config, err := pgx.ParseConfig(os.Getenv("CAO_NATIVE_RUN_CANONICAL_DSN")) if err != nil { t.Fatal(err) } diff --git a/server/internal/postgresx/native_test.go b/server/internal/postgresx/native_test.go index 0c2a94a5c9..3e84dd6364 100644 --- a/server/internal/postgresx/native_test.go +++ b/server/internal/postgresx/native_test.go @@ -15,6 +15,7 @@ import ( "github.com/githubnext/gh-cao/server/internal/model" "github.com/githubnext/gh-cao/server/internal/query" + "github.com/githubnext/gh-cao/server/internal/testutil/pgtemplate" ) func nativeTestStore(t *testing.T) (*Store, *pgx.ConnConfig) { @@ -23,6 +24,20 @@ func nativeTestStore(t *testing.T) (*Store, *pgx.ConnConfig) { if endpoint == "" { t.Skip("POSTGRES_URL is unset") } + if config, ok := pgtemplate.Clone(t, t.Context(), endpoint, func(ctx context.Context, config *pgx.ConnConfig) error { + store, err := NewConfig(ctx, config) + if err != nil { + return err + } + return store.Close() + }); ok { + store, err := NewConfig(t.Context(), config) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + return store, config + } config, err := pgx.ParseConfig(endpoint) if err != nil { t.Fatal(err) diff --git a/server/internal/server/dashboard_sql_corpus_test.go b/server/internal/server/dashboard_sql_corpus_test.go index 2a1c11a694..2bf55716eb 100644 --- a/server/internal/server/dashboard_sql_corpus_test.go +++ b/server/internal/server/dashboard_sql_corpus_test.go @@ -16,19 +16,33 @@ import ( ) func TestDashboardQueryCorpusUsesPostgres(t *testing.T) { - assertDashboardQueriesUsePostgres(t, nil) + assertDashboardQueriesUsePostgres(t) } +// TestRepositoryQueriesUsePostgres guards that repository queries stay in the +// deployed corpus, which TestDashboardQueryCorpusUsesPostgres executes in full. func TestRepositoryQueriesUsePostgres(t *testing.T) { - assertDashboardQueriesUsePostgres(t, []string{ + definitions, _, err := dashboardSQLCorpus() + if err != nil { + t.Fatal(err) + } + deployed := make(map[string]bool, len(definitions)) + for _, definition := range definitions { + deployed[definition.Name] = true + } + for _, name := range []string{ "repository-run-statistics", "repository-run-identities", "repository-import-totals", "repository-run-totals", "repository-aic-totals", "repository-activity", "repository-observed-run-counts", "repository-run-inventory", - }) + } { + if !deployed[name] { + t.Errorf("repository query %q is missing from the deployed dashboard corpus", name) + } + } } -func assertDashboardQueriesUsePostgres(t *testing.T, requested []string) { +func assertDashboardQueriesUsePostgres(t *testing.T) { t.Helper() store := integrationDatabase(t) definitions, parameters, err := dashboardSQLCorpus() @@ -41,11 +55,9 @@ func assertDashboardQueriesUsePostgres(t *testing.T, requested []string) { if err := query.Validate(definitions); err != nil { t.Fatalf("validate deployed dashboard query corpus: %v", err) } - if requested == nil { - requested = make([]string, len(definitions)) - for index, definition := range definitions { - requested[index] = definition.Name - } + requested := make([]string, len(definitions)) + for index, definition := range definitions { + requested[index] = definition.Name } runtimeSource := func(_ context.Context, name string) (model.Source, error) { return model.Source{ diff --git a/server/internal/server/session_archive_test.go b/server/internal/server/session_archive_test.go index 0f97a30260..bc0b194209 100644 --- a/server/internal/server/session_archive_test.go +++ b/server/internal/server/session_archive_test.go @@ -223,9 +223,13 @@ func TestSessionArchiveResultBudgetsErrorsAndRevocation(t *testing.T) { } func TestSessionArchiveResponseClosesReader(t *testing.T) { + // Outcomes share one app; each subtest restores the credential it may revoke + // and installs its own archive service. + app, _ := newLocalPostgresTestApp(t, false) + accessToken := app.accessToken for _, outcome := range []string{"success", "withdrawn", "oversize", "authorization-revoked", "authorization-revoked-at-delivery", "query-error"} { t.Run(outcome, func(t *testing.T) { - app, _ := newLocalPostgresTestApp(t, false) + app.accessToken = accessToken reader := &sessionArchiveReadProbe{} delivered := false reader.deliver = func(deliver func() error) error { diff --git a/server/internal/testutil/pgtemplate/pgtemplate.go b/server/internal/testutil/pgtemplate/pgtemplate.go new file mode 100644 index 0000000000..27eb37696c --- /dev/null +++ b/server/internal/testutil/pgtemplate/pgtemplate.go @@ -0,0 +1,208 @@ +// Package pgtemplate clones per-test PostgreSQL databases from a template +// provisioned once per test process. +// +// Provisioning a canonical store creates hundreds of daily partitions, which +// costs seconds per test. Cloning a provisioned template costs a fraction of a +// second while still giving every test a private database. The package has no +// store dependency so the store's own package tests can use it. +// +// Clones live in separate databases, so packages tested concurrently against one +// server no longer serialize on the store's database-scoped advisory locks. +// Dashboard queries lock every daily partition, so concurrent package runs +// (go test -p N) need a larger max_locks_per_transaction, for example 512; +// CI runs Postgres packages with -p 1. +package pgtemplate + +import ( + "context" + "database/sql" + "errors" + "fmt" + "hash/fnv" + "net/url" + "os" + "strings" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/jackc/pgx/v5" + _ "github.com/jackc/pgx/v5/stdlib" // registers the "pgx" database/sql driver +) + +// Schema is the search_path of every template and clone. +const Schema = "cao_test" + +// Provision initializes the template database through production code. It +// receives a configuration whose search_path is Schema and must release every +// connection before returning. +type Provision func(context.Context, *pgx.ConnConfig) error + +type template struct { + name string + unsupported bool +} + +var ( + mu sync.Mutex + templates = map[string]*template{} + sequence atomic.Uint64 +) + +// Clone returns a configuration for a private database cloned from this +// process's template for endpoint, provisioning the template on first use. It +// returns false when the configured role cannot create databases. The clone is +// dropped when the test ends. +func Clone(t testing.TB, parent context.Context, endpoint string, provision Provision) (*pgx.ConnConfig, bool) { + t.Helper() + admin, err := sql.Open("pgx", endpoint) + if err != nil { + t.Fatal("configure Postgres test connection") + } + t.Cleanup(func() { _ = admin.Close() }) + mu.Lock() + defer mu.Unlock() + current := templates[endpoint] + if current == nil { + current = provisionTemplate(t, parent, admin, endpoint, provision) + templates[endpoint] = current + } + if current.unsupported { + return nil, false + } + ctx, cancel := context.WithTimeout(parent, 30*time.Second) + defer cancel() + name := fmt.Sprintf("%s_%d", processPrefix, sequence.Add(1)) + if _, err := admin.ExecContext(ctx, "CREATE DATABASE "+pgx.Identifier{name}.Sanitize()+" TEMPLATE "+pgx.Identifier{current.name}.Sanitize()); err != nil { + t.Fatalf("clone Postgres test template %s: %v", current.name, err) + } + t.Cleanup(func() { + cleanup, stop := context.WithTimeout(context.WithoutCancel(parent), 30*time.Second) + defer stop() + if _, err := admin.ExecContext(cleanup, "DROP DATABASE IF EXISTS "+pgx.Identifier{name}.Sanitize()+" WITH (FORCE)"); err != nil { + t.Errorf("remove isolated Postgres test database %s: %v", name, err) + } + }) + return DatabaseConfig(t, endpoint, name), true +} + +func provisionTemplate(t testing.TB, parent context.Context, admin *sql.DB, endpoint string, provision Provision) *template { + t.Helper() + ctx, cancel := context.WithTimeout(context.WithoutCancel(parent), 2*time.Minute) + defer cancel() + started := time.Now() + removeStale(t, ctx, admin) + name := fmt.Sprintf("%s_tmpl", processPrefix) + if _, err := admin.ExecContext(ctx, "CREATE DATABASE "+pgx.Identifier{name}.Sanitize()); err != nil { + t.Logf("Postgres test role cannot create databases (%v); provisioning a private schema per test", err) + return &template{unsupported: true} + } + config := DatabaseConfig(t, endpoint, name) + connection, err := pgx.ConnectConfig(ctx, config.Copy()) + if err == nil { + _, err = connection.Exec(ctx, "CREATE SCHEMA "+Schema) + _ = connection.Close(ctx) + } + if err == nil { + err = provision(ctx, config) + } + if err == nil { + _, err = admin.ExecContext(ctx, "ALTER DATABASE "+pgx.Identifier{name}.Sanitize()+" WITH IS_TEMPLATE true ALLOW_CONNECTIONS false") + } + if err != nil { + drop(ctx, admin, name) + t.Fatalf("provision Postgres test template: %v", err) + } + t.Logf("provisioned Postgres test template %s in %s", name, time.Since(started).Round(time.Millisecond)) + return &template{name: name} +} + +// Database names identify the owning host and process so later processes can +// reclaim databases left by interrupted runs: cao_test____…. +var ( + hostTag = hostHash() + processPrefix = fmt.Sprintf("cao_test_%s_%d_%d", hostTag, os.Getpid(), time.Now().Unix()) +) + +func hostHash() string { + host, _ := os.Hostname() + digest := fnv.New32a() + _, _ = digest.Write([]byte(host)) + return fmt.Sprintf("%08x", digest.Sum32()) +} + +// removeStale drops templates and clones whose owning process on this host has +// exited, and any older than a day. Live processes' databases stay intact. +func removeStale(t testing.TB, ctx context.Context, admin *sql.DB) { + t.Helper() + rows, err := admin.QueryContext(ctx, `SELECT datname, datistemplate FROM pg_database WHERE datname LIKE 'cao\_test\_%'`) + if err != nil { + return + } + type candidate struct { + name string + template bool + } + var stale []candidate + cutoff := time.Now().Add(-24 * time.Hour).Unix() + for rows.Next() { + var current candidate + if rows.Scan(¤t.name, ¤t.template) != nil { + break + } + var host string + var pid int + var started int64 + if _, err := fmt.Sscanf(strings.ReplaceAll(current.name, "_", " "), "cao test %s %d %d", &host, &pid, &started); err != nil { + continue + } + if started < cutoff || (host == hostTag && pid != os.Getpid() && !processAlive(pid)) { + stale = append(stale, current) + } + } + _ = rows.Close() + for _, current := range stale { + if current.template { + if _, err := admin.ExecContext(ctx, "ALTER DATABASE "+pgx.Identifier{current.name}.Sanitize()+" WITH IS_TEMPLATE false"); err != nil { + continue + } + } + drop(ctx, admin, current.name) + t.Logf("removed stale Postgres test database %s", current.name) + } +} + +func processAlive(pid int) bool { + process, err := os.FindProcess(pid) + if err != nil { + return false + } + // Treat unknown outcomes, such as unsupported signals, as alive. + err = process.Signal(syscall.Signal(0)) + return !errors.Is(err, os.ErrProcessDone) && !errors.Is(err, syscall.ESRCH) +} + +func drop(ctx context.Context, admin *sql.DB, name string) { + _, _ = admin.ExecContext(ctx, "DROP DATABASE IF EXISTS "+pgx.Identifier{name}.Sanitize()+" WITH (FORCE)") +} + +// DatabaseConfig targets database name with search_path Schema while +// preserving every other endpoint setting, so ConnString identifies the clone +// for subprocesses and adapters. +func DatabaseConfig(t testing.TB, endpoint, name string) *pgx.ConnConfig { + t.Helper() + target := endpoint + " dbname='" + name + "'" + if parsed, err := url.Parse(endpoint); err == nil && (parsed.Scheme == "postgres" || parsed.Scheme == "postgresql") { + parsed.Path = "/" + name + parsed.RawPath = "" + target = parsed.String() + } + config, err := pgx.ParseConfig(target) + if err != nil { + t.Fatal("parse Postgres test configuration") + } + config.RuntimeParams["search_path"] = Schema + return config +} diff --git a/server/internal/testutil/postgres.go b/server/internal/testutil/postgres.go index 022e656519..be30e393ac 100644 --- a/server/internal/testutil/postgres.go +++ b/server/internal/testutil/postgres.go @@ -12,6 +12,7 @@ import ( "github.com/jackc/pgx/v5" "github.com/githubnext/gh-cao/server/internal/postgresx" + "github.com/githubnext/gh-cao/server/internal/testutil/pgtemplate" ) func ConnectionString(t testing.TB, config *pgx.ConnConfig) string { @@ -28,16 +29,20 @@ func ConnectionString(t testing.TB, config *pgx.ConnConfig) string { return endpoint.String() } -// Postgres creates a private schema, including when initialization fails. -// Service-backed tests never replace another test's or the operator's data. +// Postgres creates a private database or schema, including when initialization +// fails. Service-backed tests never replace another test's or the operator's data. func Postgres(t testing.TB, parent context.Context, environment ...string) *postgresx.Store { t.Helper() store, _ := PostgresWithConfig(t, parent, environment...) return store } -// PostgresWithConfig also exposes the isolated schema connection for operational +// PostgresWithConfig also exposes the isolated connection for operational // adapters that need their own tables in the same disposable database. +// +// Each test receives a database cloned from a per-process template provisioned +// through the production store; see pgtemplate. Roles that cannot create +// databases fall back to a private schema in the named database. func PostgresWithConfig(t testing.TB, parent context.Context, environment ...string) (*postgresx.Store, *pgx.ConnConfig) { t.Helper() endpoint := "" @@ -49,13 +54,36 @@ func PostgresWithConfig(t testing.TB, parent context.Context, environment ...str if endpoint == "" { t.Skipf("set one of %v to run Postgres integration tests", environment) } + config, ok := pgtemplate.Clone(t, parent, endpoint, func(ctx context.Context, config *pgx.ConnConfig) error { + store, err := postgresx.NewConfig(ctx, config) + if err != nil { + return err + } + return store.Close() + }) + if !ok { + config = isolatedSchema(t, parent, endpoint) + } ctx, cancel := context.WithTimeout(parent, 30*time.Second) defer cancel() + store, err := postgresx.NewConfig(ctx, config) + if err != nil { + t.Fatal("initialize isolated Postgres test store") + } + t.Cleanup(func() { _ = store.Close() }) + return store, config +} + +// isolatedSchema provisions a private schema in the configured database. +func isolatedSchema(t testing.TB, parent context.Context, endpoint string) *pgx.ConnConfig { + t.Helper() admin, err := sql.Open("pgx", endpoint) if err != nil { t.Fatal("configure Postgres test connection") } t.Cleanup(func() { _ = admin.Close() }) + ctx, cancel := context.WithTimeout(parent, 30*time.Second) + defer cancel() schema := fmt.Sprintf("cao_test_%d", time.Now().UnixNano()) if _, err := admin.ExecContext(ctx, "CREATE SCHEMA "+schema); err != nil { t.Fatal("create isolated Postgres test schema") @@ -104,10 +132,5 @@ func PostgresWithConfig(t testing.TB, parent context.Context, environment ...str t.Fatal("parse Postgres test configuration") } config.RuntimeParams["search_path"] = schema - store, err := postgresx.NewConfig(ctx, config) - if err != nil { - t.Fatal("initialize isolated Postgres test schema") - } - t.Cleanup(func() { _ = store.Close() }) - return store, config + return config } From bce8b2d2aa1a84453491f78c95e24b3821f3f983 Mon Sep 17 00:00:00 2001 From: Peli de Halleux <4175913+pelikhan@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:11:52 -0800 Subject: [PATCH 6/7] Report per-test durations from Go unit test shards The storage shard timeout log showed only the test running at the 10m deadline, not which tests consumed the budget. Run Go unit tests with -v so every shard log records per-test durations and the next slow shard can be attributed directly. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/cgo.yml | 4 ++-- tests/unit/cao-go-ci-plan.test.mjs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cgo.yml b/.github/workflows/cgo.yml index 88fd0bfef7..d4287df442 100644 --- a/.github/workflows/cgo.yml +++ b/.github/workflows/cgo.yml @@ -286,9 +286,9 @@ jobs: } }')" test -n "$pattern" - go -C server test -p 1 "$GO_TEST_MODE" -run "^($pattern)$" "${packages[@]}" + go -C server test -p 1 "$GO_TEST_MODE" -v -run "^($pattern)$" "${packages[@]}" else - go -C server test -p 1 "$GO_TEST_MODE" "${packages[@]}" + go -C server test -p 1 "$GO_TEST_MODE" -v "${packages[@]}" fi server-integration: diff --git a/tests/unit/cao-go-ci-plan.test.mjs b/tests/unit/cao-go-ci-plan.test.mjs index c92a2c682a..d78699c02a 100644 --- a/tests/unit/cao-go-ci-plan.test.mjs +++ b/tests/unit/cao-go-ci-plan.test.mjs @@ -88,7 +88,7 @@ test("Go unit jobs partition every package exactly once, including future packag const result = spawnSync("bash", ["-c", ` go() { if [[ "$3" == "list" ]]; then printf '%s\\n' "$MOCK_PACKAGES"; - else shift 6; printf '%s\\n' "$@"; fi + else shift 7; printf '%s\\n' "$@"; fi } ${step.run} `], { @@ -149,7 +149,7 @@ test("storage shards execute every discovered test exactly once with existing te else [[ "$*" == *" $GO_TEST_MODE "* ]] || return 42 [[ "\${!#}" == "./internal/postgresx" ]] || return 43 - printf '%s\\n' "$8" + printf '%s\\n' "$9" fi } `; From eee517feca1a9904b92d91f7d6be93aea1890b85 Mon Sep 17 00:00:00 2001 From: Peli de Halleux <4175913+pelikhan@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:30:04 -0800 Subject: [PATCH 7/7] Check rows.Err and defer Close in pgtemplate stale cleanup Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../testutil/pgtemplate/pgtemplate.go | 46 +++++++++++-------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/server/internal/testutil/pgtemplate/pgtemplate.go b/server/internal/testutil/pgtemplate/pgtemplate.go index 27eb37696c..937063ad6b 100644 --- a/server/internal/testutil/pgtemplate/pgtemplate.go +++ b/server/internal/testutil/pgtemplate/pgtemplate.go @@ -137,20 +137,39 @@ func hostHash() string { // exited, and any older than a day. Live processes' databases stay intact. func removeStale(t testing.TB, ctx context.Context, admin *sql.DB) { t.Helper() - rows, err := admin.QueryContext(ctx, `SELECT datname, datistemplate FROM pg_database WHERE datname LIKE 'cao\_test\_%'`) + stale, err := staleDatabases(ctx, admin) if err != nil { + t.Logf("skipped stale Postgres test database cleanup: %v", err) return } - type candidate struct { - name string - template bool + for _, current := range stale { + if current.template { + if _, err := admin.ExecContext(ctx, "ALTER DATABASE "+pgx.Identifier{current.name}.Sanitize()+" WITH IS_TEMPLATE false"); err != nil { + continue + } + } + drop(ctx, admin, current.name) + t.Logf("removed stale Postgres test database %s", current.name) } - var stale []candidate +} + +type staleCandidate struct { + name string + template bool +} + +func staleDatabases(ctx context.Context, admin *sql.DB) ([]staleCandidate, error) { + rows, err := admin.QueryContext(ctx, `SELECT datname, datistemplate FROM pg_database WHERE datname LIKE 'cao\_test\_%'`) + if err != nil { + return nil, err + } + defer rows.Close() + var stale []staleCandidate cutoff := time.Now().Add(-24 * time.Hour).Unix() for rows.Next() { - var current candidate - if rows.Scan(¤t.name, ¤t.template) != nil { - break + var current staleCandidate + if err := rows.Scan(¤t.name, ¤t.template); err != nil { + return nil, err } var host string var pid int @@ -162,16 +181,7 @@ func removeStale(t testing.TB, ctx context.Context, admin *sql.DB) { stale = append(stale, current) } } - _ = rows.Close() - for _, current := range stale { - if current.template { - if _, err := admin.ExecContext(ctx, "ALTER DATABASE "+pgx.Identifier{current.name}.Sanitize()+" WITH IS_TEMPLATE false"); err != nil { - continue - } - } - drop(ctx, admin, current.name) - t.Logf("removed stale Postgres test database %s", current.name) - } + return stale, rows.Err() } func processAlive(pid int) bool {