Speed up Go CI and report daily validation failures

.github/workflows/cgo.yml

@@ -257,7 +257,7 @@ jobs:
           GO_TEST_AREA: ${{ matrix.area }}
           TEST_SHARD: ${{ matrix.shard }}
           TEST_SHARD_COUNT: ${{ matrix.shards }}
-          GO_TEST_MODE: ${{ github.event_name == 'pull_request' && '-short' || '-count=1' }}
+          GO_TEST_MODE: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '-count=1' || '-short' }}
           POSTGRES_URL: postgresql://postgres@127.0.0.1:${{ job.services.postgres.ports['5432'] }}/cao_server_test?sslmode=disable
           CAO_TEST_POSTGRES_URL: postgresql://postgres@127.0.0.1:${{ job.services.postgres.ports['5432'] }}/cao_server_test?sslmode=disable
         run: |
@@ -626,6 +626,7 @@ jobs:
         env:
           TEST_SHARD: ${{ matrix.shard }}
           TEST_SHARD_COUNT: ${{ strategy.job-total }}
+          GO_TEST_MODE: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '-count=1' || '-short' }}
         run: |
           set -euo pipefail
           pattern=$(go -C server test -p 1 -list '^(Test|Example|Fuzz)' ./... |
@@ -637,7 +638,7 @@ jobs:
                 }
               }')
           test -n "$pattern"
-          go -C server test -p 1 -run "^($pattern)$" ./...
+          go -C server test -p 1 -run "^($pattern)$" ./... "$GO_TEST_MODE"
       - name: Build dashboard and server
         if: matrix.shard == 0
         run: |

.github/workflows/ci-daily.yml

@@ -163,6 +163,7 @@ jobs:
     permissions:
       contents: read
       actions: read
+      issues: write
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -171,12 +172,13 @@ jobs:
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
           SUITES: ${{ toJSON(needs) }}
+          REPORT_FAILURE: ${{ github.ref == 'refs/heads/main' && needs.revision.result == 'success' }}
         with:
           script: |
             const { writeFileSync, appendFileSync } = require('node:fs')
             const { pathToFileURL } = require('node:url')
             const { resolve } = require('node:path')
-            const { buildCIReport } = await import(pathToFileURL(resolve('scripts/ci-report.mjs')).href)
+            const { buildCIReport, reportCIFailure } = await import(pathToFileURL(resolve('scripts/ci-report.mjs')).href)
             const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
               ...context.repo, run_id: context.runId, filter: 'latest', per_page: 100,
             })
@@ -186,6 +188,7 @@ jobs:
             writeFileSync('ci-daily-report.json', JSON.stringify(report, null, 2) + '\n')
             writeFileSync('ci-daily-report.md', report.markdown)
             appendFileSync(process.env.GITHUB_STEP_SUMMARY, report.markdown)
+            if (process.env.REPORT_FAILURE === 'true') await reportCIFailure(github, context.repo, report)
             if (report.status !== 'success') core.setFailed('Daily validation failed or is incomplete; see the suite report.')
       - name: Retain daily report
         if: always()

CONTRIBUTING.md

@@ -144,7 +144,7 @@ and authenticated compatibility tests, bootstrap, and service-backed integration
 run after merge rather than on the PR critical path. A green PR is intentionally
 not a claim that every integration or performance scenario passed.
 
-Main pushes select full suites through each workflow's path filters. Root
+Main pushes select suites through each workflow's path filters. Root
 contract areas are further selected by `scripts/ci-test-plan.mjs`; documentation-only
 changes omit root integration/load/API checks. Unknown paths or an unavailable
 comparison base widen selection. Fast contract checks run concurrently with units;
@@ -153,8 +153,12 @@ the existing `test` aggregate check requires both to succeed.
 Go core tests partition the sorted package list across four jobs, each with its
 own PostgreSQL service. Each job compiles and runs only its assigned packages,
 serially (`-p 1`) to stay within PostgreSQL's lock budget. Storage tests retain
-four test-name shards within their single package. Both retain short mode on
-PRs and uncached full execution after merge.
+four test-name shards within their single package.
+PRs and main pushes use `-short`, including the service-backed
+dashboard Go suite. The 16 core tests measured at 10 seconds or more in the
+investigated CI run (cross-module CLI build, fleet simulation, PostgreSQL
+ingestion/publication, read conformance, and query benchmarks) skip in short
+mode. Daily and manual Go runs omit `-short` and retain these scenarios.
 
 `Daily full validation` (`.github/workflows/ci-daily.yml`) runs at 06:37 UTC
 and can be dispatched manually on main. It reuses the complete contract,
@@ -169,7 +173,11 @@ The daily run publishes an Actions summary and a 30-day `ci-daily-report`
 artifact (Markdown and JSON), including every suite's outcome, total completed
 runner-minutes, and the 20 slowest jobs. Per-suite traces, profiles, and detailed
 reports remain attached to the same run. Failed, cancelled, or skipped suites
-make the aggregate report incomplete. Campaign lifecycle API-quota exhaustion
+make the aggregate report incomplete. A failed or incomplete daily run on main
+opens a `Daily full validation failed` issue, or updates the existing open bot
+issue with the latest failing revision, suite outcomes, and job links. Successful
+runs do not open issues or automatically close an investigation.
+Campaign lifecycle API-quota exhaustion
 also fails daily/manual full runs; main pushes retain infrastructure-incomplete
 warnings instead of blocking on shared quota.
 These measurements exclude queue time and billing multipliers. Use them to

scripts/ci-report.mjs

@@ -36,3 +36,20 @@ export function buildCIReport(suites, jobs, url, sha) {
   ].join("\n");
   return { status, sha, url, suites: results, runnerMinutes, jobs: timedJobs, markdown };
 }
+
+export async function reportCIFailure(github, repo, report) {
+  if (report.status === "success") return;
+  const marker = "<!-- cao-daily-validation-failure -->";
+  const title = "Daily full validation failed";
+  const body = `${marker}\n\n${report.markdown}`;
+  const issues = await github.paginate(github.rest.issues.listForRepo, {
+    ...repo, state: "open", creator: "github-actions[bot]", per_page: 100,
+  });
+  const existing = issues.find((issue) =>
+    !issue.pull_request && issue.body?.includes(marker));
+  if (existing) {
+    await github.rest.issues.update({ ...repo, issue_number: existing.number, body });
+  } else {
+    await github.rest.issues.create({ ...repo, title, body });
+  }
+}

server/cmd/cao-dashboard/adx_setup_test.go

@@ -13,6 +13,9 @@ import (
 )
 
 func TestADXSetupSettingsMatchRuntime(t *testing.T) {
+	if testing.Short() {
+		t.Skip("slow cross-module CLI build; covered by daily full validation")
+	}
 	// The extension and server are separate modules; exercise the actual CLI
 	// output through the runtime parsers instead of duplicating their contract.
 	binary := filepath.Join(t.TempDir(), "gh-cao")

server/cmd/cao-dashboard/benchmark_queries_test.go

@@ -236,6 +236,9 @@ func TestBuildBenchmarkMeasurementReturnsMeasurement(t *testing.T) {
 }
 
 func TestBenchmarkQueriesWithDeployedSubset(t *testing.T) {
+	if testing.Short() {
+		t.Skip("slow PostgreSQL benchmark; covered by daily full validation")
+	}
 	url := os.Getenv("POSTGRES_URL")
 	if url == "" {
 		t.Skip("POSTGRES_URL is not set")
@@ -299,6 +302,9 @@ func TestBenchmarkQueriesWithDeployedSubset(t *testing.T) {
 }
 
 func TestBenchmarkCleanupPreservesOtherNamespaces(t *testing.T) {
+	if testing.Short() {
+		t.Skip("slow PostgreSQL benchmark; covered by daily full validation")
+	}
 	url := os.Getenv("POSTGRES_URL")
 	if url == "" {
 		t.Skip("POSTGRES_URL is not set")

server/cmd/cao-dashboard/main_test.go

@@ -26,6 +26,9 @@ import (
 )
 
 func TestServeWithExistingPostgresOperationalTables(t *testing.T) {
+	if testing.Short() {
+		t.Skip("slow PostgreSQL server integration; covered by daily full validation")
+	}
 	endpoint := os.Getenv("POSTGRES_URL")
 	if endpoint == "" {
 		t.Skip("POSTGRES_URL is not set")

server/internal/adxingest/postgres_integration_test.go

@@ -53,6 +53,9 @@ func publishSession(t *testing.T, data *postgresx.Store, tokens int) {
 }
 
 func TestPostgresSimulatorDurableRestartAndCanonicalReplacement(t *testing.T) {
+	if testing.Short() {
+		t.Skip("slow PostgreSQL ingestion integration; covered by daily full validation")
+	}
 	data := testutil.Postgres(t, t.Context(), "POSTGRES_URL")
 	publishSession(t, data, 0)
 	endpoint := os.Getenv("POSTGRES_URL")

server/internal/collectnative/audit_engine_postgres_test.go

@@ -25,6 +25,9 @@ func (engineAuditProvider) Jobs(_ context.Context, run collectnative.Run, visit
 }
 
 func TestNativeLegacyEngineAndGatewayPostgresPublication(t *testing.T) {
+	if testing.Short() {
+		t.Skip("slow PostgreSQL publication integration; covered by daily full validation")
+	}
 	for _, test := range []struct{ engine, content string }{
 		{"codex", "thinking\ntool github.read({})\ntokens used\n60,001"},
 		{"claude", `prefix {'usage':{'input_tokens':60000,'output_tokens':1},'cost':0.25}`},

server/internal/dashboardstore/conformance_test.go

@@ -22,6 +22,9 @@ type readFixture struct {
 }
 
 func TestReadContract(t *testing.T) {
+	if testing.Short() {
+		t.Skip("slow PostgreSQL read conformance; covered by daily full validation")
+	}
 	repositories := []model.Row{
 		{"id": "repo:1", "owner": "octo", "name": "one"},
 		{"id": "repo:2", "owner": "octo", "name": "two"},

server/internal/ingest/audit_curation_integration_test.go

@@ -19,6 +19,9 @@ import (
 )
 
 func TestUnchangedArtifactRevisionCuratesExistingAudits(t *testing.T) {
+	if testing.Short() {
+		t.Skip("slow PostgreSQL ingestion integration; covered by daily full validation")
+	}
 	ctx, store, config := ingestTestStoreConfig(t)
 	options := Options{DatabaseQueriesPath: "../../../dashboard/site/src/data/queries/database.json"}
 	directory, _ := testutil.RecentSnapshot(t, "../../testdata/deployed-subset")
@@ -47,6 +50,9 @@ func TestUnchangedArtifactRevisionCuratesExistingAudits(t *testing.T) {
 }
 
 func TestAuditCurationValidatesEveryTransportedRecord(t *testing.T) {
+	if testing.Short() {
+		t.Skip("slow PostgreSQL ingestion integration; covered by daily full validation")
+	}
 	ctx, store := ingestTestStore(t)
 	directory := scratchDirectory(t)
 	recent, _ := testutil.RecentSnapshot(t, "../../testdata/deployed-subset")
@@ -111,6 +117,9 @@ func TestAuditCurationValidatesEveryTransportedRecord(t *testing.T) {
 }
 
 func TestUnsupportedAuditEvidencePreservesPublishedIngestion(t *testing.T) {
+	if testing.Short() {
+		t.Skip("slow PostgreSQL ingestion integration; covered by daily full validation")
+	}
 	ctx, store := ingestTestStore(t)
 	directory, _ := testutil.RecentSnapshot(t, "../../testdata/deployed-subset")
 	first, err := Run(ctx, store, directory, Options{DatabaseQueriesPath: "../../../dashboard/site/src/data/queries/database.json"})
@@ -134,6 +143,9 @@ func TestUnsupportedAuditEvidencePreservesPublishedIngestion(t *testing.T) {
 }
 
 func TestAuditCurationDoesNotHideTransportedOrphan(t *testing.T) {
+	if testing.Short() {
+		t.Skip("slow PostgreSQL ingestion integration; covered by daily full validation")
+	}
 	ctx, store := ingestTestStore(t)
 	options := Options{DatabaseQueriesPath: "../../../dashboard/site/src/data/queries/database.json"}
 	recent, _ := testutil.RecentSnapshot(t, "../../testdata/deployed-subset")

server/internal/ingest/ingest_test.go

@@ -62,6 +62,9 @@ func TestValidateManifestFailsClosed(t *testing.T) {
 }
 
 func TestLocalIngestionReceiptsRejectForeignTransactions(t *testing.T) {
+	if testing.Short() {
+		t.Skip("slow PostgreSQL ingestion integration; covered by daily full validation")
+	}
 	for _, row := range []model.Row{
 		{"id": "unrelated", "kind": "local-dashboard-ingestion"},
 		{"id": "local-dashboard-ingestion", "kind": "unrelated"},

server/internal/ingest/reclamation_integration_test.go

@@ -57,6 +57,9 @@ func ingestTestStoreConfig(t *testing.T) (context.Context, *postgresx.Store, *pg
 }
 
 func TestFreshNativeIngestionReuseForceAndQueries(t *testing.T) {
+	if testing.Short() {
+		t.Skip("slow PostgreSQL ingestion integration; covered by daily full validation")
+	}
 	ctx, store := ingestTestStore(t)
 	options := Options{DatabaseQueriesPath: "../../../dashboard/site/src/data/queries/database.json"}
 	directory, _ := testutil.RecentSnapshot(t, "../../testdata/deployed-subset")

server/internal/ingest/recovery_integration_test.go

@@ -13,6 +13,9 @@ import (
 )
 
 func TestEmptyPostgresRebuildAndFailedIngestionPreservesCurrentData(t *testing.T) {
+	if testing.Short() {
+		t.Skip("slow PostgreSQL ingestion integration; covered by daily full validation")
+	}
 	ctx, store := ingestTestStore(t)
 	directory, _ := testutil.RecentSnapshot(t, "../../testdata/deployed-subset")
 	options := Options{DatabaseQueriesPath: "../../../dashboard/site/src/data/queries/database.json"}

server/internal/simulator/dashboard_read_test.go

@@ -108,6 +108,9 @@ func newDashboardReadFixture(t *testing.T) dashboardReadFixture {
 }
 
 func TestDashboardReadSyntheticScenario(t *testing.T) {
+	if testing.Short() {
+		t.Skip("slow PostgreSQL dashboard scenario; covered by daily full validation")
+	}
 	fixture := newDashboardReadFixture(t)
 	app, err := server.New(t.Context(), fixture.services, fixture.config)
 	if err != nil {

server/internal/webhookgateway/stress_test.go

@@ -314,6 +314,9 @@ func runFleetStress(t *testing.T, repositories, developers, days int) {
 }
 
 func TestGitHubSimulatorFleetSmoke(t *testing.T) {
+	if testing.Short() {
+		t.Skip("slow fleet simulation; covered by daily full validation")
+	}
 	runFleetStress(t, 10, 10, 2)
 }
 

tests/unit/cao-ci-tiers.test.mjs

@@ -4,7 +4,7 @@ import { spawnSync } from "node:child_process";
 import test from "node:test";
 import { parse } from "yaml";
 import { changedFiles, testPlan } from "../../scripts/ci-test-plan.mjs";
-import { buildCIReport } from "../../scripts/ci-report.mjs";
+import { buildCIReport, reportCIFailure } from "../../scripts/ci-report.mjs";
 import { unitTestAreas } from "../../scripts/run-unit-area.mjs";
 
 const load = (name) => parse(readFileSync(`.github/workflows/${name}`, "utf8"));
@@ -61,6 +61,13 @@ test("daily calls every suite at the same revision and reports even on failure",
     assert.equal(job.secrets, undefined, "test suites do not inherit deployment secrets");
   }
   assert.equal(daily.jobs.report.if, "always()");
+  assert.equal(daily.jobs.report.permissions.issues, "write");
+  assert.equal(daily.permissions.issues, undefined);
+  const publish = daily.jobs.report.steps.find((step) => step.name === "Publish suite outcomes and slowest jobs");
+  assert.equal(publish.env.REPORT_FAILURE,
+    "${{ github.ref == 'refs/heads/main' && needs.revision.result == 'success' }}");
+  assert.match(publish.with.script, /if \(process\.env\.REPORT_FAILURE === 'true'\) await reportCIFailure/);
+  assert.match(publish.with.script, /if \(report\.status !== 'success'\) core\.setFailed/);
   assert.deepEqual(daily.jobs.deployed.with, { full: true });
   assert.deepEqual(daily.jobs["company-stress"].with, { preset: "large" });
   assert.equal(daily.jobs["worker-memory"].needs, "revision");
@@ -117,3 +124,72 @@ test("daily reporting counts runner time, ranks slow jobs, and rejects incomplet
   assert.equal(buildCIReport({ suite: { result: "success" } }, jobs, "", "").status, "incomplete");
   assert.equal(buildCIReport({ suite: { result: "success" } }, [jobs[0]], "", "").status, "success");
 });
+
+test("daily validation opens one failure issue and updates it with the latest evidence", async () => {
+  const repo = { owner: "octo", repo: "control" };
+  const calls = [];
+  const open = [{ number: 1, title: "Daily full validation failed", body: "unrelated" }];
+  const github = {
+    rest: {
+      issues: {
+        listForRepo: Symbol("list"),
+        create: async (request) => {
+          calls.push(["create", request]);
+          open.push({ number: 2, ...request });
+        },
+        update: async (request) => calls.push(["update", request]),
+      },
+    },
+    paginate: async (method, request) => {
+      assert.equal(method, github.rest.issues.listForRepo);
+      assert.deepEqual(request, { ...repo, state: "open", creator: "github-actions[bot]", per_page: 100 });
+      return open;
+    },
+  };
+  const report = buildCIReport({ server: { result: "failure" } }, [], "https://example.com/run", "first-sha");
+  await reportCIFailure(github, repo, report);
+  assert.equal(calls.length, 1);
+  assert.equal(calls[0][0], "create");
+  assert.equal(calls[0][1].title, "Daily full validation failed");
+  assert.ok(calls[0][1].body.includes(report.markdown));
+  const latest = buildCIReport({ server: { result: "cancelled" } }, [], "https://example.com/retry", "next-sha");
+  await reportCIFailure(github, repo, latest);
+  assert.deepEqual(calls[1], ["update", {
+    ...repo, issue_number: 2, body: `<!-- cao-daily-validation-failure -->\n\n${latest.markdown}`,
+  }]);
+  // A copied marker in a PR must not turn a pull request into the failure tracker.
+  open[1].pull_request = {};
+  await reportCIFailure(github, repo, latest);
+  assert.equal(calls[2][0], "create");
+});
+
+test("successful daily validation does not call the issue API", async () => {
+  const job = {
+    name: "server", status: "completed", conclusion: "success",
+    started_at: "2026-01-01T00:00:00Z", completed_at: "2026-01-01T00:00:30Z",
+  };
+  const report = buildCIReport({ server: { result: "success" } }, [job], "", "");
+  assert.equal(report.status, "success");
+  await reportCIFailure({}, {}, report);
+});
+
+test("daily failure reporting propagates issue API errors", async () => {
+  const error = new Error("issue API unavailable");
+  const report = buildCIReport({ server: { result: "failure" } }, [], "", "");
+  for (const operation of ["list", "create", "update"]) {
+    const github = {
+      rest: {
+        issues: {
+          listForRepo: Symbol("list"),
+          create: async () => { throw error; },
+          update: async () => { throw error; },
+        },
+      },
+      paginate: async () => {
+        if (operation === "list") throw error;
+        return operation === "create" ? [] : [{ number: 2, body: "<!-- cao-daily-validation-failure -->" }];
+      },
+    };
+    await assert.rejects(reportCIFailure(github, {}, report), error);
+  }
+});