From f45f968889480657293e9a236b8d68f88299172a Mon Sep 17 00:00:00 2001 From: Peli de Halleux <4175913+pelikhan@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:53:53 -0800 Subject: [PATCH 1/5] Remove materialize-cao.mjs and retired cao.mjs references Runtime packaging moves into scripts/build-cao-runtime.mjs. The installer no longer materializes the runtime; native gh cao setup/update stage the attested artifact. Drop the setup-cao-runtime action and local source materialization. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/actions/setup-cao-runtime/action.yml | 16 - .../references/event-ledger.md | 4 - .github/workflows/actions-runtime.yml | 2 - .github/workflows/cao-bootstrap.yml | 1 - .../workflows/cao-framework-health.lock.yml | 4 +- .github/workflows/cao-framework-health.md | 2 +- .github/workflows/cgo.yml | 2 - .github/workflows/install-windows.yml | 2 - .github/workflows/shared/materialize-cao.mjs | 556 ------------------ ARCHITECTURE.md | 4 +- CONTRIBUTING.md | 6 +- adr/canonical-cao-runtime-source-layout.md | 18 +- aw.yml | 2 - install-gh-cao.sh | 23 +- internal/cli/native_management_runtime.go | 2 +- .../cli/native_management_runtime_test.go | 2 +- internal/cli/root.go | 1 - internal/cli/setup_runtime.go | 1 - runtime/package.json | 1 - scripts/build-cao-runtime.mjs | 180 +++++- scripts/campaign-lifecycle-matrix.mjs | 1 - scripts/validate-cao-runtime.mjs | 3 +- .../internal/simulator/azure_journey_test.go | 2 +- ...ootstrap_compatibility_integration_test.go | 22 +- .../simulator/bootstrap_integration_test.go | 1 - specs/repository-layout-and-installation.md | 23 +- tests/e2e/dashboard-massive-scale.spec.mjs | 19 +- tests/integration/campaign-lifecycle.test.mjs | 49 +- .../cao-runtime-installation.test.mjs | 14 +- tests/integration/install-script.test.mjs | 1 - tests/unit/cao-materialize.test.mjs | 344 ----------- tests/unit/cao-runtime-artifact.test.mjs | 226 ++----- tests/unit/install-script.test.mjs | 42 +- .../unit/workflow-contract-campaigns.test.mjs | 10 +- 34 files changed, 284 insertions(+), 1302 deletions(-) delete mode 100644 .github/actions/setup-cao-runtime/action.yml delete mode 100644 .github/workflows/shared/materialize-cao.mjs delete mode 100644 tests/unit/cao-materialize.test.mjs diff --git a/.github/actions/setup-cao-runtime/action.yml b/.github/actions/setup-cao-runtime/action.yml deleted file mode 100644 index 15b01f70d1..0000000000 --- a/.github/actions/setup-cao-runtime/action.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Set up CAO runtime -description: Verify that CAO runtime files use their canonical catalog source paths - -inputs: - bundle: - description: CAO runtime bundle to verify - required: true - -runs: - using: composite - steps: - - name: Verify canonical CAO runtime layout - shell: bash - env: - CAO_RUNTIME_BUNDLE: ${{ inputs.bundle }} - run: node "$GITHUB_WORKSPACE/.github/workflows/shared/materialize-cao.mjs" verify "$CAO_RUNTIME_BUNDLE" diff --git a/.github/skills/add-operational-value/references/event-ledger.md b/.github/skills/add-operational-value/references/event-ledger.md index f8a416ff2c..268f4dd8e0 100644 --- a/.github/skills/add-operational-value/references/event-ledger.md +++ b/.github/skills/add-operational-value/references/event-ledger.md @@ -214,10 +214,6 @@ original ledger, not the computed pulse model, and makes no remote requests: ```bash node .github/skills/add-operational-value/scripts/export-evidence.mjs \ ledger.json .cao/gh-aw-logs-shards/workflow-value-evidence.jsonl -node activity/cao.mjs hash-payloads \ - --shard-dir .cao/gh-aw-logs-shards --normalized-dir .cao/gh-aw-logs-normalized \ - --runs-dir .cao/gh-aw-logs-runs --records-dir .cao/gh-aw-logs-records \ - --output .cao/payload-hashes.json ``` Use an existing, appropriately scoped publication directory. Import is not diff --git a/.github/workflows/actions-runtime.yml b/.github/workflows/actions-runtime.yml index 7582b8f15c..cc7cc2dd5c 100644 --- a/.github/workflows/actions-runtime.yml +++ b/.github/workflows/actions-runtime.yml @@ -5,7 +5,6 @@ on: pull_request: paths: - .github/workflows/actions-runtime.yml - - .github/actions/setup-cao-runtime/** - .github/workflows/shared/** - dashboard/site/src/data/** @@ -20,7 +19,6 @@ on: branches: [main] paths: - .github/workflows/actions-runtime.yml - - .github/actions/setup-cao-runtime/** - .github/workflows/shared/** - dashboard/site/src/data/** diff --git a/.github/workflows/cao-bootstrap.yml b/.github/workflows/cao-bootstrap.yml index 9ea3db818d..46456f4d92 100644 --- a/.github/workflows/cao-bootstrap.yml +++ b/.github/workflows/cao-bootstrap.yml @@ -8,7 +8,6 @@ on: paths: - .github/workflows/cao-bootstrap.yml - .github/workflows/shared/** - - .github/actions/setup-cao-runtime/** - aw.yml - install-gh-cao.sh diff --git a/.github/workflows/cao-framework-health.lock.yml b/.github/workflows/cao-framework-health.lock.yml index 9c171cf29a..db11262c9f 100644 --- a/.github/workflows/cao-framework-health.lock.yml +++ b/.github/workflows/cao-framework-health.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"50d11321534c74e5a520bd63fd7b918fedc42180748ecee766e85ee94aeca045","body_hash":"01dd637a5d60c1d41246c2dd474c5d3c1d6138c393fbd59bd4fdc12fe024f51f","compiler_version":"v1.2.2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.103"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"ffd5d5f1e79f6a521d3e336294ae53b85ee95407f82703b2997b566043ddaa50","body_hash":"01dd637a5d60c1d41246c2dd474c5d3c1d6138c393fbd59bd4fdc12fe024f51f","compiler_version":"v1.2.2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.103"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"},{"repo":"github/gh-aw-actions/setup","sha":"53d0749e19424a918acd82b4d140264ce48f2f14","version":"v1.2.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.29.11","digest":"sha256:8774a73c351395b41d2ad0af4ddb4689007c67de3cd08bd9f43bb8a0e300690e","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.29.11@sha256:8774a73c351395b41d2ad0af4ddb4689007c67de3cd08bd9f43bb8a0e300690e"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.29.11","digest":"sha256:88d2d0e315a504d93fd4fb8847fd0f6b7f4469f41a6fc6047b3dc9654c56ab2c","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.29.11@sha256:88d2d0e315a504d93fd4fb8847fd0f6b7f4469f41a6fc6047b3dc9654c56ab2c"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.29.11","digest":"sha256:97782bc0d8ff7f80f19ae2fdc5e1668517aa732a6fa077e64b8d3919238496d2","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.29.11@sha256:97782bc0d8ff7f80f19ae2fdc5e1668517aa732a6fa077e64b8d3919238496d2"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.29.11","digest":"sha256:861739ecf924dd2db5e5fb652db2ad82aaafd1615623297ddabc8f41a36b2847","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.29.11@sha256:861739ecf924dd2db5e5fb652db2ad82aaafd1615623297ddabc8f41a36b2847"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.5.6","digest":"sha256:342cd624e1f179d91be2de7a146fc4f387092e62e6c8a47506d7932e113d2311","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.5.6@sha256:342cd624e1f179d91be2de7a146fc4f387092e62e6c8a47506d7932e113d2311"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.1.0","digest":"sha256:f6056fc10bfffcc8c81a6417a2e9ca12808647e8dd6049642a4feaadee0e0f8a","pinned_image":"ghcr.io/github/github-mcp-server:v2.1.0@sha256:f6056fc10bfffcc8c81a6417a2e9ca12808647e8dd6049642a4feaadee0e0f8a"}],"mcp_servers":[{"name":"safeoutputs","tools":["create_issue","noop","report_incomplete","report_missing_data","report_missing_tool"]}],"threat_detection":{"mode":"enabled"},"admission":false,"agent_metadata":{"d0b9e7072036c3cc9c38":{"name":"W3C Specification"},"fbd10cd783979952bdc7":{"name":"Agentic Workflows"}}} # This file was automatically generated by gh-aw (v1.2.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -597,7 +597,7 @@ jobs: name: Measure core component fluctuation uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - script: "const fs = require('node:fs');\nconst { execFileSync } = require('node:child_process');\nconst directory = '/tmp/gh-aw/agent/framework-health';\nconst evidence = JSON.parse(fs.readFileSync(`${directory}/context.json`, 'utf8'));\nconst git = args => execFileSync('git', args, { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024 }).trim();\nif (git(['rev-parse', 'HEAD']) !== evidence.source_sha\n || git(['rev-parse', '--is-shallow-repository']) !== 'false') {\n throw new Error('Component fluctuation requires complete history at the assessed SHA');\n}\nconst components = [\n { id: 'control', claims: ['control.authority', 'control.routing'], paths: [\n '.github/workflows/shared/authority.mjs', '.github/workflows/shared/policy.mjs',\n '.github/workflows/shared/operations.mjs', '.github/workflows/shared/control.md',\n '.github/workflows/shared/control.mjs', '.github/workflows/shared/inventory-credentials.mjs',\n 'server/internal/admission', 'server/internal/tokenbroker',\n 'tests/unit/control-*.test.mjs', 'tests/integration/control-*.test.mjs',\n ] },\n { id: 'acquisition', claims: ['acquisition.bounded', 'acquisition.provenance'], paths: [\n 'activity', 'server/internal/collect', 'server/internal/collectnative', 'server/internal/ingest',\n 'tests/unit/activity-*.test.mjs', 'tests/integration/activity-*.test.mjs',\n ] },\n { id: 'canonical', claims: ['canonical.atomicity', 'canonical.retention'], paths: [\n 'dashboard/site/src/data/model', 'dashboard/site/src/data/normalize',\n 'dashboard/site/src/data/storage', 'dashboard/site/src/data/adapters',\n 'server/internal/model', 'server/internal/postgresx',\n 'dashboard/site/test/unit/data-*.test.js', 'tests/contracts',\n ] },\n { id: 'queries', claims: ['queries.declarative', 'queries.parity'], paths: [\n 'dashboard/site/src/data/queries', 'dashboard/site/src/data-worker.js',\n 'dashboard/site/src/presenter.js', 'dashboard/site/src/reactive.js',\n 'dashboard/site/dashboard.json', 'dashboard/site/dashboard-fragments',\n 'internal/dashboardquery', 'server/internal/query',\n 'dashboard/site/test/unit/query-*.test.js', 'tests/unit/dashboard-query-*.test.mjs',\n 'tests/e2e/dashboard-query-parity.mjs',\n ] },\n { id: 'operational', claims: ['operational.durability', 'operational.isolation'], paths: [\n 'server/internal/operational', 'server/internal/redisx',\n ] },\n { id: 'hosting', claims: ['hosting.origin-auth', 'hosting.recovery'], paths: [\n 'server/cmd', 'server/internal/server', 'server/internal/hostedcli',\n 'server/internal/webhookgateway', 'server/internal/webhookgatewayhost',\n 'server/internal/simulator', 'server/spec',\n ] },\n { id: 'distribution', claims: ['distribution.provenance', 'distribution.compatibility'], paths: [\n 'cmd/gh-cao', 'internal/cli', 'internal/ghcli', 'runtime', 'install-gh-cao.sh',\n '.github/workflows/shared/materialize-cao.mjs', '.github/actions',\n 'tests/integration/install-script.test.mjs', 'tests/integration/cao-runtime-installation.test.mjs',\n 'tests/unit/cao-runtime-*.test.mjs',\n ] },\n { id: 'outcomes', claims: ['outcomes.acceptance', 'outcomes.quality'], paths: [\n 'activity/operational-value*.mjs', 'activity/computations', 'server/internal/operationalvalue',\n 'tests/unit/cao-operational-value*.test.mjs', 'tests/integration/cao-operational-value-local.test.mjs',\n ] },\n];\nconst measure = (paths, window) => {\n const output = git(['log', '--first-parent', '--diff-merges=first-parent', '--no-renames',\n '--since-as-filter=' + window.start, '--until=' + window.end, '--max-count=1001',\n '--format=%x1e%H%x09%cI', '--numstat', evidence.source_sha, '--', ...paths]);\n const records = output.split('\\x1e').filter(value => value.trim());\n const result = {\n ...window, coverage: records.length < 1001 ? 'complete' : 'partial',\n commits: 0, files_changed: 0, source_added: 0, source_deleted: 0,\n test_added: 0, test_deleted: 0, generated_files_changed: 0, binary_files_changed: 0,\n };\n const files = new Set(), generated = new Set(), binaries = new Set(), sourceDays = new Set();\n for (const record of records.slice(0, 1000)) {\n const [header, ...lines] = record.trim().split('\\n');\n const [sha, time] = header.split('\\t');\n if (!/^[a-f0-9]{40}$/.test(sha) || !Number.isFinite(Date.parse(time))) {\n throw new Error('Invalid first-parent history record');\n }\n if (Date.parse(time) < Date.parse(window.start) || Date.parse(time) >= Date.parse(window.end)) continue;\n result.commits += 1;\n for (const line of lines.filter(value => value.trim())) {\n const [added, deleted, path] = line.split('\\t');\n if (!path) throw new Error('Invalid numstat record');\n files.add(path);\n if (/\\.lock\\.yml$|(^|\\/)package-lock\\.json$|(^|\\/)generated\\/|\\.generated\\.|postgresx\\/schema\\./.test(path)) {\n generated.add(path);\n } else if (added === '-' || deleted === '-') {\n binaries.add(path);\n } else {\n if (!/^\\d+$/.test(added) || !/^\\d+$/.test(deleted)) throw new Error('Invalid numstat counts');\n const kind = /(^|\\/)tests?\\/|_test\\.go$|\\.(test|spec)\\.[cm]?[jt]sx?$/.test(path) ? 'test' : 'source';\n const additions = Number(added), deletions = Number(deleted);\n result[`${kind}_added`] += additions;\n result[`${kind}_deleted`] += deletions;\n if (kind === 'source' && additions + deletions > 0) {\n sourceDays.add(new Date(time).toISOString().slice(0, 10));\n }\n }\n }\n }\n result.files_changed = files.size;\n result.generated_files_changed = generated.size;\n result.binary_files_changed = binaries.size;\n result.source_net = result.source_added - result.source_deleted;\n result.source_days_changed = sourceDays.size;\n return result;\n};\nconst measurements = components.map(component => ({\n ...component, current: measure(component.paths, evidence.ci_windows.current),\n previous: measure(component.paths, evidence.ci_windows.previous),\n}));\nfs.writeFileSync(`${directory}/components.json`, JSON.stringify({\n basis: 'first-parent committer-time churn, not merge-time acceptance or quality',\n limit: '1000 first-parent commits per component per window',\n components: measurements,\n}));\nconst churn = row => row.source_added + row.source_deleted;\nconst cell = row => {\n const value = churn(row);\n const days = row.source_days_changed;\n const lines = `${value} ${value === 1 ? 'line' : 'lines'}`;\n if (row.coverage !== 'complete') {\n return `N/A (${days}/7 observed days; ${lines}; partial history)`;\n }\n const intensity = days === 0 ? '·' : days === 1 ? '▁'\n : days <= 3 ? '▂' : days <= 5 ? '▄' : '█';\n return `${intensity} ${days}/7 days; ${lines}`;\n};\nconst heatmap = [\n '**Core component change frequency (UTC days with source edits; activity, not stability)**',\n '',\n `Previous: ${evidence.ci_windows.previous.start.slice(0, 10)} to ${evidence.ci_windows.previous.end.slice(0, 10)} UTC; current: ${evidence.ci_windows.current.start.slice(0, 10)} to ${evidence.ci_windows.current.end.slice(0, 10)} UTC (end exclusive).`,\n '',\n '| Component | Previous 7 days | Current 7 days |',\n '| --- | --- | --- |',\n ...measurements.map(row => `| ${row.id} | ${cell(row.previous)} | ${cell(row.current)} |`),\n '',\n 'Fixed scale: · 0 days; ▁ 1; ▂ 2–3; ▄ 4–5; █ 6–7. N/A is incomplete, not zero.',\n 'Lines are source additions + deletions. Adjacent complete 7-day UTC windows; first-parent committer-time, not merge time. Test/generated changes excluded; overlapping components must not be summed.',\n];\nfs.writeFileSync(`${directory}/component-heatmap.md`, heatmap.join('\\n'));\n" + script: "const fs = require('node:fs');\nconst { execFileSync } = require('node:child_process');\nconst directory = '/tmp/gh-aw/agent/framework-health';\nconst evidence = JSON.parse(fs.readFileSync(`${directory}/context.json`, 'utf8'));\nconst git = args => execFileSync('git', args, { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024 }).trim();\nif (git(['rev-parse', 'HEAD']) !== evidence.source_sha\n || git(['rev-parse', '--is-shallow-repository']) !== 'false') {\n throw new Error('Component fluctuation requires complete history at the assessed SHA');\n}\nconst components = [\n { id: 'control', claims: ['control.authority', 'control.routing'], paths: [\n '.github/workflows/shared/authority.mjs', '.github/workflows/shared/policy.mjs',\n '.github/workflows/shared/operations.mjs', '.github/workflows/shared/control.md',\n '.github/workflows/shared/control.mjs', '.github/workflows/shared/inventory-credentials.mjs',\n 'server/internal/admission', 'server/internal/tokenbroker',\n 'tests/unit/control-*.test.mjs', 'tests/integration/control-*.test.mjs',\n ] },\n { id: 'acquisition', claims: ['acquisition.bounded', 'acquisition.provenance'], paths: [\n 'activity', 'server/internal/collect', 'server/internal/collectnative', 'server/internal/ingest',\n 'tests/unit/activity-*.test.mjs', 'tests/integration/activity-*.test.mjs',\n ] },\n { id: 'canonical', claims: ['canonical.atomicity', 'canonical.retention'], paths: [\n 'dashboard/site/src/data/model', 'dashboard/site/src/data/normalize',\n 'dashboard/site/src/data/storage', 'dashboard/site/src/data/adapters',\n 'server/internal/model', 'server/internal/postgresx',\n 'dashboard/site/test/unit/data-*.test.js', 'tests/contracts',\n ] },\n { id: 'queries', claims: ['queries.declarative', 'queries.parity'], paths: [\n 'dashboard/site/src/data/queries', 'dashboard/site/src/data-worker.js',\n 'dashboard/site/src/presenter.js', 'dashboard/site/src/reactive.js',\n 'dashboard/site/dashboard.json', 'dashboard/site/dashboard-fragments',\n 'internal/dashboardquery', 'server/internal/query',\n 'dashboard/site/test/unit/query-*.test.js', 'tests/unit/dashboard-query-*.test.mjs',\n 'tests/e2e/dashboard-query-parity.mjs',\n ] },\n { id: 'operational', claims: ['operational.durability', 'operational.isolation'], paths: [\n 'server/internal/operational', 'server/internal/redisx',\n ] },\n { id: 'hosting', claims: ['hosting.origin-auth', 'hosting.recovery'], paths: [\n 'server/cmd', 'server/internal/server', 'server/internal/hostedcli',\n 'server/internal/webhookgateway', 'server/internal/webhookgatewayhost',\n 'server/internal/simulator', 'server/spec',\n ] },\n { id: 'distribution', claims: ['distribution.provenance', 'distribution.compatibility'], paths: [\n 'cmd/gh-cao', 'internal/cli', 'internal/ghcli', 'runtime', 'install-gh-cao.sh',\n 'scripts/build-cao-runtime.mjs', '.github/actions',\n 'tests/integration/install-script.test.mjs', 'tests/integration/cao-runtime-installation.test.mjs',\n 'tests/unit/cao-runtime-*.test.mjs',\n ] },\n { id: 'outcomes', claims: ['outcomes.acceptance', 'outcomes.quality'], paths: [\n 'activity/operational-value*.mjs', 'activity/computations', 'server/internal/operationalvalue',\n 'tests/unit/cao-operational-value*.test.mjs', 'tests/integration/cao-operational-value-local.test.mjs',\n ] },\n];\nconst measure = (paths, window) => {\n const output = git(['log', '--first-parent', '--diff-merges=first-parent', '--no-renames',\n '--since-as-filter=' + window.start, '--until=' + window.end, '--max-count=1001',\n '--format=%x1e%H%x09%cI', '--numstat', evidence.source_sha, '--', ...paths]);\n const records = output.split('\\x1e').filter(value => value.trim());\n const result = {\n ...window, coverage: records.length < 1001 ? 'complete' : 'partial',\n commits: 0, files_changed: 0, source_added: 0, source_deleted: 0,\n test_added: 0, test_deleted: 0, generated_files_changed: 0, binary_files_changed: 0,\n };\n const files = new Set(), generated = new Set(), binaries = new Set(), sourceDays = new Set();\n for (const record of records.slice(0, 1000)) {\n const [header, ...lines] = record.trim().split('\\n');\n const [sha, time] = header.split('\\t');\n if (!/^[a-f0-9]{40}$/.test(sha) || !Number.isFinite(Date.parse(time))) {\n throw new Error('Invalid first-parent history record');\n }\n if (Date.parse(time) < Date.parse(window.start) || Date.parse(time) >= Date.parse(window.end)) continue;\n result.commits += 1;\n for (const line of lines.filter(value => value.trim())) {\n const [added, deleted, path] = line.split('\\t');\n if (!path) throw new Error('Invalid numstat record');\n files.add(path);\n if (/\\.lock\\.yml$|(^|\\/)package-lock\\.json$|(^|\\/)generated\\/|\\.generated\\.|postgresx\\/schema\\./.test(path)) {\n generated.add(path);\n } else if (added === '-' || deleted === '-') {\n binaries.add(path);\n } else {\n if (!/^\\d+$/.test(added) || !/^\\d+$/.test(deleted)) throw new Error('Invalid numstat counts');\n const kind = /(^|\\/)tests?\\/|_test\\.go$|\\.(test|spec)\\.[cm]?[jt]sx?$/.test(path) ? 'test' : 'source';\n const additions = Number(added), deletions = Number(deleted);\n result[`${kind}_added`] += additions;\n result[`${kind}_deleted`] += deletions;\n if (kind === 'source' && additions + deletions > 0) {\n sourceDays.add(new Date(time).toISOString().slice(0, 10));\n }\n }\n }\n }\n result.files_changed = files.size;\n result.generated_files_changed = generated.size;\n result.binary_files_changed = binaries.size;\n result.source_net = result.source_added - result.source_deleted;\n result.source_days_changed = sourceDays.size;\n return result;\n};\nconst measurements = components.map(component => ({\n ...component, current: measure(component.paths, evidence.ci_windows.current),\n previous: measure(component.paths, evidence.ci_windows.previous),\n}));\nfs.writeFileSync(`${directory}/components.json`, JSON.stringify({\n basis: 'first-parent committer-time churn, not merge-time acceptance or quality',\n limit: '1000 first-parent commits per component per window',\n components: measurements,\n}));\nconst churn = row => row.source_added + row.source_deleted;\nconst cell = row => {\n const value = churn(row);\n const days = row.source_days_changed;\n const lines = `${value} ${value === 1 ? 'line' : 'lines'}`;\n if (row.coverage !== 'complete') {\n return `N/A (${days}/7 observed days; ${lines}; partial history)`;\n }\n const intensity = days === 0 ? '·' : days === 1 ? '▁'\n : days <= 3 ? '▂' : days <= 5 ? '▄' : '█';\n return `${intensity} ${days}/7 days; ${lines}`;\n};\nconst heatmap = [\n '**Core component change frequency (UTC days with source edits; activity, not stability)**',\n '',\n `Previous: ${evidence.ci_windows.previous.start.slice(0, 10)} to ${evidence.ci_windows.previous.end.slice(0, 10)} UTC; current: ${evidence.ci_windows.current.start.slice(0, 10)} to ${evidence.ci_windows.current.end.slice(0, 10)} UTC (end exclusive).`,\n '',\n '| Component | Previous 7 days | Current 7 days |',\n '| --- | --- | --- |',\n ...measurements.map(row => `| ${row.id} | ${cell(row.previous)} | ${cell(row.current)} |`),\n '',\n 'Fixed scale: · 0 days; ▁ 1; ▂ 2–3; ▄ 4–5; █ 6–7. N/A is incomplete, not zero.',\n 'Lines are source additions + deletions. Adjacent complete 7-day UTC windows; first-parent committer-time, not merge time. Test/generated changes excluded; overlapping components must not be summed.',\n];\nfs.writeFileSync(`${directory}/component-heatmap.md`, heatmap.join('\\n'));\n" - if: ${{ steps.framework_health.outputs.duplicate != 'true' }} name: Collect preview acceptance evidence uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 diff --git a/.github/workflows/cao-framework-health.md b/.github/workflows/cao-framework-health.md index c355f31721..bed9d60221 100644 --- a/.github/workflows/cao-framework-health.md +++ b/.github/workflows/cao-framework-health.md @@ -379,7 +379,7 @@ steps: ] }, { id: 'distribution', claims: ['distribution.provenance', 'distribution.compatibility'], paths: [ 'cmd/gh-cao', 'internal/cli', 'internal/ghcli', 'runtime', 'install-gh-cao.sh', - '.github/workflows/shared/materialize-cao.mjs', '.github/actions', + 'scripts/build-cao-runtime.mjs', '.github/actions', 'tests/integration/install-script.test.mjs', 'tests/integration/cao-runtime-installation.test.mjs', 'tests/unit/cao-runtime-*.test.mjs', ] }, diff --git a/.github/workflows/cgo.yml b/.github/workflows/cgo.yml index d4287df442..92e1225865 100644 --- a/.github/workflows/cgo.yml +++ b/.github/workflows/cgo.yml @@ -27,7 +27,6 @@ on: - tests/unit/dashboard-daily-run-counts-conformance.test.mjs - tests/integration/activity-actions-runtime.test.mjs - tests/helpers/actions-runtime/** - - .github/actions/setup-cao-runtime/** - server/** - tests/fixtures/company-stress/** - scripts/company-stress.mjs @@ -64,7 +63,6 @@ on: - tests/unit/dashboard-daily-run-counts-conformance.test.mjs - tests/integration/activity-actions-runtime.test.mjs - tests/helpers/actions-runtime/** - - .github/actions/setup-cao-runtime/** - tests/e2e/dashboard-server.spec.mjs - tests/playwright/configs/dashboard-server.config.mjs - "!dashboard/site/**/*.md" diff --git a/.github/workflows/install-windows.yml b/.github/workflows/install-windows.yml index 5e26f34626..6d4537809e 100644 --- a/.github/workflows/install-windows.yml +++ b/.github/workflows/install-windows.yml @@ -4,14 +4,12 @@ on: pull_request: paths: - .github/workflows/install-windows.yml - - .github/workflows/shared/materialize-cao.mjs - install-gh-cao.sh - tests/unit/install-script.test.mjs push: branches: [main] paths: - .github/workflows/install-windows.yml - - .github/workflows/shared/materialize-cao.mjs - install-gh-cao.sh - tests/unit/install-script.test.mjs workflow_dispatch: diff --git a/.github/workflows/shared/materialize-cao.mjs b/.github/workflows/shared/materialize-cao.mjs deleted file mode 100644 index 301d1c7f55..0000000000 --- a/.github/workflows/shared/materialize-cao.mjs +++ /dev/null @@ -1,556 +0,0 @@ -import { chmodSync, cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import path from 'node:path'; -import { spawnSync } from 'node:child_process'; -import { pathToFileURL } from 'node:url'; -import { gunzipSync } from 'node:zlib'; -import { createHash } from 'node:crypto'; - -const repository = 'githubnext/gh-cao'; -const rootPackageNames = [`${repository}/control-plane`, repository]; -const runtimeWaitArray = new Int32Array(new SharedArrayBuffer(4)); -const rootResources = [ - 'dashboard', - '.github/workflows/shared/value-acquisition.mjs', - '.github/workflows/shared/actions-log.mjs', - '.github/actions/setup-cao-runtime', - '.github/actions/setup-gh-aw', - '.github/cao/instructions.md', - '.github/workflows/shared/control.md', - '.github/workflows/shared/operation.md', - '.github/workflows/shared/review-bundle.md', -]; -const retiredLauncherDigests = new Set([ - 'c55be5e923b46a247eeee6d8c2f0926883f469e9577b4ac4b99d7171972e4c4d', - 'f186ab9735fb4d2a44333a81eac9ed18638ae2c4304af37ab351cc39ecf983c3' -]); - -function removeRetiredLauncher(repositoryRoot) { - const filename = path.join(repositoryRoot, 'cao.sh'); - const entry = lstatSync(filename, { throwIfNoEntry: false }); - if (!entry?.isFile()) return; - const digest = createHash('sha256').update(readFileSync(filename)).digest('hex'); - if (retiredLauncherDigests.has(digest)) rmSync(filename); -} - -function validateCampaign(campaign) { - if (campaign === 'activity' || !/^[a-z0-9-]+$/.test(campaign)) { - throw new Error(`Invalid CAO campaign name: ${campaign}`); - } -} - -function packageName(record) { - if (typeof record.package === 'string' && record.package.trim()) return record.package.trim(); - if (typeof record.campaign === 'string' && record.campaign.trim()) return record.campaign.trim(); - if (typeof record.source === 'string') return record.source.split('@')[0].trim(); - return ''; -} - -function installedRecords(root) { - const records = []; - for (const directory of ['packages', 'campaigns']) { - const recordDirectory = path.join(root, '.github', 'aw', directory); - let entries; - try { - entries = readdirSync(recordDirectory, { withFileTypes: true }); - } catch (error) { - if (error?.code === 'ENOENT') continue; - throw error; - } - for (const entry of entries) { - if (!entry.isFile() || !entry.name.endsWith('.json')) continue; - const record = JSON.parse(readFileSync(path.join(recordDirectory, entry.name), 'utf8')); - const name = packageName(record); - if (name === repository || name.startsWith(`${repository}/`)) records.push({ name, record }); - } - } - return records; -} - -function resolvedRevision(record) { - const revision = typeof record.resolvedCommit === 'string' ? record.resolvedCommit.trim() : ''; - if (!/^[0-9a-f]{40}$/i.test(revision)) { - throw new Error(`CAO package record must contain a full resolvedCommit SHA: ${revision || '(missing)'}`); - } - return revision.toLowerCase(); -} - -function exactRecordFor(records, campaign) { - if (campaign === 'root') { - return rootPackageNames - .map((name) => records.find((candidate) => candidate.name === name)?.record) - .find(Boolean); - } - return records.find((candidate) => candidate.name === `${repository}/${campaign}`)?.record; -} - -function recordFor(records, campaign) { - const name = campaign === 'root' ? repository : `${repository}/${campaign}`; - const exact = exactRecordFor(records, campaign); - if (exact) return exact; - if (campaign === 'dashboard') { - const root = exactRecordFor(records, 'root'); - if (root) return root; - } - throw new Error(`No installed CAO package record found for ${name}`); -} - -export function installedCaoRevision(campaign = 'root', repositoryRoot = process.cwd()) { - validateCampaign(campaign); - return resolvedRevision(recordFor(installedRecords(repositoryRoot), campaign)); -} - -async function downloadArchive(revision, destination) { - const response = await fetch(`https://codeload.github.com/${repository}/tar.gz/${encodeURIComponent(revision)}`); - if (!response.ok) throw new Error(`Unable to download CAO ${revision}: HTTP ${response.status}`); - writeFileSync(destination, Buffer.from(await response.arrayBuffer())); -} - -function archiveRoot(directory) { - const entries = readdirSync(directory, { withFileTypes: true }).filter((entry) => entry.isDirectory()); - if (entries.length !== 1) throw new Error('CAO archive did not contain exactly one repository root'); - return path.join(directory, entries[0].name); -} - -function tarString(buffer, start, length) { - return buffer.subarray(start, start + length).toString('utf8').replace(/\0.*$/, ''); -} - -function tarSize(buffer) { - const value = tarString(buffer, 124, 12).trim(); - if (!/^[0-7]*$/.test(value)) throw new Error('CAO archive has an invalid entry size'); - return Number.parseInt(value || '0', 8); -} - -function tarDestination(directory, entry) { - entry = entry.replace(/\/+$/, ''); - if (!entry || entry.startsWith('/') || entry.split('/').some((part) => part === '' || part === '.' || part === '..')) { - throw new Error(`CAO archive has an unsafe entry path: ${JSON.stringify(entry)}`); - } - const destination = path.resolve(directory, ...entry.split('/')); - if (!destination.startsWith(`${path.resolve(directory)}${path.sep}`)) { - throw new Error(`CAO archive has an unsafe entry path: ${JSON.stringify(entry)}`); - } - return destination; -} - -function paxAttributes(data) { - const attributes = {}; - let offset = 0; - while (offset < data.length) { - const separator = data.indexOf(0x20, offset); - const length = Number.parseInt(data.subarray(offset, separator).toString('utf8'), 10); - if (!Number.isSafeInteger(length) || length <= 0 || offset + length > data.length) { - throw new Error('CAO archive has an invalid PAX header'); - } - const record = data.subarray(separator + 1, offset + length - 1).toString('utf8'); - const equals = record.indexOf('='); - if (equals > 0) attributes[record.slice(0, equals)] = record.slice(equals + 1); - offset += length; - } - return attributes; -} - -export function extractCaoArchive(archive, directory, native = process.platform === 'win32') { - if (!native) { - const tar = spawnSync('tar', ['-xzf', archive, '-C', directory], { encoding: 'utf8' }); - if (tar.error || tar.status !== 0) { - throw new Error((tar.stderr || tar.error?.message || 'tar failed').trim()); - } - return; - } - - const data = gunzipSync(readFileSync(archive)); - let offset = 0; - let nextPath; - while (offset + 512 <= data.length) { - const header = data.subarray(offset, offset + 512); - if (header.every((byte) => byte === 0)) break; - const size = tarSize(header); - const payloadStart = offset + 512; - const payloadEnd = payloadStart + size; - if (payloadEnd > data.length) throw new Error('CAO archive ended before an entry was complete'); - const type = String.fromCharCode(header[156] || 0); - const payload = data.subarray(payloadStart, payloadEnd); - if (type === 'x' || type === 'g') { - const attributes = paxAttributes(payload); - if (attributes.path) nextPath = attributes.path; - } else if (type === 'L') { - nextPath = payload.toString('utf8').replace(/\0.*$/, ''); - } else { - const entry = nextPath || [tarString(header, 345, 155), tarString(header, 0, 100)].filter(Boolean).join('/'); - nextPath = undefined; - const destination = tarDestination(directory, entry); - if (type === '5') { - mkdirSync(destination, { recursive: true }); - } else if (type === '\0' || type === '0') { - const encodedMode = tarString(header, 100, 8).trim(); - if (!/^[0-7]*$/.test(encodedMode)) throw new Error('CAO archive has an invalid file mode'); - const mode = 0o644 | (Number.parseInt(encodedMode || '0', 8) & 0o111); - mkdirSync(path.dirname(destination), { recursive: true }); - writeFileSync(destination, payload, { mode }); - chmodSync(destination, mode); - } else { - throw new Error(`CAO archive has unsupported entry type: ${type}`); - } - } - offset = payloadStart + Math.ceil(size / 512) * 512; - } -} - -function copyResource(sourceRoot, repositoryRoot, resource) { - const source = path.join(sourceRoot, ...resource.split('/')); - const destination = path.join(repositoryRoot, ...resource.split('/')); - rmSync(destination, { force: true, recursive: true }); - cpSync(source, destination, { force: true, recursive: true }); -} - -function copyTrackedResource(sourceRoot, repositoryRoot, resource) { - const listing = spawnSync('git', ['-C', sourceRoot, 'ls-files', '-z', '--', resource], { - encoding: 'utf8', - }); - if (listing.error || listing.status !== 0) { - throw new Error( - `Unable to list tracked CAO source files for ${resource}: ` - + `${(listing.stderr || listing.error?.message || `exit ${listing.status}`).trim()}`, - ); - } - const files = listing.stdout.split('\0').filter(Boolean); - if (files.length === 0) { - throw new Error(`CAO source resource has no tracked files: ${resource}`); - } - rmSync(path.join(repositoryRoot, ...resource.split('/')), { force: true, recursive: true }); - for (const file of files) { - const source = path.join(sourceRoot, ...file.split('/')); - const destination = path.join(repositoryRoot, ...file.split('/')); - mkdirSync(path.dirname(destination), { recursive: true }); - cpSync(source, destination, { force: true }); - } -} - -function materializeCampaignIntelligence(campaign, repositoryRoot) { - const source = path.join(repositoryRoot, campaign, 'intelligence.json'); - const destination = path.join(repositoryRoot, '.github', 'cao', 'intelligence', `${campaign}.json`); - rmSync(destination, { force: true }); - if (!existsSync(source)) return; - mkdirSync(path.dirname(destination), { recursive: true }); - cpSync(source, destination, { force: true }); -} - -function validateResources(sourceRoot, resources) { - for (const resource of resources) { - const source = path.join(sourceRoot, ...resource.split('/')); - try { - statSync(source); - } catch (error) { - if (error?.code === 'ENOENT') { - throw new Error(`CAO source revision is missing required resource: ${resource}`); - } - throw error; - } - } -} - -export function packCaoRuntime(sourceRoot, destination, { tracked = false, prepare } = {}) { - const manifestPath = path.join(sourceRoot, 'runtime', 'package.json'); - validateResources(sourceRoot, ['runtime/package.json']); - const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); - if (!Array.isArray(manifest.files) || manifest.files.some((file) => ( - typeof file !== 'string' || file.split('/').some((part) => part === '.' || part === '..') - || !rootResources.some((resource) => ( - file === resource || file.startsWith(`${resource}/`) - )) - ))) { - throw new Error('CAO runtime package must declare files within its owned runtime destinations'); - } - validateResources(sourceRoot, [...rootResources, ...manifest.files]); - const staging = path.join(destination, 'source'); - const unpacked = path.join(destination, 'unpacked'); - mkdirSync(staging, { recursive: true }); - mkdirSync(unpacked, { recursive: true }); - for (const resource of rootResources) { - if (tracked) copyTrackedResource(sourceRoot, staging, resource); - else copyResource(sourceRoot, staging, resource); - } - cpSync(manifestPath, path.join(staging, 'package.json')); - const npmArguments = [ - 'pack', '--ignore-scripts', '--json', '--pack-destination', destination, - '--cache', path.join(destination, 'cache'), - ]; - if (process.platform === 'win32') { - npmArguments.unshift(process.env.npm_execpath ?? path.join(path.dirname(process.execPath), 'node_modules', 'npm', 'bin', 'npm-cli.js')); - } - const pack = (cwd) => { - const result = spawnSync(process.platform === 'win32' ? process.execPath : 'npm', npmArguments, { - cwd, encoding: 'utf8', maxBuffer: 16 * 1024 * 1024, - }); - if (result.error || result.status !== 0) { - throw new Error(`Unable to pack CAO runtime: ${(result.stderr || result.error?.message || `exit ${result.status}`).trim()}`); - } - const [packed] = JSON.parse(result.stdout); - if (!packed?.filename || path.basename(packed.filename) !== packed.filename) { - throw new Error('npm pack did not produce a valid CAO runtime archive'); - } - return packed; - }; - let packed = pack(staging); - extractCaoArchive(path.join(destination, packed.filename), unpacked); - const packedRoot = archiveRoot(unpacked); - validateResources(packedRoot, [...rootResources, ...manifest.files]); - if (prepare) { - prepare(packedRoot); - packed = pack(packedRoot); - } - return { - sourceRoot: packedRoot, - archive: path.join(destination, packed.filename), - files: packed.files.map(({ path }) => path), - }; -} - -function runtimeFiles(directory, prefix = '') { - return readdirSync(path.join(directory, prefix), { withFileTypes: true }).flatMap((entry) => { - const file = prefix ? `${prefix}/${entry.name}` : entry.name; - if (entry.isDirectory()) return runtimeFiles(directory, file); - if (!entry.isFile()) throw new Error(`CAO runtime contains a non-regular file: ${file}`); - return [file]; - }).sort(); -} - -export function sealCaoRuntime(sourceRoot, revision) { - if (!/^[0-9a-f]{40}$/.test(revision)) throw new Error('CAO runtime requires a full lowercase revision SHA'); - const files = runtimeFiles(sourceRoot).filter((file) => file !== 'package.json' && file !== 'cao-runtime.json'); - const metadata = { - schemaVersion: 1, repository, revision, - files: Object.fromEntries(files.map((file) => [ - file, createHash('sha256').update(readFileSync(path.join(sourceRoot, file))).digest('hex'), - ])), - }; - writeFileSync(path.join(sourceRoot, 'cao-runtime.json'), `${JSON.stringify(metadata, null, 2)}\n`); - const manifestPath = path.join(sourceRoot, 'package.json'); - const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); - manifest.files = [...manifest.files, 'cao-runtime.json']; - writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); -} - -export function validateCaoRuntimeArtifact(sourceRoot, revision) { - validateResources(sourceRoot, ['cao-runtime.json', ...rootResources]); - const metadata = JSON.parse(readFileSync(path.join(sourceRoot, 'cao-runtime.json'), 'utf8')); - if (metadata.schemaVersion !== 1 || metadata.repository !== repository || metadata.revision !== revision) { - throw new Error(`CAO runtime artifact provenance does not match ${repository}@${revision}`); - } - if (!metadata.files || typeof metadata.files !== 'object' || Array.isArray(metadata.files)) { - throw new Error('CAO runtime artifact has no valid file inventory'); - } - const files = runtimeFiles(sourceRoot).filter((file) => file !== 'package.json' && file !== 'cao-runtime.json'); - if (JSON.stringify(files) !== JSON.stringify(Object.keys(metadata.files).sort())) { - throw new Error('CAO runtime artifact file inventory does not match its contents'); - } - for (const file of files) { - if (file !== 'cao.sh' && !rootResources.some((resource) => file === resource || file.startsWith(`${resource}/`))) { - throw new Error(`CAO runtime artifact contains an unowned path: ${file}`); - } - const digest = createHash('sha256').update(readFileSync(path.join(sourceRoot, file))).digest('hex'); - if (metadata.files[file] !== digest) throw new Error(`CAO runtime artifact checksum mismatch: ${file}`); - } - verifyCaoRuntime('dashboard', sourceRoot); -} - -function gh(arguments_) { - const result = spawnSync('gh', arguments_, { - encoding: 'utf8', maxBuffer: 16 * 1024 * 1024, timeout: 120_000, - }); - if (result.error || result.status !== 0) { - throw new Error(`Unable to acquire or verify prebuilt CAO runtime: ${(result.stderr || result.error?.message || `gh exited ${result.status}`).trim()}`); - } - return result.stdout; -} - -function githubApi(endpoint) { - return JSON.parse(gh(['api', '--hostname', 'github.com', '--paginate', '--slurp', endpoint])); -} - -export function downloadCaoRuntimeArtifact(record, directory, { - attempts = 60, - intervalMs = 5_000, - wait = (milliseconds) => Atomics.wait(runtimeWaitArray, 0, 0, milliseconds), -} = {}) { - const revision = resolvedRevision(record); - const name = `cao-runtime-${revision}`; - mkdirSync(directory, { recursive: true }); - const ref = typeof record.source === 'string' ? record.source.split('@')[1] : ''; - if (/^v\d+\.\d+\.\d+$/.test(ref ?? '')) { - gh(['release', 'download', ref, '--repo', `github.com/${repository}`, '--pattern', `${name}.tgz`, '--dir', directory]); - } else { - let run; - for (let attempt = 0; attempt <= attempts; attempt += 1) { - const runs = githubApi(`repos/${repository}/actions/workflows/cao-runtime.yml/runs?head_sha=${revision}&per_page=100`) - .flatMap((page) => page.workflow_runs) - .filter((candidate) => candidate.head_sha === revision - && ['push', 'release'].includes(candidate.event) - && Number.isSafeInteger(candidate.id)); - for (const candidate of runs.filter((item) => item.conclusion === 'success')) { - const artifacts = githubApi(`repos/${repository}/actions/runs/${candidate.id}/artifacts?per_page=100`) - .flatMap((page) => page.artifacts); - if (artifacts.some((artifact) => artifact.name === name && artifact.expired === false)) { - run = candidate; - break; - } - } - if (run) break; - const active = runs.some((candidate) => - ['queued', 'in_progress', 'pending', 'requested', 'waiting'].includes(candidate.status)); - if (!active) { - const failed = runs.find((candidate) => - candidate.status === 'completed' && candidate.conclusion && candidate.conclusion !== 'success'); - if (failed) { - throw new Error(`The CAO runtime build for ${revision} completed with ${failed.conclusion}. Use a revision with a successful runtime build.`); - } - throw new Error(`No successful, unexpired prebuilt CAO runtime is available for ${revision}. Wait for the CAO runtime build or select a published release; source/npm fallback is disabled.`); - } - if (attempt === attempts) { - throw new Error(`The CAO runtime build for ${revision} is still running after ${Math.ceil((attempts * intervalMs) / 60_000)} minutes. Rerun the installer after the build completes.`); - } - wait(intervalMs); - } - gh(['run', 'download', String(run.id), '--repo', `github.com/${repository}`, '--name', name, '--dir', directory]); - } - const archive = path.join(directory, `${name}.tgz`); - validateResources(directory, [`${name}.tgz`]); - gh([ - 'attestation', 'verify', archive, '--hostname', 'github.com', '--repo', repository, - '--signer-workflow', `${repository}/.github/workflows/cao-runtime.yml`, - '--source-digest', revision, '--signer-digest', revision, '--deny-self-hosted-runners', - ]); - return archive; -} - -export function materializeCaoFromSource(campaign, sourceRoot, repositoryRoot) { - validateCampaign(campaign); - const resources = campaign === 'root' ? rootResources : [campaign]; - validateResources(sourceRoot, resources); - if (campaign === 'root' || campaign === 'dashboard') { - const temporaryDirectory = mkdtempSync(path.join(tmpdir(), 'cao-pack-')); - try { - const packed = packCaoRuntime(sourceRoot, temporaryDirectory, { tracked: true }); - for (const resource of resources) copyResource(packed.sourceRoot, repositoryRoot, resource); - } finally { - rmSync(temporaryDirectory, { force: true, recursive: true }); - } - } else { - for (const resource of resources) copyTrackedResource(sourceRoot, repositoryRoot, resource); - } - if (campaign === 'root') removeRetiredLauncher(repositoryRoot); - else materializeCampaignIntelligence(campaign, repositoryRoot); - return resources; -} - -export function planCaoMaterialization(records, campaign) { - validateCampaign(campaign); - if (campaign !== 'root') { - const record = recordFor(records, campaign); - return [{ - campaign, - record, - revision: resolvedRevision(record), - resources: [campaign], - }]; - } - - const focused = ['dashboard'] - .map((focusedCampaign) => ({ - campaign: focusedCampaign, - record: exactRecordFor(records, focusedCampaign), - resources: [focusedCampaign], - })) - .filter(({ record }) => record); - const overriddenResources = new Set(focused.flatMap(({ resources }) => resources)); - return [{ - campaign: 'root', - record: recordFor(records, 'root'), - resources: rootResources.filter((resource) => !overriddenResources.has(resource)), - }, ...focused].map((plan) => ({ ...plan, revision: resolvedRevision(plan.record) })); -} - -export async function materializeCao(campaign = 'root', repositoryRoot = process.cwd()) { - const plans = planCaoMaterialization(installedRecords(repositoryRoot), campaign); - const temporaryDirectory = mkdtempSync(path.join(tmpdir(), 'cao-materialize-')); - try { - const sourceRoots = new Map(); - for (const [index, plan] of plans.entries()) { - let sourceRoot = sourceRoots.get(plan.revision); - if (!sourceRoot) { - const extractionDirectory = path.join(temporaryDirectory, String(index)); - mkdirSync(extractionDirectory); - const runtime = ['root', 'dashboard'].includes(plan.campaign); - const archive = runtime - ? downloadCaoRuntimeArtifact(plan.record, path.join(temporaryDirectory, `download-${index}`)) - : path.join(temporaryDirectory, `${index}.tar.gz`); - if (!runtime) await downloadArchive(plan.revision, archive); - try { - extractCaoArchive(archive, extractionDirectory, runtime || process.platform === 'win32'); - } catch (error) { - throw new Error(`Unable to extract CAO ${plan.revision}: ${error.message}`, { cause: error }); - } - sourceRoot = archiveRoot(extractionDirectory); - if (runtime) validateCaoRuntimeArtifact(sourceRoot, plan.revision); - sourceRoots.set(plan.revision, sourceRoot); - } - plan.sourceRoot = sourceRoot; - validateResources(sourceRoot, plan.resources); - } - for (const plan of plans) { - for (const resource of plan.resources) copyResource(plan.sourceRoot, repositoryRoot, resource); - } - if (campaign === 'root') removeRetiredLauncher(repositoryRoot); - else materializeCampaignIntelligence(campaign, repositoryRoot); - return { - campaign, - revision: plans[0].revision, - revisions: Object.fromEntries(plans.map((plan) => [plan.campaign, plan.revision])), - resources: plans.flatMap((plan) => plan.resources), - }; - } finally { - rmSync(temporaryDirectory, { force: true, recursive: true }); - } -} - -export function verifyCaoRuntime(bundle, repositoryRoot = process.cwd()) { - const required = { - dashboard: ['dashboard/site/package.json', 'dashboard/report/configure-site.mjs'], - }[bundle]; - if (!required) throw new Error(`Unknown CAO runtime bundle: ${bundle}`); - const missing = required.filter((file) => { - try { - readFileSync(path.join(repositoryRoot, ...file.split('/'))); - return false; - } catch (error) { - if (error?.code === 'ENOENT') return true; - throw error; - } - }); - if (missing.length > 0) { - throw new Error(`CAO ${bundle} runtime is incomplete at its canonical source paths: ${missing.join(', ')}`); - } -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - try { - const [command, argument = 'root', sourceRoot] = process.argv.slice(2); - if (command === 'materialize') { - const result = await materializeCao(argument); - console.log(`Materialized CAO ${result.campaign} ${result.revision}: ${result.resources.join(', ')}`); - } else if (command === 'materialize-source') { - if (!sourceRoot) throw new Error('materialize-source requires a source directory'); - const resources = materializeCaoFromSource(argument, path.resolve(sourceRoot), process.cwd()); - console.log(`Materialized CAO ${argument} from ${path.resolve(sourceRoot)}: ${resources.join(', ')}`); - } else if (command === 'verify') { - verifyCaoRuntime(argument); - } else { - throw new Error('Usage: materialize-cao.mjs '); - } - } catch (error) { - console.error(error instanceof Error ? error.message : String(error)); - process.exitCode = 1; - } -} diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index e9805e12b0..e5e227c252 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1576,9 +1576,7 @@ and basic local policy structure/compiler-pin checks, with text or JSON results and nonzero status for failed or skipped checks. Authentication may contact GitHub; raw authentication output and subprocess errors are never reported. Doctor does not replace production policy/workflow validation, grant authority, -or bootstrap database storage. Actions and Node-only tools invoke -`node activity/cao.mjs ` explicitly for canonical JSONL preparation. -There is no shell launcher. +or bootstrap database storage. There is no Node.js CLI or shell launcher. `gh cao deploy` owns both offline bundle generation (`init`, `generate`, `check`) and guarded Azure development deployment (`azure`). `internal/topology/` owns the embedded Puccini TOSCA dependency engine and provider parameter contract, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2819efb764..2446d16b2e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -307,14 +307,12 @@ the built binary with an empty `PATH` outside the checkout to enforce this bound Campaign installation through `add` and package updates through `update` pass evolving gh-aw options unchanged. Standalone workflow installation retains its bounded `--force` and `--engine` options. -Activity/Dashboard pipeline commands are deliberately not registered in Go; -their Actions, Node command modules and runtime ownership remain unchanged. +Retired Activity/Dashboard pipeline commands are deliberately not registered in Go. CI invokes the shipped JavaScript validator directly with -`node activity/cao.mjs validate --json`; it does not build or download the Go +`node .github/workflows/shared/control.mjs validate-policy`; it does not build or download the Go extension or require consumer npm metadata. User-facing commands belong in `gh cao`; Actions execution support belongs in JavaScript. Control-plane validation has one JavaScript implementation and no Go or shell-wrapper command. -Preserve its report and exit-code contracts in `tests/unit/cao-validation.test.mjs`. Retain native policy guards used by operator construction and mutation. The CI JSON artifact stays outside the checkout in `RUNNER_TEMP`. diff --git a/adr/canonical-cao-runtime-source-layout.md b/adr/canonical-cao-runtime-source-layout.md index 8a917d8c07..04187a4885 100644 --- a/adr/canonical-cao-runtime-source-layout.md +++ b/adr/canonical-cao-runtime-source-layout.md @@ -42,11 +42,10 @@ CAO still needs these properties: CAO uses the catalog source layout as its only executable runtime layout. gh-aw installs workflow sources and a bounded trusted bootstrap under supported -destinations. The bootstrap includes -`.github/workflows/shared/materialize-cao.mjs`. Official CAO installation and -update entry points invoke that materializer after gh-aw resolves the package: +destinations. The native `gh cao` extension materializes the runtime after +gh-aw resolves the package: -- the root installer materializes the root runtime on every invocation; +- `gh cao setup` stages the root runtime when the local runtime is incomplete; - `cao add` materializes the selected campaign before reading its declaration; and - `cao update` materializes each updated package before merging declarations. @@ -56,8 +55,7 @@ repository and a full 40-character `resolvedCommit`, downloads that immutable archive, and copies a bounded resource set to identical repository-relative paths. Tag-only and legacy records fail closed rather than resolving mutable provenance at installation time. Root materialization owns `activity/`, -`dashboard/`, the local runtime verification and gh-aw setup -actions, and `.github/cao/instructions.md`. If exact focused Activity or Dashboard package +`dashboard/`, the local gh-aw setup action, and `.github/cao/instructions.md`. If exact focused Activity or Dashboard package records coexist with the root record, their revisions retain ownership of their respective directories and are preflighted before any destination is replaced. Operational campaign materialization owns the complete top-level @@ -65,10 +63,8 @@ replaced. Operational campaign materialization owns the complete top-level a new revision cannot leave deleted runtime files behind. Activity and Dashboard workflows check out the control repository at -`github.workflow_sha`. They invoke the repository-local -`.github/actions/setup-cao-runtime` action, which verifies the required -canonical runtime files and fails before execution when the installation is -incomplete. Workflows do not fetch another CAO checkout or probe alternate +`github.workflow_sha` and fail before execution when the required canonical +runtime files are incomplete. Workflows do not fetch another CAO checkout or probe alternate runtime paths. `.github/aw/` is reserved for gh-aw instruction overlays, package records, @@ -149,7 +145,7 @@ repository revision. The decision is enforced by: -- installed-layout simulation in `tests/unit/cao-materialize.test.mjs`; +- runtime artifact tests in `tests/unit/cao-runtime-artifact.test.mjs`; - Activity and Dashboard workflow contract tests; - campaign, CLI, installer, dashboard build, and local-server tests; - searches that reject CAO runtime paths under `.github/aw/`; and diff --git a/aw.yml b/aw.yml index 42b24e2297..d8a7ab77b0 100644 --- a/aw.yml +++ b/aw.yml @@ -28,8 +28,6 @@ resources: destination: .github/workflows/shared/operations.mjs - source: .github/workflows/shared/github-apps.mjs destination: .github/workflows/shared/github-apps.mjs - - source: .github/workflows/shared/materialize-cao.mjs - destination: .github/workflows/shared/materialize-cao.mjs - source: .github/aw/instructions.md destination: .github/aw/instructions.md - source: .github/workflows/shared/value-acquisition.mjs diff --git a/install-gh-cao.sh b/install-gh-cao.sh index 6db36df4c4..d3f8d20f1c 100755 --- a/install-gh-cao.sh +++ b/install-gh-cao.sh @@ -585,10 +585,7 @@ upgrade_gh_aw() { # Bash (curl | bash), child processes must not consume the unread script lines. main() { local policy_path=".github/workflows/cao.json" - local dashboard_runtime="dashboard/report/configure-site.mjs" local control_runtime=".github/workflows/shared/control.mjs" - local materializer=".github/workflows/shared/materialize-cao.mjs" - local runtime_action=".github/actions/setup-cao-runtime/action.yml" local required_gh="v2.68.0" local required_gh_aw="v1.2.2" local catalog_source="githubnext/gh-cao" @@ -789,7 +786,7 @@ main() { # gh-aw's markdown scanner rejects the GitHub App manifest page embedded in # .github/workflows/shared/github-apps.mjs, so catalog installs skip it. - if [[ ! -f "$dashboard_runtime" || ! -f "$control_runtime" || ! -f "$materializer" || ! -f "$runtime_action" ]]; then + if [[ ! -f "$control_runtime" ]]; then note "Installs 3 workflows, 5 shared control plane resources, and project settings." if ! run_quiet_command \ "Installing control plane components..." \ @@ -810,24 +807,6 @@ main() { else note "CAO control plane components are already installed." fi - if [[ -d "$catalog_source" ]]; then - if ! run_quiet_command \ - "Materializing CAO runtime..." \ - "CAO runtime materialized and verified." \ - node "$materializer" materialize-source root "$catalog_source"; then - [[ -z "$quiet_command_output" ]] || printf '%s\n' "$quiet_command_output" >&2 - return 1 - fi - else - note "A newly published catalog revision may wait for its exact verified runtime build." - if ! run_quiet_command \ - "Verifying exact CAO runtime..." \ - "CAO runtime materialized and verified." \ - node "$materializer" materialize root; then - [[ -z "$quiet_command_output" ]] || printf '%s\n' "$quiet_command_output" >&2 - return 1 - fi - fi if [[ ! -f "$policy_path" ]]; then if ! run_quiet_command \ diff --git a/internal/cli/native_management_runtime.go b/internal/cli/native_management_runtime.go index 591d3cd004..890edfeef0 100644 --- a/internal/cli/native_management_runtime.go +++ b/internal/cli/native_management_runtime.go @@ -20,7 +20,7 @@ import ( var regexpStableVersion = regexp.MustCompile(`^v[0-9]+\.[0-9]+\.[0-9]+$`) var nativeRuntimeResources = []string{ - "dashboard", ".github/workflows/shared/value-acquisition.mjs", ".github/workflows/shared/actions-log.mjs", ".github/actions/setup-cao-runtime", ".github/actions/setup-gh-aw", + "dashboard", ".github/workflows/shared/value-acquisition.mjs", ".github/workflows/shared/actions-log.mjs", ".github/actions/setup-gh-aw", ".github/cao/instructions.md", ".github/workflows/shared/control.md", ".github/workflows/shared/review-bundle.md", ".github/workflows/shared/operation.md", diff --git a/internal/cli/native_management_runtime_test.go b/internal/cli/native_management_runtime_test.go index 7b75b74452..c456d7e8cf 100644 --- a/internal/cli/native_management_runtime_test.go +++ b/internal/cli/native_management_runtime_test.go @@ -95,7 +95,7 @@ func nativeRuntimeArchiveFixture(t *testing.T, revision string, corrupt bool) st if name == "activity" || name == "dashboard" { continue } - if name == ".github/actions/setup-cao-runtime" || name == ".github/actions/setup-gh-aw" { + if name == ".github/actions/setup-gh-aw" { name += "/action.yml" } files[name] = "resource: " + name diff --git a/internal/cli/root.go b/internal/cli/root.go index ea8f5c06b4..a9e7b48521 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -34,7 +34,6 @@ func newCommand(version string, ghVersion func(context.Context) (string, error), All operator commands run natively in Go; Node.js is not required. The binary embeds the reviewed dashboard catalog and queries canonical PostgreSQL. -Activity and Dashboard Actions invoke node activity/cao.mjs directly. Help, version, and catalog discovery work offline.`, Version: version, Args: cobra.NoArgs, diff --git a/internal/cli/setup_runtime.go b/internal/cli/setup_runtime.go index e7c2cd2639..5bbd2f3af7 100644 --- a/internal/cli/setup_runtime.go +++ b/internal/cli/setup_runtime.go @@ -24,7 +24,6 @@ var setupRuntimeFiles = []string{ ".github/workflows/shared/policy.mjs", ".github/workflows/shared/authority.mjs", ".github/workflows/shared/operations.mjs", - ".github/actions/setup-cao-runtime/action.yml", ".github/actions/setup-gh-aw/action.yml", } diff --git a/runtime/package.json b/runtime/package.json index 36a57028de..fba4ac103d 100644 --- a/runtime/package.json +++ b/runtime/package.json @@ -42,7 +42,6 @@ "dashboard/site/scripts/prepare-terminal.mjs", "dashboard/site/scripts/llms.mjs", "dashboard/site/service-worker.js", - ".github/actions/setup-cao-runtime", ".github/actions/setup-gh-aw", ".github/cao/instructions.md", ".github/workflows/shared/control.md", diff --git a/scripts/build-cao-runtime.mjs b/scripts/build-cao-runtime.mjs index 6b8cf5ddeb..b4eda6632f 100644 --- a/scripts/build-cao-runtime.mjs +++ b/scripts/build-cao-runtime.mjs @@ -1,45 +1,171 @@ import { buildSync } from "esbuild"; -import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { cpSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; import { createHash } from "node:crypto"; import { tmpdir } from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; import { createRequire } from "node:module"; -import { packCaoRuntime, sealCaoRuntime, validateCaoRuntimeArtifact } from "../.github/workflows/shared/materialize-cao.mjs"; + +const repository = "githubnext/gh-cao"; +// Keep aligned with nativeRuntimeResources in internal/cli/native_management_runtime.go. +export const runtimeResources = [ + "dashboard", + ".github/workflows/shared/value-acquisition.mjs", + ".github/workflows/shared/actions-log.mjs", + ".github/actions/setup-gh-aw", + ".github/cao/instructions.md", + ".github/workflows/shared/control.md", + ".github/workflows/shared/operation.md", + ".github/workflows/shared/review-bundle.md", +]; +const requiredRuntimeFiles = ["dashboard/site/package.json", "dashboard/report/configure-site.mjs"]; + +function run(command, arguments_, options = {}) { + const result = spawnSync(command, arguments_, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024, ...options }); + if (result.error || result.status !== 0) { + throw new Error(`${command} ${arguments_[0]} failed: ${(result.stderr || result.error?.message || `exit ${result.status}`).trim()}`); + } + return result.stdout; +} + +function requireResources(root, resources) { + for (const resource of resources) { + try { + statSync(path.join(root, ...resource.split("/"))); + } catch (error) { + if (error?.code === "ENOENT") throw new Error(`CAO runtime is missing required resource: ${resource}`); + throw error; + } + } +} + +function owned(file) { + return runtimeResources.some((resource) => file === resource || file.startsWith(`${resource}/`)); +} + +function copyTrackedResource(sourceRoot, destinationRoot, resource) { + const files = run("git", ["-C", sourceRoot, "ls-files", "-z", "--", resource]).split("\0").filter(Boolean); + if (files.length === 0) throw new Error(`CAO source resource has no tracked files: ${resource}`); + for (const file of files) { + const destination = path.join(destinationRoot, ...file.split("/")); + mkdirSync(path.dirname(destination), { recursive: true }); + cpSync(path.join(sourceRoot, ...file.split("/")), destination); + } +} + +function npmPack(cwd, destination) { + const arguments_ = ["pack", "--ignore-scripts", "--json", "--pack-destination", destination, "--cache", path.join(destination, "cache")]; + const [command, prefix] = process.platform === "win32" + ? [process.execPath, [process.env.npm_execpath ?? path.join(path.dirname(process.execPath), "node_modules", "npm", "bin", "npm-cli.js")]] + : ["npm", []]; + const [packed] = JSON.parse(run(command, [...prefix, ...arguments_], { cwd })); + if (!packed?.filename || path.basename(packed.filename) !== packed.filename) { + throw new Error("npm pack did not produce a valid CAO runtime archive"); + } + return packed; +} + +function runtimeFiles(directory, prefix = "") { + return readdirSync(path.join(directory, prefix), { withFileTypes: true }).flatMap((entry) => { + const file = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) return runtimeFiles(directory, file); + if (!entry.isFile()) throw new Error(`CAO runtime contains a non-regular file: ${file}`); + return [file]; + }).sort(); +} + +function inventoryFiles(root) { + return runtimeFiles(root).filter((file) => file !== "package.json" && file !== "cao-runtime.json"); +} + +function sha256(filename) { + return createHash("sha256").update(readFileSync(filename)).digest("hex"); +} + +function sealCaoRuntime(root, revision) { + const metadata = { + schemaVersion: 1, repository, revision, + files: Object.fromEntries(inventoryFiles(root).map((file) => [file, sha256(path.join(root, file))])), + }; + writeFileSync(path.join(root, "cao-runtime.json"), `${JSON.stringify(metadata, null, 2)}\n`); + const manifestPath = path.join(root, "package.json"); + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); + manifest.files = [...manifest.files, "cao-runtime.json"]; + writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); +} + +export function validateCaoRuntimeArtifact(root, revision) { + requireResources(root, ["cao-runtime.json", ...runtimeResources]); + const metadata = JSON.parse(readFileSync(path.join(root, "cao-runtime.json"), "utf8")); + if (metadata.schemaVersion !== 1 || metadata.repository !== repository || metadata.revision !== revision) { + throw new Error(`CAO runtime artifact provenance does not match ${repository}@${revision}`); + } + if (!metadata.files || typeof metadata.files !== "object" || Array.isArray(metadata.files)) { + throw new Error("CAO runtime artifact has no valid file inventory"); + } + const files = inventoryFiles(root); + if (JSON.stringify(files) !== JSON.stringify(Object.keys(metadata.files).sort())) { + throw new Error("CAO runtime artifact file inventory does not match its contents"); + } + for (const file of files) { + if (!owned(file)) throw new Error(`CAO runtime artifact contains an unowned path: ${file}`); + if (metadata.files[file] !== sha256(path.join(root, file))) throw new Error(`CAO runtime artifact checksum mismatch: ${file}`); + } + requireResources(root, requiredRuntimeFiles); +} + +function vendorYaml(sourceRoot, root) { + const vendor = path.join(root, "dashboard/site/src/vendor"); + mkdirSync(vendor, { recursive: true }); + buildSync({ + absWorkingDir: sourceRoot, + stdin: { contents: 'export { parseAllDocuments } from "yaml";', resolveDir: sourceRoot }, + outfile: path.join(vendor, "yaml.js"), + bundle: true, format: "esm", platform: "neutral", target: "es2022", legalComments: "eof", + }); + const require = createRequire(path.join(sourceRoot, "package.json")); + cpSync(path.join(path.dirname(require.resolve("yaml/package.json")), "LICENSE"), path.join(vendor, "yaml.LICENSE")); + const consumer = path.join(root, "dashboard/site/src/validator-dashboard.js"); + const source = readFileSync(consumer, "utf8"); + const compiled = source.replace("from 'yaml';", "from './vendor/yaml.js';"); + if (compiled === source) throw new Error("Dashboard YAML dependency changed; runtime bundling must be updated"); + writeFileSync(consumer, compiled); +} export function buildCaoRuntime(sourceRoot, destination, revision) { if (!/^[0-9a-f]{40}$/.test(revision)) throw new Error("A full lowercase source revision SHA is required"); + const manifestPath = path.join(sourceRoot, "runtime", "package.json"); + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); + if (!Array.isArray(manifest.files) || manifest.files.some((file) => ( + typeof file !== "string" || file.split("/").some((part) => part === "." || part === "..") || !owned(file) + ))) { + throw new Error("CAO runtime package must declare files within its owned runtime destinations"); + } + requireResources(sourceRoot, [...runtimeResources, ...manifest.files]); const temporary = mkdtempSync(path.join(tmpdir(), "cao-runtime-build-")); try { - const packed = packCaoRuntime(sourceRoot, temporary, { - tracked: true, - prepare(root) { - const vendor = path.join(root, "dashboard/site/src/vendor"); - mkdirSync(vendor, { recursive: true }); - buildSync({ - absWorkingDir: sourceRoot, - stdin: { contents: 'export { parseAllDocuments } from "yaml";', resolveDir: sourceRoot }, - outfile: path.join(vendor, "yaml.js"), - bundle: true, format: "esm", platform: "neutral", target: "es2022", legalComments: "eof", - }); - const require = createRequire(path.join(sourceRoot, "package.json")); - cpSync(path.join(path.dirname(require.resolve("yaml/package.json")), "LICENSE"), path.join(vendor, "yaml.LICENSE")); - const consumer = path.join(root, "dashboard/site/src/validator-dashboard.js"); - const source = readFileSync(consumer, "utf8"); - const compiled = source.replace("from 'yaml';", "from './vendor/yaml.js';"); - if (compiled === source) throw new Error("Dashboard YAML dependency changed; runtime bundling must be updated"); - writeFileSync(consumer, compiled); - sealCaoRuntime(root, revision); - }, - }); - validateCaoRuntimeArtifact(packed.sourceRoot, revision); + const staging = path.join(temporary, "source"); + const unpacked = path.join(temporary, "unpacked"); + mkdirSync(staging, { recursive: true }); + mkdirSync(unpacked, { recursive: true }); + for (const resource of runtimeResources) copyTrackedResource(sourceRoot, staging, resource); + cpSync(manifestPath, path.join(staging, "package.json")); + const selected = npmPack(staging, temporary); + run("tar", ["-xzf", path.join(temporary, selected.filename), "-C", unpacked]); + const root = path.join(unpacked, "package"); + requireResources(root, [...runtimeResources, ...manifest.files]); + vendorYaml(sourceRoot, root); + sealCaoRuntime(root, revision); + const packed = npmPack(root, temporary); + validateCaoRuntimeArtifact(root, revision); mkdirSync(destination, { recursive: true }); const name = `cao-runtime-${revision}.tgz`; const archive = path.join(destination, name); - cpSync(packed.archive, archive); - const digest = createHash("sha256").update(readFileSync(archive)).digest("hex"); + cpSync(path.join(temporary, packed.filename), archive); + const digest = sha256(archive); writeFileSync(`${archive}.sha256`, `${digest} ${name}\n`); - return { archive, digest, files: packed.files }; + return { archive, digest, files: packed.files.map(({ path: file }) => file) }; } finally { rmSync(temporary, { recursive: true, force: true }); } diff --git a/scripts/campaign-lifecycle-matrix.mjs b/scripts/campaign-lifecycle-matrix.mjs index a6bfa6d3b6..5d1688b081 100644 --- a/scripts/campaign-lifecycle-matrix.mjs +++ b/scripts/campaign-lifecycle-matrix.mjs @@ -12,7 +12,6 @@ const suites = [ prefixes: [ "dashboard/", "install-gh-cao.sh", - ".github/actions/setup-cao-runtime/", ".github/actions/setup-gh-aw/", "cao-evolution/.github/graders/", ".github/workflows/graders/cao-evolution-", diff --git a/scripts/validate-cao-runtime.mjs b/scripts/validate-cao-runtime.mjs index 1f55fdc1dc..8749b2c3b0 100644 --- a/scripts/validate-cao-runtime.mjs +++ b/scripts/validate-cao-runtime.mjs @@ -24,7 +24,6 @@ const checks = [ command: process.execPath, arguments: [ "--test", - "tests/unit/cao-materialize.test.mjs", "tests/unit/cao-runtime-artifact.test.mjs", ], }, @@ -34,7 +33,7 @@ const checks = [ arguments: [ "--test", "--test-name-pattern", - "^(installer fixture archive contains every tracked runtime source and no generated files|streamed install\\.sh with .* materializes the runtime and initializes policy)$", + "^(installer fixture archive contains every tracked runtime source and no generated files|streamed install-gh-cao\\.sh with .* installs the control plane and initializes policy)$", "tests/unit/install-script.test.mjs", ], }, diff --git a/server/internal/simulator/azure_journey_test.go b/server/internal/simulator/azure_journey_test.go index 62b1183251..bc730e8a64 100644 --- a/server/internal/simulator/azure_journey_test.go +++ b/server/internal/simulator/azure_journey_test.go @@ -48,7 +48,7 @@ func setupAzurePrivateFarm(t *testing.T, ctx context.Context, f *setupFixture, c } installerEnv := setupAzureEnvironment(f.env, map[string]string{"PATH": os.Getenv("PATH")}) bootstrapRun(t, ctx, f.checkout, installerEnv, "bash", filepath.Join(catalog, "install-gh-cao.sh"), catalog) - for _, name := range []string{"dashboard/report/configure-site.mjs", ".github/workflows/cao-validate.yml"} { + for _, name := range []string{".github/workflows/cao-validate.yml"} { if len(setupRead(t, filepath.Join(f.checkout, name))) == 0 { t.Fatalf("installer omitted %s", name) } diff --git a/server/internal/simulator/bootstrap_compatibility_integration_test.go b/server/internal/simulator/bootstrap_compatibility_integration_test.go index f4f3c95217..de7bd9c9d2 100644 --- a/server/internal/simulator/bootstrap_compatibility_integration_test.go +++ b/server/internal/simulator/bootstrap_compatibility_integration_test.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "net/url" "os" "os/exec" "path/filepath" @@ -16,7 +15,7 @@ import ( // The upstream installer and version-only extension are inert fixtures. The // streamed CAO installer, gh extension discovery, native CAO binary, checksum -// verification and source materializer still execute as real subprocesses. +// verification still execute as real subprocesses. const bootstrapAWInstaller = `#!/usr/bin/env bash set -euo pipefail [[ -z "${GH_TOKEN:-}${GITHUB_TOKEN:-}${GH_ENTERPRISE_TOKEN:-}${GITHUB_ENTERPRISE_TOKEN:-}${INPUT_VERSION:-}" ]] || exit 65 @@ -58,25 +57,6 @@ func testBootstrapInstallerCompatibility(t *testing.T, ctx context.Context, cata t.Error(err) } }() - // This fixture was installed from local source, not an attested Actions - // artifact. Keep materialization on that same production source path; - // artifact retrieval/provenance is a separate runtime-installation suite. - materializer := filepath.Join(consumer, ".github/workflows/shared/materialize-cao.mjs") - materializerBefore := setupRead(t, materializer) - defer setupWrite(t, materializer, materializerBefore) - module := (&url.URL{Scheme: "file", Path: filepath.ToSlash(filepath.Join(catalog, ".github/workflows/shared/materialize-cao.mjs"))}).String() - moduleJSON, err := json.Marshal(module) - if err != nil { - t.Fatal(err) - } - catalogJSON, err := json.Marshal(catalog) - if err != nil { - t.Fatal(err) - } - setupWrite(t, materializer, fmt.Appendf(nil, `import { materializeCaoFromSource } from %s; -if (process.argv.slice(2).join(" ") !== "materialize root") throw new Error("unexpected materialization request"); -materializeCaoFromSource("root", %s, process.cwd()); -`, moduleJSON, catalogJSON)) policy := setupRead(t, filepath.Join(consumer, ".github/workflows/cao.json")) tree := setupCheckoutState(t, consumer) index := bootstrapRun(t, ctx, consumer, environment, "git", "ls-files", "--stage") diff --git a/server/internal/simulator/bootstrap_integration_test.go b/server/internal/simulator/bootstrap_integration_test.go index bf089b6ea5..d5bfd6c672 100644 --- a/server/internal/simulator/bootstrap_integration_test.go +++ b/server/internal/simulator/bootstrap_integration_test.go @@ -94,7 +94,6 @@ func TestCAOBootstrapIntegration(t *testing.T) { bootstrapAssertRepository(t, api, true) for _, name := range []string{ ".github/workflows/cao-validate.yml", ".github/workflows/shared/control.mjs", - ".github/actions/setup-cao-runtime/action.yml", "dashboard/report/configure-site.mjs", ".github/workflows/shared/materialize-cao.mjs", } { if info, err := os.Stat(filepath.Join(consumer, name)); err != nil || info.Size() == 0 { t.Fatalf("missing or empty installed resource %s: %v", name, err) diff --git a/specs/repository-layout-and-installation.md b/specs/repository-layout-and-installation.md index 786b6e5d89..71a60369f3 100644 --- a/specs/repository-layout-and-installation.md +++ b/specs/repository-layout-and-installation.md @@ -53,8 +53,6 @@ control repository, or more than one of those roles. | `specs/dashboard-data.md` | CAO catalog or Agent Plugin installer | Canonical data specification bundled with the separate Dashboard extension | | `.github/workflows/` | Control repository and gh-aw | Workflow sources, generated locks, and CAO policy | | `.github/workflows/cao.json` | Control repository | Persistent non-secret rollout policy | -| `.github/workflows/shared/materialize-cao.mjs` | CAO bootstrap | Trusted canonical-runtime materializer | -| `.github/actions/setup-cao-runtime/` | CAO bootstrap | Local runtime verification action | | `.github/actions/setup-gh-aw/` | CAO bootstrap | Local policy-pinned gh-aw setup action | | `.github/cao/instructions.md` | Control repository | Optional CAO operator guidance | | `.github/aw/` | gh-aw | Instructions, campaign records, ownership metadata, and gh-aw runtime data | @@ -120,8 +118,9 @@ closed, without latest-release or install-time source/npm fallback. Operational campaigns retain source-archive acquisition by their immutable revision. Workflow execution MUST NOT fetch a branch, tag, or alternate runtime -layout. Local developer source materialization is a separate, explicit path -that may use npm packing. +layout. Runtime artifacts are built from a catalog checkout by +`scripts/build-cao-runtime.mjs`; there is no install-time source materialization +path. When the root record and exact focused Activity or Dashboard records coexist, the focused record MUST own its corresponding canonical directory. The root @@ -130,8 +129,7 @@ preflighted before the first replacement. ### 4.2 Replacement -Root materialization owns `activity/`, `dashboard/`, -`.github/actions/setup-cao-runtime/`, `.github/actions/setup-gh-aw/`, and +Root materialization owns `activity/`, `dashboard/`, `.github/actions/setup-gh-aw/`, and `.github/cao/instructions.md`. Every repository-local action referenced by a workflow that the root campaign installs MUST be provided at its canonical path by the root installation; installed workflow files MUST NOT be rewritten to @@ -255,14 +253,16 @@ from this runtime boundary. Control-plane validation MUST have one JavaScript implementation, without a Go or shell-wrapper command. Its workflow MUST execute the shipped JavaScript runtime directly with -`node activity/cao.mjs validate --json` at its installed revision, using the +`node .github/workflows/shared/control.mjs validate-policy` at its installed revision, using the production policy resolver. It MUST NOT build or download the Go extension or require consumer npm metadata. Runtime tests MUST preserve the structured report and exit-code contract. CI MUST keep its report under `RUNNER_TEMP`, not in the repository working tree. -The supported root CAO installer MUST materialize the root runtime after gh-aw -has resolved the root package record. For a main or SHA source, the +Native `gh cao setup` MUST stage the root runtime from the exact attested +artifact for the resolved root package record when the local runtime is +incomplete; the root installer installs the control plane and leaves runtime +materialization to `gh cao setup`. For a main or SHA source, the materializer MUST wait at most five minutes when the exact runtime build is queued or running, then fail closed if it does not complete successfully; it MUST NOT substitute another revision or build from source. `cao add` MUST materialize a selected @@ -409,9 +409,8 @@ component installation as a complete CAO installation. ## 5. Workflow execution and verification Activity and Dashboard workflows MUST check out the control repository at -`github.workflow_sha`. They MUST use the repository-local -`$/.github/actions/setup-cao-runtime` action to verify their required canonical -runtime bundle before executing it. +`github.workflow_sha` and verify their required canonical runtime bundle +before executing it. Runtime verification MUST be local and fail closed when a required canonical file is absent. A workflow MUST NOT download another CAO checkout, use a diff --git a/tests/e2e/dashboard-massive-scale.spec.mjs b/tests/e2e/dashboard-massive-scale.spec.mjs index 4a9d9a66b6..7d69003be5 100644 --- a/tests/e2e/dashboard-massive-scale.spec.mjs +++ b/tests/e2e/dashboard-massive-scale.spec.mjs @@ -6,6 +6,7 @@ import { createServer } from "node:http"; import { tmpdir } from "node:os"; import { extname, join, resolve, sep } from "node:path"; import { once } from "node:events"; +import { pathToFileURL } from "node:url"; const siteRoot = resolve("dashboard/site"); const repositories = numberSetting("DASHBOARD_STRESS_REPOSITORIES", 10_000); @@ -127,12 +128,18 @@ test.beforeAll(async () => { ]); manifest = JSON.parse(generationMemory.stdout); await runMeasuredCommand(process.execPath, [ - resolve("activity/cao.mjs"), - "hash-payloads", - "--shard-dir", rawShardDirectory, - "--runs-dir", runsDirectory, - "--records-dir", recordsDirectory, - "--output", manifestPath, + "--input-type=module", + "--eval", + `import { writeFile } from "node:fs/promises"; +const { normalizeSnapshotPayloads } = await import(process.argv[1]); +const [shardDirectory, runsDirectory, recordsDirectory, output] = process.argv.slice(2); +const hashes = await normalizeSnapshotPayloads({ shardDirectory, runsDirectory, recordsDirectory }); +await writeFile(output, JSON.stringify(hashes, null, 2) + "\\n");`, + pathToFileURL(resolve("dashboard/report/normalized-payloads.mjs")).href, + rawShardDirectory, + runsDirectory, + recordsDirectory, + manifestPath, ]); payloadHashes = JSON.parse(await readFile(manifestPath, "utf8")); diff --git a/tests/integration/campaign-lifecycle.test.mjs b/tests/integration/campaign-lifecycle.test.mjs index 3cbaf28128..2a504aae1d 100644 --- a/tests/integration/campaign-lifecycle.test.mjs +++ b/tests/integration/campaign-lifecycle.test.mjs @@ -20,7 +20,6 @@ const campaignUpdateSource = "https://github.com/githubnext/gh-cao"; // differ from the catalog because gh aw refuses to add workflows into their // own source repository; the consumer is never pushed. const consumerRepository = "octocat/Hello-World"; -const materializerScript = resolve(".github/workflows/shared/materialize-cao.mjs"); const installerSource = readFileSync(resolve("install-gh-cao.sh"), "utf8"); const controlRuntimeFiles = [ ".github/workflows/shared/control.mjs", @@ -53,39 +52,13 @@ const optimizationCampaignSource = focusedCampaignSource("optimization"); const repoStatusWorkflowSource = `https://github.com/${campaignSource.slice(0, campaignSource.lastIndexOf("@"))}/blob/${campaignSource.slice(campaignSource.lastIndexOf("@") + 1)}/.github/workflows/daily-repo-status.md`; const selfCareCampaignSource = focusedCampaignSource("self-care"); const softwareDevelopmentPracticesCampaignSource = focusedCampaignSource("software-development-practices"); -const activityExpectedFiles = [ - "activity/actions-context.mjs", +const runtimeExpectedFiles = [ ".github/workflows/shared/actions-log.mjs", - "activity/campaign-intelligence.mjs", - "activity/cao.mjs", "dashboard/report/collections.mjs", - "activity/commands/audit-jsonl.mjs", - "activity/commands/cluster-problems.mjs", - "activity/commands/compact-jsonl.mjs", - "activity/commands/dashboard-complexity.mjs", - "activity/commands/discover-workflows.mjs", - "activity/commands/repair-database.mjs", - "activity/commands/download.mjs", - "activity/commands/hash-payloads.mjs", - "activity/commands/index.mjs", - "activity/commands/ingest-jsonl.mjs", - "activity/commands/ingest.mjs", - "activity/commands/operational-value.mjs", - "activity/commands/prune-dashboard.mjs", - "activity/collect-workflow-logs.mjs", - "activity/computations/intelligence-contracts.mjs", - "activity/computations/intelligence.mjs", - "activity/computations/index.mjs", - "activity/computations/runtime-health.mjs", - "activity/control-settings.mjs", "dashboard/report/debug.mjs", - "activity/gh-aw-logs.mjs", - "activity/inventory.mjs", "dashboard/report/inventory-sources.mjs", "dashboard/report/package-manifest.mjs", "dashboard/report/normalized-phase.mjs", - "activity/repository-visibility.mjs", - "activity/token-intervention-lifecycle.mjs", "dashboard/lib/version.mjs", ]; const caoEvolutionExpectedFiles = [ @@ -208,6 +181,7 @@ async function installCampaign(source) { if (campaign === "root") { run("git", ["remote", "add", "origin", `https://github.com/${consumerRepository}.git`], consumer); run("bash", ["-s", "--", source], consumer, installerSource); + materializeRuntime("root", consumer); return consumer; } run("gh", [ @@ -218,7 +192,7 @@ async function installCampaign(source) { "--no-security-scanner", ], consumer); if (campaign !== "activity" && campaign !== "dashboard") { - run(process.execPath, [materializerScript, "materialize", campaign], consumer); + materializeRuntime(campaign, consumer); } return consumer; } catch (error) { @@ -228,12 +202,15 @@ async function installCampaign(source) { }); } +function materializeRuntime(campaign, cwd) { + run("gh", ["cao", "update", "--campaign", campaign, "--no-security-scanner"], cwd); +} + function assertInstalledBootstrap(repository, ghAwVersion, controlRepository) { const policy = ".github/workflows/cao.json"; const control = ".github/workflows/shared/control.mjs"; - run(process.execPath, ["activity/cao.mjs", "--help"], repository); - for (const bundle of ["activity", "dashboard"]) { - run(process.execPath, [".github/workflows/shared/materialize-cao.mjs", "verify", bundle], repository); + for (const file of ["dashboard/site/package.json", "dashboard/report/configure-site.mjs"]) { + assert.ok(existsSync(join(repository, file)), `CAO runtime omitted ${file}`); } run(process.execPath, [control, "validate-policy", policy], repository); assert.equal(run(process.execPath, [control, "compiler-version", policy], repository).trim(), ghAwVersion); @@ -344,8 +321,8 @@ test("root campaign bootstraps an empty CAO and preserves resources during workf readdirSync(join(consumer, ".github", "workflows")).filter((name) => name.endsWith(".lock.yml")), [], ); - for (const relativePath of activityExpectedFiles) { - assert.ok(existsSync(join(consumer, relativePath)), `root campaign omitted activity file ${relativePath}`); + for (const relativePath of runtimeExpectedFiles) { + assert.ok(existsSync(join(consumer, relativePath)), `root campaign omitted runtime file ${relativePath}`); } for (const relativePath of dashboardExpectedFiles) { assert.ok(existsSync(join(consumer, relativePath)), `root campaign omitted dashboard file ${relativePath}`); @@ -623,7 +600,7 @@ test("root CAO rematerialization restores dashboard workflows, producers, and re "--force", "--no-security-scanner", ], consumer); - run(process.execPath, [materializerScript, "materialize", "root"], consumer); + materializeRuntime("root", consumer); assert.ok( !readFileSync(deployPath, "utf8").includes("# local integration-test change"), @@ -677,7 +654,7 @@ test("gh aw update replaces workflows and restores campaign-owned assets after c "--cool-down", "0", ], consumer); - run(process.execPath, [materializerScript, "materialize", "dependabot"], consumer); + materializeRuntime("dependabot", consumer); const updatedOrchestrator = readFileSync(orchestratorPath, "utf8"); assert.ok( diff --git a/tests/integration/cao-runtime-installation.test.mjs b/tests/integration/cao-runtime-installation.test.mjs index f0f11d2a6f..082aac4909 100644 --- a/tests/integration/cao-runtime-installation.test.mjs +++ b/tests/integration/cao-runtime-installation.test.mjs @@ -6,8 +6,7 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; import test from "node:test"; import { parse } from "yaml"; -import { extractCaoArchive, planCaoMaterialization, validateCaoRuntimeArtifact } from "../../.github/workflows/shared/materialize-cao.mjs"; -import { buildCaoRuntime } from "../../scripts/build-cao-runtime.mjs"; +import { buildCaoRuntime, runtimeResources, validateCaoRuntimeArtifact } from "../../scripts/build-cao-runtime.mjs"; const catalog = path.resolve(import.meta.dirname, "../.."); @@ -39,7 +38,7 @@ test("a producer archive runs and builds the dashboard from a fresh documented i const artifact = buildCaoRuntime(catalog, path.join(temporary, "artifacts"), revision); const unpacked = path.join(temporary, "unpacked"); mkdirSync(unpacked); - extractCaoArchive(artifact.archive, unpacked, true); + run("tar", ["-xzf", artifact.archive, "-C", unpacked], temporary); const packedRoot = path.join(unpacked, "package"); validateCaoRuntimeArtifact(packedRoot, revision); const manifest = parse(readFileSync(path.join(catalog, "aw.yml"), "utf8")); @@ -52,10 +51,7 @@ test("a producer archive runs and builds the dashboard from a fresh documented i path.join(catalog, ".github/workflows/aw.json"), path.join(installed, ".github/workflows/aw.json"), ); - const [{ resources }] = planCaoMaterialization([{ - name: "githubnext/gh-cao", record: { resolvedCommit: revision }, - }], "root"); - for (const resource of resources) { + for (const resource of runtimeResources) { const target = path.join(installed, resource); mkdirSync(path.dirname(target), { recursive: true }); cpSync(path.join(packedRoot, resource), target, { recursive: true }); @@ -74,8 +70,8 @@ test("a producer archive runs and builds the dashboard from a fresh documented i "commit", "--quiet", "-m", "Install CAO runtime", ], installed); run("git", ["clone", "--quiet", "--no-local", installed, checkout], temporary); - for (const bundle of ["dashboard"]) { - run(process.execPath, [".github/workflows/shared/materialize-cao.mjs", "verify", bundle], checkout); + for (const file of ["dashboard/site/package.json", "dashboard/report/configure-site.mjs"]) { + assert.ok(existsSync(path.join(checkout, file)), `missing runtime file: ${file}`); } assert.equal(existsSync(path.join(checkout, "activity")), false); assert.equal(existsSync(path.join(checkout, "dashboard/site/src/data/storage/sqlite-indexeddb.js")), false); diff --git a/tests/integration/install-script.test.mjs b/tests/integration/install-script.test.mjs index ea590fb70e..747c69068e 100644 --- a/tests/integration/install-script.test.mjs +++ b/tests/integration/install-script.test.mjs @@ -206,5 +206,4 @@ exec "$CAO_INSTALLER_GH" "$@" assert.equal(await readFile(path.join(home, ".curlrc"), "utf8"), curlrc); assert.match(await readFile(log, "utf8"), /auth status\n[\s\S]*repo view --json nameWithOwner/); assert.deepEqual((await readFile(metadataLog, "utf8")).trim().split("\n"), Object.keys(responses)); - await run(process.execPath, [".github/workflows/shared/materialize-cao.mjs", "verify", "dashboard"], { cwd: consumer, env }); }); diff --git a/tests/unit/cao-materialize.test.mjs b/tests/unit/cao-materialize.test.mjs deleted file mode 100644 index 1968460357..0000000000 --- a/tests/unit/cao-materialize.test.mjs +++ /dev/null @@ -1,344 +0,0 @@ -import assert from "node:assert/strict"; -import { spawnSync } from "node:child_process"; -import { - existsSync, - lstatSync, - mkdirSync, - mkdtempSync, - readFileSync, - readdirSync, - rmSync, - statSync, - symlinkSync, - writeFileSync, -} from "node:fs"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import test from "node:test"; -import { gzipSync } from "node:zlib"; -import { parse } from "yaml"; -import { - extractCaoArchive, - materializeCaoFromSource, - packCaoRuntime, - planCaoMaterialization, - verifyCaoRuntime, -} from "../../.github/workflows/shared/materialize-cao.mjs"; - -const sourceRoot = path.resolve(import.meta.dirname, "..", ".."); -const runtimeManifest = JSON.parse(readFileSync(path.join(sourceRoot, "runtime", "package.json"), "utf8")); -const retiredLauncher = `#!/bin/sh -set -eu - -root="$(CDPATH='' cd -P "$(dirname "$0")" && pwd)" - -cli="$root/activity/cao.mjs" -if [ ! -f "$cli" ]; then - echo "cao CLI is unavailable; install the Central Agentic Ops root campaign." >&2 - exit 1 -fi - -exec node "$cli" --pipeline "$@" -`; - -for (const [name, contents, removed] of [ - ["pipeline launcher", retiredLauncher, true], - ["older launcher", retiredLauncher.replace(" --pipeline ", " "), true], - ["consumer script", "#!/bin/sh\necho consumer-owned\n", false], - ["directory", null, false], - ["symlink", null, false], -]) { - test(`root materialization ${removed ? "retires" : "preserves"} ${name}`, { - skip: name === "symlink" && process.platform === "win32", - }, (t) => { - const destination = mkdtempSync(path.join(tmpdir(), "cao-retired-launcher-")); - t.after(() => rmSync(destination, { recursive: true, force: true })); - const filename = path.join(destination, "cao.sh"); - if (name === "directory") mkdirSync(filename); - else if (name === "symlink") { - writeFileSync(path.join(destination, "consumer-script"), retiredLauncher); - symlinkSync("consumer-script", filename); - } else writeFileSync(filename, contents); - materializeCaoFromSource("root", sourceRoot, destination); - assert.equal(existsSync(filename), !removed); - if (name === "consumer script") assert.equal(readFileSync(filename, "utf8"), contents); - if (name === "directory") assert.ok(lstatSync(filename).isDirectory()); - if (name === "symlink") { - assert.ok(lstatSync(filename).isSymbolicLink()); - assert.equal(readFileSync(filename, "utf8"), retiredLauncher); - } - }); -} - -test("focused materialization and failed root preflight preserve a retired launcher", (t) => { - const destination = mkdtempSync(path.join(tmpdir(), "cao-retired-launcher-scope-")); - t.after(() => rmSync(destination, { recursive: true, force: true })); - const filename = path.join(destination, "cao.sh"); - writeFileSync(filename, retiredLauncher); - materializeCaoFromSource("dependabot", sourceRoot, destination); - assert.equal(readFileSync(filename, "utf8"), retiredLauncher); - assert.throws(() => materializeCaoFromSource("root", destination, destination), /missing|incomplete/i); - assert.equal(readFileSync(filename, "utf8"), retiredLauncher); -}); - -function filesBelow(root, prefix = "") { - return readdirSync(path.join(root, prefix)).flatMap((name) => { - const relative = path.join(prefix, name); - return statSync(path.join(root, relative)).isDirectory() - ? filesBelow(root, relative) - : [relative.split(path.sep).join("/")]; - }); -} - -function tarArchive(entries) { - const blocks = []; - for (const { name, content = "", directory = false, mode = 0o644 } of entries) { - const data = Buffer.from(content); - const header = Buffer.alloc(512); - header.write(name); - header.write(mode.toString(8).padStart(7, "0"), 100); - header.write(data.length.toString(8).padStart(11, "0"), 124); - header[156] = directory ? 53 : 48; - blocks.push(header, data, Buffer.alloc((512 - data.length % 512) % 512)); - } - return gzipSync(Buffer.concat([...blocks, Buffer.alloc(1024)])); -} - -test("Windows CAO extraction does not invoke tar and rejects unsafe archive paths", () => { - const destination = mkdtempSync(path.join(tmpdir(), "cao-materialize-windows-")); - const archive = path.join(destination, "cao.tar.gz"); - try { - writeFileSync(archive, tarArchive([ - { name: "cao-root/", directory: true }, - { name: "cao-root/file.txt", content: "CAO" }, - ])); - extractCaoArchive(archive, destination, true); - assert.equal(readFileSync(path.join(destination, "cao-root", "file.txt"), "utf8"), "CAO"); - - writeFileSync(archive, tarArchive([{ name: "../escape.txt", content: "unsafe" }])); - assert.throws(() => extractCaoArchive(archive, destination, true), /unsafe entry path/); - } finally { - rmSync(destination, { force: true, recursive: true }); - } -}); - -test("native extraction preserves executable bits without restoring special or world-writable permissions", () => { - const destination = mkdtempSync(path.join(tmpdir(), "cao-materialize-modes-")); - const archive = path.join(destination, "cao.tar.gz"); - try { - writeFileSync(archive, tarArchive([ - { name: "package/tool.sh", content: "#!/bin/sh\n", mode: 0o6757 }, - { name: "package/data.json", content: "{}", mode: 0o666 }, - ])); - extractCaoArchive(archive, destination, true); - if (process.platform !== "win32") { - assert.equal(statSync(path.join(destination, "package/tool.sh")).mode & 0o7777, 0o755); - assert.equal(statSync(path.join(destination, "package/data.json")).mode & 0o7777, 0o644); - } - } finally { - rmSync(destination, { force: true, recursive: true }); - } -}); - -test("the runtime package uses npm files selection without shipping development or plugin resources", () => { - const destination = mkdtempSync(path.join(tmpdir(), "cao-runtime-package-")); - try { - const packed = packCaoRuntime(sourceRoot, destination, { tracked: true }); - assert.ok(packed.files.includes("dashboard/report/configure-site.mjs")); - assert.ok(packed.files.includes("dashboard/site/scripts/build.mjs")); - assert.ok(packed.files.includes("dashboard/site/src/data-worker.js")); - assert.ok(packed.files.includes("dashboard/site/package-lock.json")); - assert.ok(!packed.files.includes("activity/quickstart.mjs")); - assert.ok(!packed.files.includes("activity/commands/quickstart.mjs")); - assert.deepEqual(JSON.parse(readFileSync(path.join(packed.sourceRoot, "package.json"), "utf8")), runtimeManifest); - for (const file of packed.files) { - assert.doesNotMatch(file, /(?:^|\/)(?:test|tests|fixtures|screenshots|skills|com\.github\.copilot)(?:\/|$)/); - assert.doesNotMatch(file, /(?:^|\/)(?:PLAN\.md|eslint\.config\.mjs|vitest\.config\.mjs|playwright\.config\.mjs|plugin\.json)$/); - } - } finally { - rmSync(destination, { force: true, recursive: true }); - } -}); - - -test("CAO materialization validates the complete source bundle before replacing runtime files", () => { - const source = mkdtempSync(path.join(tmpdir(), "cao-materialize-incomplete-")); - const destination = mkdtempSync(path.join(tmpdir(), "cao-materialize-existing-")); - const existingRuntime = path.join(destination, "activity", "existing-runtime.mjs"); - try { - mkdirSync(path.dirname(existingRuntime), { recursive: true }); - writeFileSync(existingRuntime, "existing"); - - assert.throws( - () => materializeCaoFromSource("root", source, destination), - /missing required resource: dashboard/, - ); - assert.equal(readFileSync(existingRuntime, "utf8"), "existing"); - } finally { - rmSync(source, { force: true, recursive: true }); - rmSync(destination, { force: true, recursive: true }); - } -}); - -test("root materialization leaves separately installed plugin resources untouched", () => { - const destination = mkdtempSync(path.join(tmpdir(), "cao-materialize-separate-plugin-")); - const pluginFiles = [ - "plugin.json", - "skills/setup-cao/SKILL.md", - "com.github.copilot/extensions/cao-dashboard/extension.mjs", - "specs/dashboard-data.md", - ".github/skills/generate-dashboard-ir/SKILL.md", - ]; - try { - for (const file of pluginFiles) { - mkdirSync(path.dirname(path.join(destination, file)), { recursive: true }); - writeFileSync(path.join(destination, file), "separately installed"); - } - materializeCaoFromSource("root", sourceRoot, destination); - for (const file of pluginFiles) { - assert.equal(readFileSync(path.join(destination, file), "utf8"), "separately installed"); - } - } finally { - rmSync(destination, { force: true, recursive: true }); - } -}); - -for (const missingResource of [ - ".github/workflows/shared/value-acquisition.mjs", - "dashboard/site/src", - "dashboard/site/scripts/build.mjs", - "dashboard/site/package-lock.json", -]) { - test(`CAO materialization rejects a bundle missing ${missingResource} before replacement`, () => { - const source = mkdtempSync(path.join(tmpdir(), "cao-materialize-incomplete-plugin-")); - const destination = mkdtempSync(path.join(tmpdir(), "cao-materialize-existing-plugin-")); - try { - const [{ resources }] = planCaoMaterialization([{ - name: "githubnext/gh-cao", - record: { resolvedCommit: "1".repeat(40) }, - }], "root"); - for (const resource of [...resources, ...runtimeManifest.files]) { - if (resource === missingResource) continue; - const target = path.join(source, resource); - if (statSync(path.join(sourceRoot, resource)).isDirectory()) { - mkdirSync(target, { recursive: true }); - } else { - mkdirSync(path.dirname(target), { recursive: true }); - writeFileSync(target, "source"); - } - } - mkdirSync(path.join(source, "runtime")); - writeFileSync(path.join(source, "runtime", "package.json"), JSON.stringify(runtimeManifest)); - const existingRuntime = path.join(destination, "activity", "existing-runtime.mjs"); - const existingExtension = path.join(destination, "com.github.copilot", "extensions", "cao-dashboard", "extension.mjs"); - for (const file of [existingRuntime, existingExtension]) { - mkdirSync(path.dirname(file), { recursive: true }); - writeFileSync(file, "existing"); - } - assert.throws( - () => materializeCaoFromSource("root", source, destination), - { message: `CAO source revision is missing required resource: ${missingResource}` }, - ); - for (const file of [existingRuntime, existingExtension]) { - assert.equal(readFileSync(file, "utf8"), "existing"); - } - } finally { - rmSync(source, { force: true, recursive: true }); - rmSync(destination, { force: true, recursive: true }); - } - }); -} - -test("CAO materialization rejects paths outside a campaign slug", () => { - assert.throws( - () => materializeCaoFromSource("../outside", sourceRoot, sourceRoot), - /Invalid CAO campaign name/, - ); -}); - -test("root materialization preserves exact focused package revisions", () => { - const rootRevision = "1".repeat(40); - const dashboardRevision = "2".repeat(40); - const plans = planCaoMaterialization([ - { - name: "githubnext/gh-cao/control-plane", - record: { resolvedCommit: rootRevision }, - }, - { - name: "githubnext/gh-cao/dashboard", - record: { resolvedCommit: dashboardRevision }, - }, - ], "root"); - - assert.deepEqual( - plans.map(({ campaign, revision, resources }) => ({ campaign, revision, resources })), - [ - { - campaign: "root", - revision: rootRevision, - resources: [ - ".github/workflows/shared/value-acquisition.mjs", - ".github/workflows/shared/actions-log.mjs", - ".github/actions/setup-cao-runtime", - ".github/actions/setup-gh-aw", - ".github/cao/instructions.md", - ".github/workflows/shared/control.md", - ".github/workflows/shared/operation.md", - ".github/workflows/shared/review-bundle.md", - ], - }, - { - campaign: "dashboard", - revision: dashboardRevision, - resources: ["dashboard"], - }, - ], - ); -}); - -test("root installation provides every local action used by its installed workflows", () => { - const installed = new Set(); - const workflows = []; - const visit = (manifestPath) => { - const manifest = parse(readFileSync(path.join(sourceRoot, manifestPath), "utf8")); - const manifestDirectory = path.posix.dirname(manifestPath); - for (const { destination } of manifest.resources ?? []) installed.add(destination); - for (const include of manifest.includes ?? []) { - const source = typeof include === "string" ? include : include.source; - const resolved = source.startsWith(".github/") - ? source - : path.posix.normalize(path.posix.join(manifestDirectory, source)); - if (path.posix.basename(resolved) === "aw.yml") visit(resolved); - else workflows.push(resolved); - } - }; - visit("aw.yml"); - const [{ resources }] = planCaoMaterialization([{ - name: "githubnext/gh-cao/control-plane", - record: { resolvedCommit: "1".repeat(40) }, - }], "root"); - const provided = (file) => - installed.has(file) || resources.some((resource) => file === resource || file.startsWith(`${resource}/`)); - - assert.ok(workflows.length > 0); - for (const workflow of workflows) { - const source = readFileSync(path.join(sourceRoot, workflow), "utf8"); - for (const [, action] of source.matchAll(/^\s*(?:-\s+)?uses:\s*\.\/([^\s#]+)/gm)) { - const directory = action.replace(/\/+$/, ""); - assert.ok( - provided(`${directory}/action.yml`) || provided(`${directory}/action.yaml`), - `${workflow} uses ./${directory}, which root installation does not provide`, - ); - } - } -}); - -test("CAO materialization requires immutable resolved commit provenance", () => { - assert.throws( - () => planCaoMaterialization([{ - name: "githubnext/gh-cao", - record: { source: "githubnext/gh-cao@v1.2.3" }, - }], "root"), - /must contain a full resolvedCommit SHA/, - ); -}); diff --git a/tests/unit/cao-runtime-artifact.test.mjs b/tests/unit/cao-runtime-artifact.test.mjs index 87d7ea56ee..50259dd13d 100644 --- a/tests/unit/cao-runtime-artifact.test.mjs +++ b/tests/unit/cao-runtime-artifact.test.mjs @@ -1,36 +1,15 @@ import assert from "node:assert/strict"; -import { execFile, execFileSync } from "node:child_process"; -import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { createHash } from "node:crypto"; -import { createServer } from "node:http"; +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; -import { promisify } from "node:util"; import test from "node:test"; import { parse } from "yaml"; -import { - downloadCaoRuntimeArtifact, extractCaoArchive, materializeCao, planCaoMaterialization, validateCaoRuntimeArtifact, - verifyCaoRuntime, -} from "../../.github/workflows/shared/materialize-cao.mjs"; -import { buildCaoRuntime } from "../../scripts/build-cao-runtime.mjs"; +import { buildCaoRuntime, runtimeResources, validateCaoRuntimeArtifact } from "../../scripts/build-cao-runtime.mjs"; const catalog = path.resolve(import.meta.dirname, "../.."); -const executeFile = promisify(execFile); const revision = "1".repeat(40); -const retiredLauncher = `#!/bin/sh -set -eu - -root="$(CDPATH='' cd -P "$(dirname "$0")" && pwd)" - -cli="$root/activity/cao.mjs" -if [ ! -f "$cli" ]; then - echo "cao CLI is unavailable; install the Central Agentic Ops root campaign." >&2 - exit 1 -fi - -exec node "$cli" --pipeline "$@" -`; const temporary = mkdtempSync(path.join(tmpdir(), "cao-runtime-artifact-")); const artifact = buildCaoRuntime(catalog, path.join(temporary, "built"), revision); test.after(() => rmSync(temporary, { recursive: true, force: true })); @@ -38,68 +17,10 @@ test.after(() => rmSync(temporary, { recursive: true, force: true })); function unpack(t) { const destination = mkdtempSync(path.join(temporary, "unpacked-")); t.after(() => rmSync(destination, { recursive: true, force: true })); - extractCaoArchive(artifact.archive, destination, true); + execFileSync("tar", ["-xzf", artifact.archive, "-C", destination]); return path.join(destination, "package"); } - - - - -function fakeGithub(t, { - expired = false, - missing = false, - pendingBeforeSuccess = 0, - rejection = false, -} = {}) { - const root = mkdtempSync(path.join(temporary, "github-")); - const log = path.join(root, "commands.jsonl"); - const fake = path.join(root, "gh"); - const script = `import { appendFileSync, cpSync, existsSync, readFileSync } from "node:fs"; -const args = process.argv.slice(2); -const previous = existsSync(${JSON.stringify(log)}) ? readFileSync(${JSON.stringify(log)}, "utf8").trim().split("\\n").filter(Boolean).map(JSON.parse) : []; -const workflowQueries = previous.filter((entry) => entry[0] === "api" && entry.at(-1).includes("/actions/workflows/")).length; -appendFileSync(${JSON.stringify(log)}, JSON.stringify(args) + "\\n"); -if (args[0] === "api" && args.at(-1).includes("/actions/workflows/")) { - const pending = workflowQueries < ${pendingBeforeSuccess}; - console.log(JSON.stringify([{ workflow_runs: ${missing ? "[]" : `pending ? ${JSON.stringify([ - { id: 9, head_sha: revision, conclusion: null, event: "push", status: "in_progress" }, - ])} : ${JSON.stringify([ - { id: 7, head_sha: "2".repeat(40), conclusion: "success", event: "push", status: "completed" }, - { id: 8, head_sha: revision, conclusion: "success", event: "pull_request", status: "completed" }, - { id: 9, head_sha: revision, conclusion: "success", event: "push", status: "completed" }, - ])}`} }])); -} else if (args[0] === "api") { - console.log(JSON.stringify([{ artifacts: [{ name: "cao-runtime-${revision}", expired: ${expired} }] }])); -} else if (args[0] === "run" || args[0] === "release") { - cpSync(${JSON.stringify(artifact.archive)}, args[args.indexOf("--dir") + 1] + "/cao-runtime-${revision}.tgz"); -} else if (args[0] === "attestation") { - if (${rejection}) { console.error("invalid signature or source digest"); process.exit(1); } -} else { process.exit(2); } -`; - writeFileSync(fake, `#!${process.execPath}\n${script}`); - chmodSync(fake, 0o755); - if (process.platform === "win32") { - const module = path.join(root, "fake-gh.mjs"); - writeFileSync(module, script); - writeFileSync(path.join(root, "gh.cmd"), `@"${process.execPath}" "${module}" %*\r\n`); - cpSync(process.execPath, path.join(root, "gh.exe")); - writeFileSync(path.join(root, "preload.mjs"), `${script.replace("process.argv.slice(2)", "process.argv.slice(1)")}\nprocess.exit(0);\n`); - } - const previousPath = process.env.PATH; - const previousOptions = process.env.NODE_OPTIONS; - process.env.PATH = `${root}${path.delimiter}${previousPath}`; - if (process.platform === "win32") { - process.env.NODE_OPTIONS = `${previousOptions ?? ""} --import=${pathToFileURL(path.join(root, "preload.mjs")).href}`; - } - t.after(() => { - process.env.PATH = previousPath; - if (previousOptions === undefined) delete process.env.NODE_OPTIONS; - else process.env.NODE_OPTIONS = previousOptions; - }); - return { root, commands: () => readFileSync(log, "utf8").trim().split("\n").map((line) => JSON.parse(line)) }; -} - test("producer artifact binds its production inventory to an immutable revision", (t) => { const root = unpack(t); validateCaoRuntimeArtifact(root, revision); @@ -165,32 +86,12 @@ test("the prebuilt archive loads the Governance CLI transport with dependency-fr assert.equal(result.stdout, yaml); }); - - - test("producer archives are reproducible across temporary build directories", () => { const rebuilt = buildCaoRuntime(catalog, path.join(temporary, "rebuilt"), revision); assert.equal(rebuilt.digest, artifact.digest); assert.deepEqual(readFileSync(rebuilt.archive), readFileSync(artifact.archive)); }); -test("verified older inventories may include a retired launcher without making it a root resource", (t) => { - const root = unpack(t); - const filename = path.join(root, "cao.sh"); - writeFileSync(filename, "#!/bin/sh\nexit 0\n"); - const metadataPath = path.join(root, "cao-runtime.json"); - const metadata = JSON.parse(readFileSync(metadataPath, "utf8")); - metadata.files["cao.sh"] = createHash("sha256").update(readFileSync(filename)).digest("hex"); - writeFileSync(metadataPath, JSON.stringify(metadata)); - validateCaoRuntimeArtifact(root, revision); - const [plan] = planCaoMaterialization([{ - name: "githubnext/gh-cao", record: { resolvedCommit: revision }, - }], "root"); - assert.equal(plan.resources.includes("cao.sh"), false); - writeFileSync(filename, "tampered"); - assert.throws(() => validateCaoRuntimeArtifact(root, revision), /checksum mismatch: cao\.sh/); -}); - test("artifact verification rejects altered, missing and unowned runtime files", (t) => { const root = unpack(t); const cli = path.join(root, "dashboard/report/configure-site.mjs"); @@ -208,87 +109,48 @@ test("artifact verification rejects altered, missing and unowned runtime files", assert.throws(() => validateCaoRuntimeArtifact(root, revision), /unowned path: plugin.json/); }); -for (const [description, ref, downloadCommand] of [ - ["main/SHA", revision, "run"], - ["stable release", "v1.2.3", "release"], -]) { - test(`${description} acquisition verifies the archive, workflow and both exact source digests`, (t) => { - const { root, commands } = fakeGithub(t); - const archive = downloadCaoRuntimeArtifact({ - source: `githubnext/gh-cao@${ref}`, resolvedCommit: revision, - }, path.join(root, "download")); - assert.deepEqual(readFileSync(archive), readFileSync(artifact.archive)); - const calls = commands(); - assert.ok(calls.some((args) => args[0] === downloadCommand)); - if (downloadCommand === "run") assert.equal(calls.find((args) => args[0] === "run")[2], "9"); - const verification = calls.find((args) => args[0] === "attestation"); - for (const [flag, value] of [ - ["--repo", "githubnext/gh-cao"], - ["--hostname", "github.com"], - ["--signer-workflow", "githubnext/gh-cao/.github/workflows/cao-runtime.yml"], - ["--source-digest", revision], ["--signer-digest", revision], - ]) assert.equal(verification[verification.indexOf(flag) + 1], value); - assert.ok(verification.includes("--deny-self-hosted-runners")); - }); -} - -test("main/SHA acquisition waits for the active exact runtime build", (t) => { - const { root, commands } = fakeGithub(t, { pendingBeforeSuccess: 2 }); - const waits = []; - const archive = downloadCaoRuntimeArtifact({ - source: `githubnext/gh-cao@${revision}`, resolvedCommit: revision, - }, path.join(root, "download"), { - attempts: 2, - intervalMs: 5_000, - wait: (milliseconds) => waits.push(milliseconds), - }); - assert.deepEqual(readFileSync(archive), readFileSync(artifact.archive)); - assert.deepEqual(waits, [5_000, 5_000]); - assert.equal(commands().filter((args) => args[0] === "api" && args.at(-1).includes("/actions/workflows/")).length, 3); +test("the runtime package ships only owned runtime resources without development or plugin files", () => { + assert.ok(artifact.files.includes("dashboard/report/configure-site.mjs")); + assert.ok(artifact.files.includes("dashboard/site/scripts/build.mjs")); + assert.ok(artifact.files.includes("dashboard/site/src/data-worker.js")); + assert.ok(artifact.files.includes("dashboard/site/package-lock.json")); + for (const file of artifact.files) { + assert.doesNotMatch(file, /(?:^|\/)(?:test|tests|fixtures|screenshots|skills|com\.github\.copilot)(?:\/|$)/); + assert.doesNotMatch(file, /(?:^|\/)(?:PLAN\.md|eslint\.config\.mjs|vitest\.config\.mjs|playwright\.config\.mjs|plugin\.json)$/); + } }); -for (const [description, options] of [ - ["missing", { missing: true }], - ["expired", { expired: true }], - ["unverifiable", { rejection: true }], -]) { - test(`${description} artifacts fail before any owned runtime is replaced`, async (t) => { - const { root, commands } = fakeGithub(t, options); - const consumer = path.join(root, "consumer"); - mkdirSync(path.join(consumer, ".github/aw/packages"), { recursive: true }); - writeFileSync(path.join(consumer, ".github/aw/packages/cao.json"), JSON.stringify({ - package: "githubnext/gh-cao", source: `githubnext/gh-cao@${revision}`, resolvedCommit: revision, - })); - mkdirSync(path.join(consumer, "activity")); - const existing = path.join(consumer, "activity/existing.mjs"); - writeFileSync(existing, "existing"); - writeFileSync(path.join(consumer, "cao.sh"), retiredLauncher); - await assert.rejects(materializeCao("root", consumer), /No successful, unexpired|invalid signature or source digest/); - assert.equal(readFileSync(existing, "utf8"), "existing"); - assert.equal(readFileSync(path.join(consumer, "cao.sh"), "utf8"), retiredLauncher); - assert.equal(commands().some((args) => args[0] === "release"), false); - }); -} - - -test("remote materialization consumes the prebuilt package and does not invoke npm", async (t) => { - const { root } = fakeGithub(t); - const consumer = path.join(root, "consumer"); - mkdirSync(path.join(consumer, ".github/aw/packages"), { recursive: true }); - writeFileSync(path.join(consumer, ".github/aw/packages/cao.json"), JSON.stringify({ - package: "githubnext/gh-cao", resolvedCommit: revision, - })); - const npm = path.join(root, "npm"); - writeFileSync(npm, "#!/bin/sh\nexit 99\n"); - chmodSync(npm, 0o755); - writeFileSync(path.join(consumer, "cao.sh"), retiredLauncher); - await materializeCao("root", consumer); - assert.equal(existsSync(path.join(consumer, "cao.sh")), false); - await materializeCao("root", consumer); - assert.equal(existsSync(path.join(consumer, "cao.sh")), false); - assert.deepEqual(readFileSync(path.join(consumer, "dashboard/report/configure-site.mjs")), readFileSync(path.join(catalog, "dashboard/report/configure-site.mjs"))); - assert.equal(existsSync(path.join(consumer, "package.json")), false); - assert.equal(existsSync(path.join(consumer, "cao-runtime.json")), false); +test("root installation provides every local action used by its installed workflows", () => { + const installed = new Set(); + const workflows = []; + const visit = (manifestPath) => { + const manifest = parse(readFileSync(path.join(catalog, manifestPath), "utf8")); + const manifestDirectory = path.posix.dirname(manifestPath); + for (const { destination } of manifest.resources ?? []) installed.add(destination); + for (const include of manifest.includes ?? []) { + const source = typeof include === "string" ? include : include.source; + const resolved = source.startsWith(".github/") + ? source + : path.posix.normalize(path.posix.join(manifestDirectory, source)); + if (path.posix.basename(resolved) === "aw.yml") visit(resolved); + else workflows.push(resolved); + } + }; + visit("aw.yml"); + const provided = (file) => + installed.has(file) || runtimeResources.some((resource) => file === resource || file.startsWith(`${resource}/`)); + + assert.ok(workflows.length > 0); + for (const workflow of workflows) { + const source = readFileSync(path.join(catalog, workflow), "utf8"); + for (const [, action] of source.matchAll(/^\s*(?:-\s+)?uses:\s*\.\/([^\s#]+)/gm)) { + const directory = action.replace(/\/+$/, ""); + assert.ok( + provided(`${directory}/action.yml`) || provided(`${directory}/action.yaml`), + `${workflow} uses ./${directory}, which root installation does not provide`, + ); + } + } }); test("the trusted runtime pipeline separates building and attestation from draft asset publication", () => { diff --git a/tests/unit/install-script.test.mjs b/tests/unit/install-script.test.mjs index 47fcd6d482..e9976fa26f 100644 --- a/tests/unit/install-script.test.mjs +++ b/tests/unit/install-script.test.mjs @@ -9,7 +9,6 @@ import { promisify } from "node:util"; import test from "node:test"; import vm from "node:vm"; import { parse } from "yaml"; -import { materializeCaoFromSource } from "../../.github/workflows/shared/materialize-cao.mjs"; import { buildCaoRuntime } from "../../scripts/build-cao-runtime.mjs"; import { runTerminal } from "../helpers/terminal.mjs"; @@ -30,7 +29,6 @@ const revision = "1".repeat(40); // Keep a finite subprocess deadline without imposing that budget on other hosts. const timeout = process.platform === "win32" ? 120_000 : 30_000; const policyPath = path.join(".github", "workflows", "cao.json"); -const materializer = path.join(".github", "workflows", "shared", "materialize-cao.mjs"); const curlCommand = "curl --fail --show-error --location"; const manualUpgrade = (version) => new RegExp(`Run \`${curlCommand} \\S+/install-gh-aw\\.sh \\| bash -s -- ${version.replaceAll(".", "\\.")}\`, then rerun the CAO installer`); @@ -96,7 +94,6 @@ const archiveResources = [ ".github/skills/generate-dashboard-ir", ".github/skills/dashboard-authoring/SKILL.md", ".github/skills/author-dashboard-intent/SKILL.md", - ".github/actions/setup-cao-runtime", ".github/actions/setup-gh-aw", ".github/cao/instructions.md", ".github/workflows/shared/control.md", @@ -212,8 +209,21 @@ if [[ "\${1:-} \${2:-}" == "repo create" ]]; then fi exit 0 fi +# gh cao setup acquires the exact attested runtime for the latest release. +if [[ "$*" == "release view --repo github.com/githubnext/gh-cao --json tagName --jq .tagName" ]]; then + echo v1.2.3 + exit 0 +fi +if [[ "$*" == "api --hostname github.com repos/githubnext/gh-cao/commits/v1.2.3 --jq .sha" ]]; then + echo ${revision} + exit 0 +fi +if [[ "\${1:-}" == api && "$*" == *"repos/githubnext/gh-cao/contents/aw.yml?ref=${revision}"* ]]; then + cat "$FAKE_CATALOG/aw.yml" + exit 0 +fi if [[ "\${1:-}" == api && "$*" == *"/actions/workflows/cao-runtime.yml/runs?"* ]]; then - printf '%s\\n' '[{"workflow_runs":[{"id":42,"head_sha":"${revision}","conclusion":"success","event":"push"}]}]' + printf '%s\\n' '[{"workflow_runs":[{"id":42,"head_sha":"${revision}","status":"completed","conclusion":"success","event":"push"}]}]' exit 0 fi if [[ "\${1:-}" == api && "$*" == *"/actions/runs/42/artifacts?"* ]]; then @@ -221,7 +231,6 @@ if [[ "\${1:-}" == api && "$*" == *"/actions/runs/42/artifacts?"* ]]; then exit 0 fi if [[ "\${1:-} \${2:-}" == "run download" ]]; then - [[ "\${FAKE_CAO_ARCHIVE_STATUS:-200}" == 200 ]] || { echo "HTTP 503" >&2; exit 1; } while [[ $# -gt 0 ]]; do if [[ "$1" == --dir ]]; then destination="$2"; break; fi shift @@ -230,7 +239,6 @@ if [[ "\${1:-} \${2:-}" == "run download" ]]; then exit 0 fi if [[ "\${1:-} \${2:-}" == "attestation verify" ]]; then - [[ -z "\${FAKE_CAO_ATTESTATION_FAILURE:-}" ]] || { echo "attestation rejected" >&2; exit 1; } exit 0 fi if [[ "$*" == "repo view --json nameWithOwner --jq .nameWithOwner" ]]; then @@ -524,9 +532,8 @@ async function assertCompleteInstall(consumer, env, ghAwVersion, repository, { a assert.equal(await exists(path.join(consumer, "cao.sh")), false); await executeFile("gh", ["cao", "--help"], { cwd: consumer, env, timeout }); assert.equal(await exists(path.join(consumer, "activity")), false); - for (const bundle of ["dashboard"]) { - await executeFile(process.execPath, [materializer, "verify", bundle], { cwd: consumer, env, timeout }); - } + // The installer leaves the runtime to gh cao setup, which stages the attested artifact. + assert.equal(await exists(path.join(consumer, "dashboard", "site", "package.json")), activity); for (const file of [ "plugin.json", "com.github.copilot/extensions/cao-dashboard/extension.mjs", @@ -611,7 +618,7 @@ for (const [state, initialVersion, expectedVersion, expectedLog] of [ [`gh-aw ${supportedGhAw}`, supportedGhAw, supportedGhAw, "add\n"], [`gh-aw ${newerGhAw}`, newerGhAw, newerGhAw, "add\n"], ]) { - test(`streamed install-gh-cao.sh with ${state} materializes the runtime and initializes policy`, async (t) => { + test(`streamed install-gh-cao.sh with ${state} installs the control plane and initializes policy`, async (t) => { const { consumer, repository, env, log } = await createConsumer(t, initialVersion); await streamInstaller(consumer, env); assert.equal(await log(), expectedLog); @@ -681,11 +688,11 @@ for (const response of ['{"message":"API rate limit exceeded"}', "not JSON"]) { }); } -test("documented staging preserves the complete curated runtime in a fresh checkout", async (t) => { +test("documented staging preserves the complete installation in a fresh checkout", async (t) => { const { root, consumer, repository, env } = await createConsumer(t, supportedGhAw); await streamInstaller(consumer, env); await executeFile("git", ["init", "--quiet"], { cwd: consumer }); - await executeFile("git", ["add", ".github", "dashboard"], { cwd: consumer }); + await executeFile("git", ["add", ".github"], { cwd: consumer }); const { stdout: untracked } = await executeFile("git", ["ls-files", "--others", "--exclude-standard"], { cwd: consumer }); assert.equal(untracked, "", "the documented commit must include every installed resource"); await executeFile("git", [ @@ -693,7 +700,7 @@ test("documented staging preserves the complete curated runtime in a fresh check "-c", "user.email=cao-test@example.invalid", "-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", - "commit", "--quiet", "-m", "Install curated CAO runtime", + "commit", "--quiet", "-m", "Install CAO control plane", ], { cwd: consumer }); const checkout = path.join(root, "fresh-checkout"); await executeFile("git", ["clone", "--quiet", "--no-local", consumer, checkout]); @@ -712,8 +719,7 @@ test("install-gh-cao.sh reports useful progress and does not silence curl", asyn assert.match(result.stdout, new RegExp(`Installing gh-aw ${supportedGhAw.replaceAll(".", "\\.")}`)); assert.match(result.stdout, /Installs 3 workflows, 5 shared control plane resources, and project settings\./); assert.match(result.stdout, /Installing control plane components/); - assert.match(result.stdout, /A newly published catalog revision may wait for its exact verified runtime build/); - assert.match(result.stdout, /Verifying exact CAO runtime/); + assert.doesNotMatch(result.stdout, /CAO runtime/); assert.doesNotMatch(result.stdout, /Making the CAO launcher executable/); assert.match(result.stdout, /Initializing the CAO policy/); assert.match(result.stdout, /CAO installed and verified/); @@ -889,8 +895,6 @@ test("install-gh-cao.sh reruns restore missing policy without replacing consumer for (const [failure, environment, expectedLog] of [ ["campaign add fails", { FAKE_ADD_FAILURE: "1" }, "add\n"], - ["the immutable archive download fails", { FAKE_CAO_ARCHIVE_STATUS: "503" }, "add\n"], - ["runtime provenance verification fails", { FAKE_CAO_ATTESTATION_FAILURE: "1" }, "add\n"], ]) { test(`install-gh-cao.sh exits nonzero without policy when ${failure}`, async (t) => { const { consumer, env, log } = await createConsumer(t, supportedGhAw); @@ -1287,7 +1291,7 @@ for (const [name, environment, arguments_] of [ const installEnv = { ...env, ...environment, FAKE_CONTROL_REPOSITORY: repository }; const result = await streamInstaller(consumer, installEnv, arguments_); const setup = arguments_.includes("--setup"); - assert.equal(await log(), `${creating ? "create-private\n" : ""}add\n`); + assert.equal(await log(), `${creating ? "create-private\n" : ""}add\n${setup ? "add\n" : ""}`); assert.match(result.stdout, /CAO installed and verified/); if (setup) { assert.match(result.stderr, /Non-interactive setup applied/); @@ -1658,7 +1662,7 @@ test("install-gh-cao.sh in CI applies public defaults without prompting on an av }); assert.equal(result.code, 0, result.stdout + result.stderr); assert.match(result.stdout, /Non-interactive setup applied/); - assert.equal(await log(), "add\n"); + assert.equal(await log(), "add\nadd\n"); }); test("install-gh-cao.sh in CI fails an old gh-aw without reading the controlling terminal", { skip: process.platform === "win32" }, async (t) => { diff --git a/tests/unit/workflow-contract-campaigns.test.mjs b/tests/unit/workflow-contract-campaigns.test.mjs index 11cb09ffcd..5e0fcc55b0 100644 --- a/tests/unit/workflow-contract-campaigns.test.mjs +++ b/tests/unit/workflow-contract-campaigns.test.mjs @@ -185,8 +185,8 @@ test("operational packages own their workflows without reinstalling the control assert.match(control, /path: \$\{\{ runner\.temp \}\}\/cao\/admission\.json/); assert.ok(existsSync(join(root, ".github", "workflows", "shared", "actions-log.mjs"))); assert.match( - readFileSync(join(root, ".github", "workflows", "shared", "materialize-cao.mjs"), "utf8"), - /rootResources = \[[\s\S]*?'dashboard'/, + readFileSync(join(root, "scripts", "build-cao-runtime.mjs"), "utf8"), + /runtimeResources = \[[\s\S]*?"dashboard"/, ); }); @@ -296,7 +296,7 @@ test("root campaign provides default control-repository agent context", () => { const debugSkill = readFileSync(join(root, "skills", "debug-cao", "SKILL.md"), "utf8"); assert.doesNotMatch(rootManifestSource, /source: AGENTS\.md|default-AGENTS\.md/); - assert.equal(rootManifest.resources.length, 9); + assert.equal(rootManifest.resources.length, 8); assert.equal(rootManifest.resources.some(({ destination }) => destination.startsWith(".github/skills/")), false); assert.deepEqual(rootManifest.skills, [ "skills/agentic-workflows", @@ -492,10 +492,6 @@ test("root campaign composes its operational campaigns through manifests", () => source: ".github/workflows/shared/github-apps.mjs", destination: ".github/workflows/shared/github-apps.mjs", }, - { - source: ".github/workflows/shared/materialize-cao.mjs", - destination: ".github/workflows/shared/materialize-cao.mjs", - }, { source: ".github/aw/instructions.md", destination: ".github/aw/instructions.md", From ebb2dc0390c703fc63a7dd363940b66514156c0e Mon Sep 17 00:00:00 2001 From: Peli de Halleux <4175913+pelikhan@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:55:34 -0800 Subject: [PATCH 2/5] Correct remaining retired runtime documentation Document native acquisition as the compaction replacement and remove the obsolete local-source installation path from the architecture. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- ARCHITECTURE.md | 7 ++++--- specs/data-acquisition-audit.md | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index e5e227c252..f19e7df3c0 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1698,8 +1698,9 @@ runner before extraction, then checks the inventory, checksums and ownership. When the exact main/SHA runtime build is queued or running, materialization waits for it for at most five minutes; failed, missing, expired, timed-out, or unverifiable artifacts fail closed without source/npm packing or latest-release -fallback. Explicit local source materialization remains a developer path using -npm. Normal operator installation needs Node.js and authenticated GitHub CLI +fallback. Runtime packaging from a catalog checkout belongs to +`scripts/build-cao-runtime.mjs`; local source materialization is not an +installation path. Normal operator installation needs Node.js and authenticated GitHub CLI artifact/attestation access, not npm. The materializer installs only owned canonical destinations from the archive and does not replace consumer-owned root package metadata. Installed control @@ -1808,7 +1809,7 @@ readiness before installing or upgrading anything. Repository creation and cloning are private-only and require an explicit `--create-repo OWNER/NAME`; existing clone directories are never overwritten. Bare installation ends with the explicit `gh cao setup` next step. `--setup` attaches guided setup directly -to the controlling terminal after verified materialization when one is +to the controlling terminal after control-plane installation when one is available. Without a terminal or when `CI` is set, setup applies only an exact public-control-repository profile: self-only read and Activity scope, no campaigns, no Pages configuration, and repository workflow-token diff --git a/specs/data-acquisition-audit.md b/specs/data-acquisition-audit.md index a342dd21b0..98df13de7c 100644 --- a/specs/data-acquisition-audit.md +++ b/specs/data-acquisition-audit.md @@ -44,7 +44,7 @@ The control bootstrap checks out the control repository's `.github` tree at `git -**Current acquisition state:** The inventory below supersedes historical details in the parenthetical refresh notes above. Activity requests compact `usage` and `experiment` artifacts with audit generation, performs no Actions run fallback or Jobs API enrichment, and supplies run attribution to `records.mjs`. Its inventory sidecar now reads the paginated Actions workflow registry for each resolved repository so workflow identity and state do not depend on run telemetry; per-repository failures remain partial evidence. Report collection no longer reads repository details, remote workflow inventories, lock sources, gh-aw releases, or Actions run details. Activity's `gh aw logs` invocation uses `--cached-jsonl` with a trailing wildcard shard prefix; new data lands in a freshly named shard rather than a rescanned single growing file. Ingestion (`gh cao`) was restructured from a single carried-forward shard directory (`--input-dir`) into paired `--runs-dir`/`--events-dir` phased shard directories, so compact run properties commit before detailed events; each shard still skips reprocessing when its content hash already matches the transactions table. `cao-activity.yml`'s `REPORT_RUN_LIMIT` and `activity/logs.mjs`'s validated ceiling were raised again, from a 10-run default/200-run ceiling to a 10-run default/10,000-run validated maximum (the workflow now requests `10000`), widening the effective per-run collection window back up. **New:** `activity/collect-logs.sh` now runs `cao issue-status --database ... --input-dir --graphql-cost-budget 25 --graphql-min-remaining 500` once per activity run after ingestion, adding the first production GraphQL acquisition path to the shared activity snapshot. It batches (50 issues per repository call by default) GitHub GraphQL `issue(number:)` lookups for non-pull-request Issue records already present in the local SQLite projection, prioritizing the staleest `statusObservedAt` targets first, and stops per its own point-cost budget or a remaining-quota floor before enriching matching safe-output records in place with authoritative `state`/`state_reason`/`closed_at`. This is bounded, budgeted, and reuses already-ingested targets rather than rescanning source history, but it is additional per-run GitHub traffic beyond the existing `gh aw logs` call; see §6. **Removed:** the `optimization/collect-token-efficiency.sh` invocation was pulled out of `.github/workflows/cao-activity.yml` entirely (the `bash "$token_efficiency_collector"` step and its file-existence probe no longer exist in that workflow). The script still exists at `optimization/collect-token-efficiency.sh` (installed to `optimization/collect-token-efficiency.sh` by the Optimization campaign manifest) and is still exercised directly by unit tests, but no production workflow now calls it, so its `token-efficiency-observation`/`token-efficiency-lifecycle-claim` artifact listing and per-artifact `GET /actions/runs/{id}` plus archive-download traffic no longer occurs during scheduled activity collection. **Removed:** `optimization-ai-credit-auditor.md`, `optimization-ai-credit-optimizer.md`, `optimization-agents-md-curator.md`, and `optimization-skills-curator.md` (and their `.github/workflows/graders/*-operational-value.sh` evaluators) no longer exist anywhere in the repository; all legacy per-workflow operational graders under the former shared `.github/workflows/graders/` path were deleted in favor of per-campaign `/.github/graders/` scripts feeding the new `cao operational-value` collector (`activity/operational-value.mjs`, invoked by `cao-activity.yml`'s "Compute repository operational value" step), which discovers one `operational-value.mjs` module per campaign directory, reads the already-ingested activity SQLite projection, and reserves its own `--max-github-api-rate-limit -2000` budget when a module needs live `gh api`/`gh aw` reads (for example `optimization/operational-value/optimization-token-optimizer.mjs`, which downloads the deployed dashboard snapshot only when no local activity database is available). These entries superseded the auditor/optimizer/curator rows and their grader rows below. **New:** an optional server-side, webhook-driven collector profile (`specs/server-ingestion.md`, `server/internal/collect/`, `server/internal/githubapp/`) now exists as an alternative to the Actions profile; it is off unless an operator configures a GitHub App, and when enabled it acquires evidence with the same `gh aw logs --audit` invocation and `activity/cao.mjs compact-jsonl` step the Actions profile uses, driven by verified webhooks instead of a schedule. See §3.4 for its enrollment, collection, cold-start, and rate-limit-governance acquisition paths. **New:** two additional SelfCare workers (`self-care-docs-maintainer`, `self-care-server-go-logging`) were added; see §3.3 for their bounded pull-request and commit evidence reads. **New:** `cao issue-status` now runs deferred, after `gh aw logs` collection and before phased-shard generation, under its own freshly minted `issues: read`-scoped GitHub App token (`steps.issue-status-app-token`, falling back to the same PAT/token chain as other steps) rather than the log-collection step's token; `activity/collect-logs.sh` skips its own inline `issue-status` invocation whenever `REPORT_DEFER_ISSUE_STATUS=1` is set (`cao-activity.yml` always sets it), so the enrichment now runs exactly once, as a dedicated workflow step, instead of inline inside the collection script. The `gh aw logs` invocation's own rate-limit reserve (`REPORT_MAX_GITHUB_API_RATE_LIMIT`) is now `-(base + Value headroom)`: the 100/2,000 credential base plus the policy-declared `api-requests` of every enabled collector in scope (`workflowValueApiHeadroom`), resolved once per owner shard by the `value-api-capacity` step; operational-value collection keeps the base reserve. This is a token-scoping and step-ordering change, not a new request family: the batched, budgeted (`25`-cost/`500`-floor) GraphQL shape from the prior refresh is unchanged. **New:** `dashboard/report/agent-catalog.mjs`, `activity/cli-usage.mjs`, and `activity/mcp-server.mjs` add a read-only agent catalog, CLI query-discovery help text, and an MCP server exposing named Dashboard Language queries; all three execute already-defined queries against the already-ingested local SQLite/IndexedDB projection through the same `query-executor.js` path the dashboard uses and issue no GitHub API or `gh aw logs` calls of their own. **New:** a nineteenth SelfCare worker, `self-care-agent-discoverability`, starts an in-run read-only `cao_catalog`/`cao_query` MCP server over the restored activity SQLite snapshot and audits the published `/cao/` dashboard route with a bounded HTTP/MCP/CLI call budget; it adds no GitHub API acquisition and is gated by the same 24-hour cadence family as `self-care-glossary`/`self-care-dashboard-data-schema`. Its addition also raised `self-care.md`'s `dispatch_max`/`safe-outputs.dispatch-workflow.max` from 18 to 19 and `worker_credits_per_target` from 4750 to 5000; these are dispatch-capacity and credit-budget changes, not new request families. **Corrected:** the previously undocumented `self-care-documentation-discoverability` worker (added before this audit's prior refresh, weekly-cadence-gated, curl-only `llms.txt`/`resources.json` prefetch) is now recorded in §3.3, §5, and the diagram; it was omitted from earlier refreshes despite already dispatching in production. **New:** a new `repo-assist` campaign (`repo-assist.md` orchestrator plus `repo-assist-issue-triage`/`repo-assist-issue-fix`/`repo-assist-maintenance`/`repo-assist-pr-upkeep` workers) now targets arbitrary enrolled repositories on a weekly schedule; each worker lists up to 100 open issues or 50 open `[repo-assist:` pull requests from scratch every dispatch with no persisted watermark, and three of the four workers perform a full unshallow git checkout of the target repository per selected candidate, duplicating the fixed-schedule, no-cursor pattern this audit already records for `pr-sous-chef.md`. See §3.3, §5 item 28, §6, and the diagram. **New:** the optional server-side collector profile (§3.4) gained durable per-delivery-GUID deduplication and in-place Redis-stream replacement (`AdmitDelivery`/`EnqueueDelivery`/`StreamReplaceAndAck`) for at-least-once webhook redelivery; this is Redis-side durability hardening with no change to the GitHub request family, count, or per-installation rate-limit governance already recorded there. **New:** `activity/marketplace.mjs` and the hosted Go resolver (`server/internal/marketplace/resolve.go`) each add one further `GET /repos/{owner}/{repository}` call per configured registry to derive best-effort, public-only popularity (stars/forks) and maintenance-freshness signals per package; this is one additional bounded request per registry (≤32 registries) layered on the existing per-registry ref/tree/manifest reads, not a new unbounded scan, and the hosted path reuses the existing registry-and-revision Redis TTL cache. See §3.2. **New:** `dependabot-update-planner.md` moved its Dependabot-alert collection from an inline agent-turn `list_dependabot_alerts` tool call to a `pre-steps` `github-script` prefetch that pages the same `GET /repos/{owner}/{repo}/dependabot/alerts` endpoint before the agent starts and materializes the complete result to a file; the agent now reads that file instead of calling the tool directly. This is a request-timing and provenance change (plus an added public-repository verification gate) with the same bounded pagination shape as before, not a new request family. See §3.3. **New:** the optional server-side collector's cold start (`server/internal/collect/backfill.go`) now enumerates each enrolled repository's complete historical workflow-run list, not only its repository roster: after installation/repository enumeration, `enqueueHistoricalRuns` pages `GET /repos/{owner}/{repo}/actions/runs` (100/page, newest-first) per repository, reserving one core-quota unit per page against the same per-installation `githubapp/budget.go` governance the live collector uses, and durably queues one deduplicated `collect:run-tasks` Redis-stream entry per discovered run/attempt. A persisted per-repository page cursor (`collect:run-backfill-cursors`) makes this resumable across process restarts, and a repository is marked complete only after a verification pass finds no new run identities, guarding against runs inserted ahead of an in-progress page walk. This replaces the prior audit's enumeration-only description of this cold-start path with a materially larger, paginated, per-repository historical-run discovery; it is additional GitHub REST traffic layered on top of installation/repository enumeration, bounded only by installation-level quota and a per-repository completion cursor, not by any fixed run-count ceiling. See §3.4, §5 item 29, §6, and the diagram. **New:** `self-care-specs-maintainer.md` (SelfCare's twentieth worker) was added: only in live mode against the control repository, it maintains a `(merged_at, number)` cache-memory cursor and inspects at most 30 pull requests merged into the default branch since that cursor (or the preceding seven days on a cache miss), with changed-file lists and diffs, to find merged behavior changes missing a corresponding `specs/` update. This duplicates the bounded, cursor-narrowed-on-hit/full-window-on-miss merged-PR evidence pattern already recorded for `self-care-docs-maintainer` (§3.3, §5 item 12) rather than adding a new request shape; `self-care.md`'s `dispatch_max`/`safe-outputs.dispatch-workflow.max` rose from 19 to 20 to add its 24-hour cadence gate. See §3.3 and §5 item 12. **New:** `self-care-redis-query-optimization.md` (SelfCare's twenty-first worker) was added: via its `pull_requests`/`repos`/`actions` `gh-proxy` toolset it reads the most recent open and three most recently closed pull requests from this worker as reuse/rejection evidence before selecting a Redis-backed query bottleneck; all benchmark measurement is local (synthetic fixtures against a dedicated Redis Stack instance), so the only GitHub reads are this small bounded PR-history lookup. This duplicates the bounded, no-persisted-cursor PR-reuse-signal shape already recorded for `self-care-code-improvement`/`self-care-dashboard-language-refactor` (§5 item 16) rather than adding a new request shape, and no operational-value grader exists yet for this worker. `self-care.md`'s `dispatch_max`/`safe-outputs.dispatch-workflow.max` rose from 20 to 21 to add it as a non-cadence-gated worker. See §3.3 and §5 item 16. **Changed:** `repo-assist.md`'s schedule was shortened from weekly to every six hours, quadrupling the dispatch frequency of its fixed-schedule, no-watermark open-issue/pull-request queue rescan (§5 item 28) without adding a cursor. Separately, `repo-assist-issue-triage.md` gained a `live`-mode-only `assign_to_agent` delegation outcome: after posting a triage comment with verified change scope and acceptance tests, it may call `assign_to_agent` once to assign the existing target-repository issue to Copilot under a dedicated `custom-instructions` prompt; this is a new live-mode write capability triggered by existing bounded issue evidence, not a new GitHub read/collection path, and the tool is never called in `review` mode. See §3.3 and §5 item 28. **Fixed:** `server/internal/collect/runner.go`'s `Runner.Validate` now additionally requires at least one compiled `.lock.yml` file under `/.github/workflows/` (`validateWorkflowCatalog`), and `server/Dockerfile` now copies the repository's compiled `.github/workflows/*.lock.yml` files into the hosted collector image's catalog root; `gh aw logs` discovers agentic workflow identity from these local files before querying runs, so the hosted server-side collector profile (§3.4) previously had no compiled-workflow catalog to discover against and every `Runner.Collect` would have failed validation before its `gh aw logs` call. This closes a cold-start reachability gap in the already-documented request shape rather than adding a new request family. See §3.4. **New:** a twenty-second SelfCare worker, `self-care-hosted-health` (previously undocumented; added before this refresh), evaluates `cao.githubnext.com` production health only for the control repository, in both `review` and `live` mode. Its `pre-agent-steps` issue three bounded `curl` HTTP health checks (`/api/readiness`, `/api/health`, `/api/v1/health`), then, only when the target is `githubnext/gh-cao`, call the hosted `cao` MCP (`cao_catalog`, one bounded `cao_query`) and a read-only OTEL MCP (`StreamList`, `StreamSchema`, `GetLatestTraces`, bounded aggregate `SearchSQL`) over OIDC-scoped credentials; no `gh api`/`gh aw logs`/GitHub REST or GraphQL calls are issued. `self-care.md`'s `dispatch_max`/`safe-outputs.dispatch-workflow.max` rose from 21 to 22 to add it, and it is gated by its own four-hour cadence check (20 most recent runs) independent of the 24-hour cadence family; it is the only worker dispatched in `review` mode. See §3.3 and the diagram. **Changed:** `.github/workflows/shared/control.mjs`'s `loadWorkflowInventory` now paginates `GET /repos/{owner}/{repo}/actions/workflows` (100/page, up to 101 pages, erroring past 10,000 workflows) instead of issuing one unpaginated `per_page=100` call; this widens, not narrows, per-repository request volume for admission/precompute workflow-identity reads on any repository with more than 100 declared workflows. See §3.1. **Fixed:** `server/internal/collect/events.go` adds a `repository` webhook intent (`created`/`unarchived`/`archived`/`deleted` actions) that the optional server-side collector profile now admits through the same per-repository issue-status admission and enqueue path as `issue-status`, and `backfill.go`'s cold-start repository enumeration now excludes archived repositories from collection seeding; this is new webhook-driven intent handling layered on the existing admission/enqueue request shape (§3.4), not an additional GitHub API call, and reduces, not increases, cold-start collection volume by skipping archived repositories. **New:** `install-gh-cao.sh` (operator CLI, manual, one-time per install; not scheduled or dispatched) resolves the CAO extension version with an anonymous HTTPS `repos/githubnext/gh-cao/releases?per_page=100` list (unbounded pages, stops consuming at the first published `vX.Y.Z` tag, including prereleases but excluding drafts) and, before writing files, verifies assets with one `repos/githubnext/gh-cao/releases/tags/` read; binaries and `checksums.txt` are then downloaded from release URLs, not the API. Failures fail closed with no files written. This is core-REST pressure scoped to one operator invocation, has no cache, and does not overlap `dashboard/canvas-server.mjs`'s `releases/latest` read or any workflow acquisition. +**Current acquisition state:** The inventory below supersedes historical details in the parenthetical refresh notes above. Activity requests compact `usage` and `experiment` artifacts with audit generation, performs no Actions run fallback or Jobs API enrichment, and supplies run attribution to `records.mjs`. Its inventory sidecar now reads the paginated Actions workflow registry for each resolved repository so workflow identity and state do not depend on run telemetry; per-repository failures remain partial evidence. Report collection no longer reads repository details, remote workflow inventories, lock sources, gh-aw releases, or Actions run details. Activity's `gh aw logs` invocation uses `--cached-jsonl` with a trailing wildcard shard prefix; new data lands in a freshly named shard rather than a rescanned single growing file. Ingestion (`gh cao`) was restructured from a single carried-forward shard directory (`--input-dir`) into paired `--runs-dir`/`--events-dir` phased shard directories, so compact run properties commit before detailed events; each shard still skips reprocessing when its content hash already matches the transactions table. `cao-activity.yml`'s `REPORT_RUN_LIMIT` and `activity/logs.mjs`'s validated ceiling were raised again, from a 10-run default/200-run ceiling to a 10-run default/10,000-run validated maximum (the workflow now requests `10000`), widening the effective per-run collection window back up. **New:** `activity/collect-logs.sh` now runs `cao issue-status --database ... --input-dir --graphql-cost-budget 25 --graphql-min-remaining 500` once per activity run after ingestion, adding the first production GraphQL acquisition path to the shared activity snapshot. It batches (50 issues per repository call by default) GitHub GraphQL `issue(number:)` lookups for non-pull-request Issue records already present in the local SQLite projection, prioritizing the staleest `statusObservedAt` targets first, and stops per its own point-cost budget or a remaining-quota floor before enriching matching safe-output records in place with authoritative `state`/`state_reason`/`closed_at`. This is bounded, budgeted, and reuses already-ingested targets rather than rescanning source history, but it is additional per-run GitHub traffic beyond the existing `gh aw logs` call; see §6. **Removed:** the `optimization/collect-token-efficiency.sh` invocation was pulled out of `.github/workflows/cao-activity.yml` entirely (the `bash "$token_efficiency_collector"` step and its file-existence probe no longer exist in that workflow). The script still exists at `optimization/collect-token-efficiency.sh` (installed to `optimization/collect-token-efficiency.sh` by the Optimization campaign manifest) and is still exercised directly by unit tests, but no production workflow now calls it, so its `token-efficiency-observation`/`token-efficiency-lifecycle-claim` artifact listing and per-artifact `GET /actions/runs/{id}` plus archive-download traffic no longer occurs during scheduled activity collection. **Removed:** `optimization-ai-credit-auditor.md`, `optimization-ai-credit-optimizer.md`, `optimization-agents-md-curator.md`, and `optimization-skills-curator.md` (and their `.github/workflows/graders/*-operational-value.sh` evaluators) no longer exist anywhere in the repository; all legacy per-workflow operational graders under the former shared `.github/workflows/graders/` path were deleted in favor of per-campaign `/.github/graders/` scripts feeding the new `cao operational-value` collector (`activity/operational-value.mjs`, invoked by `cao-activity.yml`'s "Compute repository operational value" step), which discovers one `operational-value.mjs` module per campaign directory, reads the already-ingested activity SQLite projection, and reserves its own `--max-github-api-rate-limit -2000` budget when a module needs live `gh api`/`gh aw` reads (for example `optimization/operational-value/optimization-token-optimizer.mjs`, which downloads the deployed dashboard snapshot only when no local activity database is available). These entries superseded the auditor/optimizer/curator rows and their grader rows below. **New:** an optional server-side, webhook-driven collector profile (`specs/server-ingestion.md`, `server/internal/collect/`, `server/internal/githubapp/`) now exists as an alternative to the Actions profile; it is off unless an operator configures a GitHub App, and when enabled it uses native Go acquisition and publication under `server/internal/collectnative/` (see `specs/server-ingestion.md` §2.4), driven by verified webhooks instead of a schedule. See §3.4 for its enrollment, collection, cold-start, and rate-limit-governance acquisition paths. **New:** two additional SelfCare workers (`self-care-docs-maintainer`, `self-care-server-go-logging`) were added; see §3.3 for their bounded pull-request and commit evidence reads. **New:** `cao issue-status` now runs deferred, after `gh aw logs` collection and before phased-shard generation, under its own freshly minted `issues: read`-scoped GitHub App token (`steps.issue-status-app-token`, falling back to the same PAT/token chain as other steps) rather than the log-collection step's token; `activity/collect-logs.sh` skips its own inline `issue-status` invocation whenever `REPORT_DEFER_ISSUE_STATUS=1` is set (`cao-activity.yml` always sets it), so the enrichment now runs exactly once, as a dedicated workflow step, instead of inline inside the collection script. The `gh aw logs` invocation's own rate-limit reserve (`REPORT_MAX_GITHUB_API_RATE_LIMIT`) is now `-(base + Value headroom)`: the 100/2,000 credential base plus the policy-declared `api-requests` of every enabled collector in scope (`workflowValueApiHeadroom`), resolved once per owner shard by the `value-api-capacity` step; operational-value collection keeps the base reserve. This is a token-scoping and step-ordering change, not a new request family: the batched, budgeted (`25`-cost/`500`-floor) GraphQL shape from the prior refresh is unchanged. **New:** `dashboard/report/agent-catalog.mjs`, `activity/cli-usage.mjs`, and `activity/mcp-server.mjs` add a read-only agent catalog, CLI query-discovery help text, and an MCP server exposing named Dashboard Language queries; all three execute already-defined queries against the already-ingested local SQLite/IndexedDB projection through the same `query-executor.js` path the dashboard uses and issue no GitHub API or `gh aw logs` calls of their own. **New:** a nineteenth SelfCare worker, `self-care-agent-discoverability`, starts an in-run read-only `cao_catalog`/`cao_query` MCP server over the restored activity SQLite snapshot and audits the published `/cao/` dashboard route with a bounded HTTP/MCP/CLI call budget; it adds no GitHub API acquisition and is gated by the same 24-hour cadence family as `self-care-glossary`/`self-care-dashboard-data-schema`. Its addition also raised `self-care.md`'s `dispatch_max`/`safe-outputs.dispatch-workflow.max` from 18 to 19 and `worker_credits_per_target` from 4750 to 5000; these are dispatch-capacity and credit-budget changes, not new request families. **Corrected:** the previously undocumented `self-care-documentation-discoverability` worker (added before this audit's prior refresh, weekly-cadence-gated, curl-only `llms.txt`/`resources.json` prefetch) is now recorded in §3.3, §5, and the diagram; it was omitted from earlier refreshes despite already dispatching in production. **New:** a new `repo-assist` campaign (`repo-assist.md` orchestrator plus `repo-assist-issue-triage`/`repo-assist-issue-fix`/`repo-assist-maintenance`/`repo-assist-pr-upkeep` workers) now targets arbitrary enrolled repositories on a weekly schedule; each worker lists up to 100 open issues or 50 open `[repo-assist:` pull requests from scratch every dispatch with no persisted watermark, and three of the four workers perform a full unshallow git checkout of the target repository per selected candidate, duplicating the fixed-schedule, no-cursor pattern this audit already records for `pr-sous-chef.md`. See §3.3, §5 item 28, §6, and the diagram. **New:** the optional server-side collector profile (§3.4) gained durable per-delivery-GUID deduplication and in-place Redis-stream replacement (`AdmitDelivery`/`EnqueueDelivery`/`StreamReplaceAndAck`) for at-least-once webhook redelivery; this is Redis-side durability hardening with no change to the GitHub request family, count, or per-installation rate-limit governance already recorded there. **New:** `activity/marketplace.mjs` and the hosted Go resolver (`server/internal/marketplace/resolve.go`) each add one further `GET /repos/{owner}/{repository}` call per configured registry to derive best-effort, public-only popularity (stars/forks) and maintenance-freshness signals per package; this is one additional bounded request per registry (≤32 registries) layered on the existing per-registry ref/tree/manifest reads, not a new unbounded scan, and the hosted path reuses the existing registry-and-revision Redis TTL cache. See §3.2. **New:** `dependabot-update-planner.md` moved its Dependabot-alert collection from an inline agent-turn `list_dependabot_alerts` tool call to a `pre-steps` `github-script` prefetch that pages the same `GET /repos/{owner}/{repo}/dependabot/alerts` endpoint before the agent starts and materializes the complete result to a file; the agent now reads that file instead of calling the tool directly. This is a request-timing and provenance change (plus an added public-repository verification gate) with the same bounded pagination shape as before, not a new request family. See §3.3. **New:** the optional server-side collector's cold start (`server/internal/collect/backfill.go`) now enumerates each enrolled repository's complete historical workflow-run list, not only its repository roster: after installation/repository enumeration, `enqueueHistoricalRuns` pages `GET /repos/{owner}/{repo}/actions/runs` (100/page, newest-first) per repository, reserving one core-quota unit per page against the same per-installation `githubapp/budget.go` governance the live collector uses, and durably queues one deduplicated `collect:run-tasks` Redis-stream entry per discovered run/attempt. A persisted per-repository page cursor (`collect:run-backfill-cursors`) makes this resumable across process restarts, and a repository is marked complete only after a verification pass finds no new run identities, guarding against runs inserted ahead of an in-progress page walk. This replaces the prior audit's enumeration-only description of this cold-start path with a materially larger, paginated, per-repository historical-run discovery; it is additional GitHub REST traffic layered on top of installation/repository enumeration, bounded only by installation-level quota and a per-repository completion cursor, not by any fixed run-count ceiling. See §3.4, §5 item 29, §6, and the diagram. **New:** `self-care-specs-maintainer.md` (SelfCare's twentieth worker) was added: only in live mode against the control repository, it maintains a `(merged_at, number)` cache-memory cursor and inspects at most 30 pull requests merged into the default branch since that cursor (or the preceding seven days on a cache miss), with changed-file lists and diffs, to find merged behavior changes missing a corresponding `specs/` update. This duplicates the bounded, cursor-narrowed-on-hit/full-window-on-miss merged-PR evidence pattern already recorded for `self-care-docs-maintainer` (§3.3, §5 item 12) rather than adding a new request shape; `self-care.md`'s `dispatch_max`/`safe-outputs.dispatch-workflow.max` rose from 19 to 20 to add its 24-hour cadence gate. See §3.3 and §5 item 12. **New:** `self-care-redis-query-optimization.md` (SelfCare's twenty-first worker) was added: via its `pull_requests`/`repos`/`actions` `gh-proxy` toolset it reads the most recent open and three most recently closed pull requests from this worker as reuse/rejection evidence before selecting a Redis-backed query bottleneck; all benchmark measurement is local (synthetic fixtures against a dedicated Redis Stack instance), so the only GitHub reads are this small bounded PR-history lookup. This duplicates the bounded, no-persisted-cursor PR-reuse-signal shape already recorded for `self-care-code-improvement`/`self-care-dashboard-language-refactor` (§5 item 16) rather than adding a new request shape, and no operational-value grader exists yet for this worker. `self-care.md`'s `dispatch_max`/`safe-outputs.dispatch-workflow.max` rose from 20 to 21 to add it as a non-cadence-gated worker. See §3.3 and §5 item 16. **Changed:** `repo-assist.md`'s schedule was shortened from weekly to every six hours, quadrupling the dispatch frequency of its fixed-schedule, no-watermark open-issue/pull-request queue rescan (§5 item 28) without adding a cursor. Separately, `repo-assist-issue-triage.md` gained a `live`-mode-only `assign_to_agent` delegation outcome: after posting a triage comment with verified change scope and acceptance tests, it may call `assign_to_agent` once to assign the existing target-repository issue to Copilot under a dedicated `custom-instructions` prompt; this is a new live-mode write capability triggered by existing bounded issue evidence, not a new GitHub read/collection path, and the tool is never called in `review` mode. See §3.3 and §5 item 28. **Fixed:** `server/internal/collect/runner.go`'s `Runner.Validate` now additionally requires at least one compiled `.lock.yml` file under `/.github/workflows/` (`validateWorkflowCatalog`), and `server/Dockerfile` now copies the repository's compiled `.github/workflows/*.lock.yml` files into the hosted collector image's catalog root; `gh aw logs` discovers agentic workflow identity from these local files before querying runs, so the hosted server-side collector profile (§3.4) previously had no compiled-workflow catalog to discover against and every `Runner.Collect` would have failed validation before its `gh aw logs` call. This closes a cold-start reachability gap in the already-documented request shape rather than adding a new request family. See §3.4. **New:** a twenty-second SelfCare worker, `self-care-hosted-health` (previously undocumented; added before this refresh), evaluates `cao.githubnext.com` production health only for the control repository, in both `review` and `live` mode. Its `pre-agent-steps` issue three bounded `curl` HTTP health checks (`/api/readiness`, `/api/health`, `/api/v1/health`), then, only when the target is `githubnext/gh-cao`, call the hosted `cao` MCP (`cao_catalog`, one bounded `cao_query`) and a read-only OTEL MCP (`StreamList`, `StreamSchema`, `GetLatestTraces`, bounded aggregate `SearchSQL`) over OIDC-scoped credentials; no `gh api`/`gh aw logs`/GitHub REST or GraphQL calls are issued. `self-care.md`'s `dispatch_max`/`safe-outputs.dispatch-workflow.max` rose from 21 to 22 to add it, and it is gated by its own four-hour cadence check (20 most recent runs) independent of the 24-hour cadence family; it is the only worker dispatched in `review` mode. See §3.3 and the diagram. **Changed:** `.github/workflows/shared/control.mjs`'s `loadWorkflowInventory` now paginates `GET /repos/{owner}/{repo}/actions/workflows` (100/page, up to 101 pages, erroring past 10,000 workflows) instead of issuing one unpaginated `per_page=100` call; this widens, not narrows, per-repository request volume for admission/precompute workflow-identity reads on any repository with more than 100 declared workflows. See §3.1. **Fixed:** `server/internal/collect/events.go` adds a `repository` webhook intent (`created`/`unarchived`/`archived`/`deleted` actions) that the optional server-side collector profile now admits through the same per-repository issue-status admission and enqueue path as `issue-status`, and `backfill.go`'s cold-start repository enumeration now excludes archived repositories from collection seeding; this is new webhook-driven intent handling layered on the existing admission/enqueue request shape (§3.4), not an additional GitHub API call, and reduces, not increases, cold-start collection volume by skipping archived repositories. **New:** `install-gh-cao.sh` (operator CLI, manual, one-time per install; not scheduled or dispatched) resolves the CAO extension version with an anonymous HTTPS `repos/githubnext/gh-cao/releases?per_page=100` list (unbounded pages, stops consuming at the first published `vX.Y.Z` tag, including prereleases but excluding drafts) and, before writing files, verifies assets with one `repos/githubnext/gh-cao/releases/tags/` read; binaries and `checksums.txt` are then downloaded from release URLs, not the API. Failures fail closed with no files written. This is core-REST pressure scoped to one operator invocation, has no cache, and does not overlap `dashboard/canvas-server.mjs`'s `releases/latest` read or any workflow acquisition. **New:** `cao mode live ` (operator CLI, not a scheduled workflow) now runs a best-effort live-write credential preflight (`activity/authentication.mjs`'s `checkLiveWriteAuthentication`) before writing the policy change: one `gh repo view`, two paginated `actions/variables`/`actions/secrets` listings, and up to two `actions/variables/{name}` reads. See §3.4. **New:** the optional server-side collector (§3.4) gained a selectable, explicitly volatile single-process memory operational-storage adapter (`specs/server-operational-storage.md`, `server/internal/operational/`) alongside the default Redis adapter; `server/internal/collect/backfill.go`'s `Backfill.Run` now accepts a `ReconstructScope` flag that, when set, always re-enumerates every GitHub App installation and its repositories (`Client.ListInstallations`/`Client.ListRepositories`, both paginated 100/page) via `reconstructScope` (`server/internal/collect/volatile_recovery.go`) before replaying the evidence lake, instead of replaying first and skipping enumeration when the lake is already populated. The memory adapter's launcher wiring (`server/internal/server/collecting.go`) sets `ReconstructScope` unconditionally for a volatile store, so every process restart under the memory adapter re-issues a full installation/repository enumeration against the same per-installation rate-limit budget `githubapp/budget.go` governs, even when cold-start would otherwise be a zero-GitHub-request lake replay; the Redis adapter's restart behavior is unchanged. This is a new, adapter-conditional request family layered on the already-documented cold-start enumeration (§3.4, §5 item 29), not a change to the per-request pagination shape or the live per-repository collection call. See §3.4 and §5 item 31. **Removed:** `self-care-redis-query-optimization.md` (the former twenty-first SelfCare worker) was retired; `self-care.md` no longer lists it in `safe-outputs.dispatch-workflow.workflows`, and its `dispatch_max`/`safe-outputs.dispatch-workflow.max` dropped from 22 to 21, returning `self-care-hosted-health` to the twenty-first and final worker. This removes the bounded one-open/three-closed-PR reuse-signal read this audit previously recorded for that worker; no replacement request family was added. **Changed:** `activity/collect-logs.sh` now ingests with a separate, narrower `--retention-days` detail window (`REPORT_DETAIL_WINDOW_DAYS`, default seven days) distinct from the `--run-retention-days` window (still the 30-day `REPORT_RUN_WINDOW_DAYS`); `gh cao`'s `ingest-jsonl` command derives both bounds through the new `sqliteRetention` helper (`activity/sqlite-retention.mjs`), which also sets a seven-day default retention window for run-linked Postgres/IndexedDB/SQLite tables server-side and in the dashboard client. This narrows how long detailed per-event records are retained after ingestion; it does not change the `gh aw logs` request shape, window, or run-count ceiling documented above. **New:** a `bug-bash` campaign (`bug-bash.md` orchestrator plus `bug-bash-investigation.md` worker) was added as a single-repository, `review`-mode-only defect-finding starter. The orchestrator's `repos`/`actions` `gh-proxy` toolset supports ranking at most one candidate (`max: 1`), and the worker's `repos`/`issues` `gh-proxy` toolset searches open `[bug-bash:investigation]` issues in the control repository for duplicate suppression before spending its investigation budget, in addition to its full unshallow (`fetch-depth: 0`) checkout of the target repository. Both reads are small and bounded (at most one candidate, one issue search), and the campaign issues no `gh aw logs` or GraphQL calls; see §3.3 and the diagram. @@ -185,7 +185,7 @@ output destination is the admitted target. | --- | --- | --- | | `server/internal/collect/events.go` (`Admitter.Admit`/`AdmitDelivery`) | Verified webhook deliveries (`workflow_run` completed, `installation`, `installation_repositories`) are mapped to enroll/unenroll/collect/remove-installation intents and applied to a Redis enrollment set; a `workflow_run` for an unenrolled repository is rejected (`ErrNotEnrolled`) rather than triggering discovery. **New:** `AdmitDelivery` additionally reserves a per-delivery-GUID key (`ReserveDelivery`, 30-second lease) before admitting a non-collect intent, then marks the delivery durably remembered (`RememberDelivery`) only after admission succeeds, so a redelivered webhook for the same GUID is recognized as a duplicate (`Admission.Duplicate`) instead of re-applying the intent; collect intents are deduplicated atomically with enqueue via `Queue.EnqueueDelivery` (below). No GitHub API call is made during admission itself. **New:** a `repository` webhook intent (`created`/`unarchived`/`archived`/`deleted` actions) is now parsed and admitted through the same per-repository issue-status-shaped admission and enqueue path `issue-status` uses, recording the repository's numeric ID and lifecycle state on the enqueued task; this adds a new recognized event type to the existing admission pipeline, not a new GitHub API call, since the lifecycle state arrives in the webhook payload itself. | Enrollment is scope, never authority; admission fails closed on an unrecognized or unenrolled repository. An admission-only deployment (`AdmitOnly`) holds no App private key and cannot call GitHub at all. Delivery deduplication is durable (survives process restarts) rather than in-memory, closing a prior at-least-once double-admission gap without adding GitHub requests. | | `server/internal/collect/queue.go` (`Queue.Enqueue`/`EnqueueDelivery`) | Collection tasks are keyed per repository in a Redis stream with a 90-second debounce, so a burst of webhook events for one repository collapses into one queued task. **New:** `EnqueueDelivery` atomically combines delivery-GUID deduplication with stream append in one Redis operation (`StreamEnqueueDelivery`); acknowledged tasks are now removed from the stream (`StreamAckAndDelete`) instead of only marked delivered, and a deferred, retried, or blocked-requeue task replaces its stream entry in place (`StreamReplaceAndAck`) rather than acknowledging the old entry and appending a new one. | `MaxLength` is now enforced only as admission backpressure (rejecting new appends once exceeded) rather than trimming outstanding work, so retried or in-flight tasks are never silently dropped to satisfy the length bound; a lease held by another worker is requeued rather than duplicated (`ErrRepositoryBusy`). | -| `server/internal/collect/runner.go` (`Runner.Collect`) | Per leased task, reserves installation rate-limit budget, mints a short-lived installation token, then runs the identical `gh aw logs --audit --cached-jsonl * --artifacts usage,experiment --start-date -d --count ` invocation and `activity/cao.mjs compact-jsonl` step the Actions profile uses (defaults: 30-day window, 10,000-run count ceiling, 1,200 MB storage cap). **Fixed:** `Runner.Validate` now additionally requires `/.github/workflows/` to contain at least one `.lock.yml` file (`validateWorkflowCatalog`), and `server/Dockerfile` now copies the compiled `.github/workflows/*.lock.yml` files into the server image's catalog root alongside `gh cao`. `gh aw logs` resolves agentic workflow identity from these local compiled workflows before querying runs; the hosted collector image previously omitted them, so every `Runner.Collect` invocation failed validation before issuing its `gh aw logs` call. This is a cold-start-correctness fix that makes the already-documented request shape reachable in the hosted deployment, not a new request family. | The reserve computed by `Budget.Reserve` is passed to `gh aw logs --max-github-api-rate-limit -`, so the CLI itself stops before exhausting the installation's budget; a fresh shard prefix avoids rescanning carried-forward shards, matching the Actions-profile shard behavior in §2. Workflow-catalog validation fails closed (`runnerValidationStageWorkflowCatalog`) before any token is minted or subprocess is run, so a misconfigured image cannot silently skip discovery. | +| `server/internal/collectnative/` | Native Go acquisition replaces the retired subprocess-based runner, including workflow artifact acquisition, metadata-only session extraction, and raw JSONL compaction. | Installation-scoped GitHub APIs use the shared quota service with explicit byte, entry, pagination, attempt, and cancellation bounds; see `specs/server-ingestion.md` §2.4. | | `server/internal/githubapp/budget.go` (`Budget.Reserve`/`Observe`/`Park`) | Tracks per-installation remaining core-rate-limit headroom and reset time in Redis, shared across worker processes; refuses to reserve when recorded headroom is at or below a floor (default 1,000) or the installation is parked after a `Retry-After`/secondary-rate-limit response. | Fails closed on unknown or low headroom rather than optimistically spending; jittered park expiry de-synchronizes concurrent workers resuming after a rate-limit response. | | `server/internal/collect/backfill.go` (`Backfill.Run`) | Cold start first replays the on-disk evidence lake with zero GitHub requests when populated and `ReconstructScope` is unset; only when replay is unavailable does it call `Client.ListInstallations`/`Client.ListRepositories` (both paginated, 100 per page) to enumerate every App installation and its repositories, then seeds one collection task per enrolled repository ordered by most-recently-pushed first. **New:** with the memory operational-storage adapter (`server/internal/operational/`, §3.4 narrative), `server/internal/server/collecting.go` sets `ReconstructScope = true` unconditionally, so `Backfill.Run` always calls `reconstructScope` (`server/internal/collect/volatile_recovery.go`) to re-enumerate every installation and repository before replaying the lake, instead of skipping enumeration on a populated-lake fast path; the Redis adapter leaves `ReconstructScope` unset and keeps the original replay-first behavior. **Changed:** archived repositories are now excluded from collection seeding (`normalizeEnumeratedRepositories` still records the normalized name for dedup purposes but skips adding an `enrolledRepository` entry when `Archived` is true); this reduces, not adds, cold-start request/task volume by skipping repositories unlikely to need active collection. It then runs `enqueueHistoricalRuns`, which pages `GET /repos/{owner}/{repo}/actions/runs` (100 per page, newest-first) once per enrolled, installation-owned repository via `Client.ListWorkflowRuns`, reserving one core-quota unit per page from the same per-installation `githubapp/budget.go` bucket the live webhook collector uses, and durably enqueues one deduplicated `RunTask` (keyed by repository/run/attempt) per discovered run onto a separate `collect:run-tasks` Redis stream. | A persisted per-repository page cursor (`collect:run-backfill-cursors`) makes paging resumable across restarts and process crashes; a repository's cursor is marked complete only after a verification pass that restarts from page one finds zero newly queued identities, guarding against a run inserted ahead of an in-progress walk shifting page offsets. One installation's or one page's enumeration failure is logged and skipped without aborting the rest of the cold start, but there is no fixed run-count ceiling — a repository with a long run history pages until its cursor completes, bounded only by the per-installation quota reserve. The memory adapter's forced re-enumeration still serializes with webhook enrollment mutations (`withMutationLock`) and still fails closed on ambiguous shard-prefix collisions before mutating scope; it is bounded by the same per-installation quota reserve, not a separate budget. | | `server/internal/collect/discovery.go` (`RepositoryDiscovery.Refresh`) | **New:** every `CAO_COLLECT_REPOSITORY_REFRESH_INTERVAL` (default 1 hour, lease-coordinated across replicas) and on `cao-dashboard backfill --repositories-only`, lists all App installations, queues one `repository-list-refresh` scope task per installation, and workers re-enumerate that installation's repositories (`ListRepositoriesPage`) through the quota budget, reconciling removals and queuing collection only for newly discovered repositories. | Never replays or lists workflow runs; failed enumeration neither withdraws nor advances the timestamp. Cost is one installation list plus a full per-installation repository enumeration every hour regardless of change, with no ETag/conditional reuse. | @@ -244,7 +244,7 @@ The activity snapshot is the shared collection boundary. It persists the bounded 23. **~~`activity/collect-logs.sh` gained a new, uncached `gh api` collection path for token-efficiency worker evidence.~~ (Removed)** The `optimization/collect-token-efficiency.sh` invocation (and its file-existence probe) was removed from `.github/workflows/cao-activity.yml`; scheduled activity collection no longer lists `token-efficiency-observation`/`token-efficiency-lifecycle-claim` Actions artifacts or issues per-artifact `GET /actions/runs/{id}` and archive-download calls. The script itself remains at `optimization/collect-token-efficiency.sh` (installed at `optimization/collect-token-efficiency.sh`) and is still exercised by unit tests, but no production workflow currently invokes it, so this N+1 traffic pattern is dormant rather than deleted. 24. **`cao issue-status` remains a budgeted GraphQL collection path for issue-status enrichment, now deferred and separately token-scoped.** The workflow runs `cao issue-status` as its own step against every non-pull-request Issue already in the local projection, batching lookups per repository and stopping at a configured GraphQL cost budget (`25`) or remaining-quota floor (`500`). It now runs after `gh aw logs` collection (not inline inside `activity/collect-logs.sh`) under a freshly minted `issues: read` GitHub App token distinct from the log-collection token, and the log-collection step's own rate-limit reserve was later reverted to `-2000`. This is bounded and reuses already-collected targets rather than performing a new discovery scan, but it is additional per-run GitHub traffic layered on top of the existing `gh aw logs` REST collection; see §2.1 and §6. 25. **The former shared `.github/workflows/graders/` operational-value evaluators were replaced by per-campaign scripts and a new collector.** `optimization-agents-md-curator-operational-value.sh`, both AI-credit graders, and several other legacy per-workflow graders under the shared path were deleted; surviving graders now live under `/.github/graders/*-operational-value.sh` and are invoked by `cao operational-value` (`activity/operational-value.mjs`) from within `cao-activity.yml`, which reads the already-ingested SQLite projection and reserves its own rate-limit budget only when a module needs live reads. This consolidates discovery of grader scripts but does not by itself change per-grader request shapes recorded elsewhere in this section. -26. **A new optional server-side collection profile duplicates the Actions profile's acquisition shape per enrolled repository, by design.** The webhook-driven collector (`server/internal/collect/`, §3.4) reuses the same `gh aw logs --audit` invocation and `activity/cao.mjs compact-jsonl` step as `cao-activity.yml`, but keys collection per repository via debounced Redis-stream tasks instead of one shared scheduled batch; a deployment that runs both profiles simultaneously against the same repository would be a fatal misconfiguration under `specs/server-ingestion.md` §2.1, not a duplicate-work bottleneck, because the two profiles are mutually exclusive per deployment. +26. **The optional server-side collector no longer invokes the retired Activity CLI.** Native Go acquisition (`server/internal/collectnative/`) replaces workflow artifact acquisition, metadata-only session extraction, and raw JSONL compaction using installation-scoped GitHub APIs and the shared quota service, without Node, `cao-activity`, or `gh-aw` subprocesses. See `specs/server-ingestion.md` §2.4 for the current acquisition and publication boundary; the original shared-CLI topology is retained only in the audit history. 27. **A new read-only agent catalog/MCP surface reads only already-ingested local data, adding no GitHub acquisition path.** `dashboard/report/agent-catalog.mjs`, `activity/cli-usage.mjs`, and `activity/mcp-server.mjs` expose named Dashboard Language queries for CLI and MCP-client discovery, executing them through the same `query-executor.js` path the dashboard uses against the already-ingested local SQLite/IndexedDB projection; no new `gh api`/`gh aw logs` calls are issued by this surface. 28. **A `repo-assist` campaign duplicates `pr-sous-chef.md`'s fixed-schedule, no-watermark queue rescan on a second, independent campaign, now at four times the frequency.** `repo-assist-issue-triage`/`repo-assist-issue-fix` each list up to 100 open issues, and `repo-assist-pr-upkeep` lists up to 50 open `[repo-assist:` pull requests, from scratch every dispatch with no persisted cursor of previously inspected or skipped items; the schedule was shortened from weekly to every six hours, quadrupling this rescan frequency without adding a cursor. Three of the four workers additionally perform a full unshallow (`fetch-depth: 0`, `fetch: ["*"]`) checkout of the target repository per selected candidate, mirroring the git-history acquisition cost `pr-sous-chef.md` added (§5 item 10) rather than reusing it. `repo-assist-issue-triage` also gained a `live`-mode `assign_to_agent` delegation step that assigns an existing, already-selected issue to Copilot after a triage comment; this is a new write path layered on the same bounded issue evidence, not an additional read. 29. **The optional server collector's cold-start backfill now walks every enrolled repository's complete historical workflow-run list, with no fixed per-repository run-count ceiling.** `backfill.go`'s `enqueueHistoricalRuns` pages `GET /repos/{owner}/{repo}/actions/runs` (100/page) per repository until a persisted cursor (`collect:run-backfill-cursors`) reaches completion, consuming one core-quota reservation per page from the same per-installation budget (`githubapp/budget.go`) the live webhook collector shares; a large App install base with long-lived repositories can therefore consume substantially more cold-start request volume than the prior audit's enumeration-only description recorded, bounded only by quota headroom and the per-repository completion cursor rather than by any run-count cap. See §3.4 and §6. From 28a836f7e7052ca932da5e86bdd2ec1bb3674760 Mon Sep 17 00:00:00 2001 From: Peli de Halleux <4175913+pelikhan@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:30:12 -0800 Subject: [PATCH 3/5] Fix runtime archive extraction on Windows Use a relative archive filename from its staging directory so GNU tar does not interpret Windows drive letters as remote hosts. Extend producer regression coverage to enforce the portable invocation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- scripts/build-cao-runtime.mjs | 3 ++- tests/unit/cao-runtime-artifact.test.mjs | 23 +++++++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/scripts/build-cao-runtime.mjs b/scripts/build-cao-runtime.mjs index b4eda6632f..6a9d60131e 100644 --- a/scripts/build-cao-runtime.mjs +++ b/scripts/build-cao-runtime.mjs @@ -152,7 +152,8 @@ export function buildCaoRuntime(sourceRoot, destination, revision) { for (const resource of runtimeResources) copyTrackedResource(sourceRoot, staging, resource); cpSync(manifestPath, path.join(staging, "package.json")); const selected = npmPack(staging, temporary); - run("tar", ["-xzf", path.join(temporary, selected.filename), "-C", unpacked]); + // GNU tar interprets a Windows drive-letter archive path as a remote host. + run("tar", ["-xzf", selected.filename, "-C", unpacked], { cwd: temporary }); const root = path.join(unpacked, "package"); requireResources(root, [...runtimeResources, ...manifest.files]); vendorYaml(sourceRoot, root); diff --git a/tests/unit/cao-runtime-artifact.test.mjs b/tests/unit/cao-runtime-artifact.test.mjs index 50259dd13d..538e1d9fd8 100644 --- a/tests/unit/cao-runtime-artifact.test.mjs +++ b/tests/unit/cao-runtime-artifact.test.mjs @@ -1,7 +1,8 @@ import assert from "node:assert/strict"; -import { execFileSync } from "node:child_process"; +import childProcess, { execFileSync } from "node:child_process"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; +import { syncBuiltinESMExports } from "node:module"; import path from "node:path"; import { pathToFileURL } from "node:url"; import test from "node:test"; @@ -86,8 +87,26 @@ test("the prebuilt archive loads the Governance CLI transport with dependency-fr assert.equal(result.stdout, yaml); }); -test("producer archives are reproducible across temporary build directories", () => { +test("producer archives are reproducible with drive-safe relative tar archive paths", (t) => { + const spawnSync = childProcess.spawnSync; + let extractions = 0; + t.mock.method(childProcess, "spawnSync", (command, args, options) => { + if (command === "tar") { + extractions++; + assert.equal(args[0], "-xzf"); + assert.equal(args[1], path.basename(args[1])); + assert.doesNotMatch(args[1], /:/); + assert.ok(existsSync(path.join(options.cwd, args[1]))); + } + return spawnSync(command, args, options); + }); + syncBuiltinESMExports(); + t.after(() => { + t.mock.restoreAll(); + syncBuiltinESMExports(); + }); const rebuilt = buildCaoRuntime(catalog, path.join(temporary, "rebuilt"), revision); + assert.equal(extractions, 1); assert.equal(rebuilt.digest, artifact.digest); assert.deepEqual(readFileSync(rebuilt.archive), readFileSync(artifact.archive)); }); From b1a4f91e67b93969a47e3863a05f46b91d5a9554 Mon Sep 17 00:00:00 2001 From: Peli de Halleux <4175913+pelikhan@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:37:43 -0800 Subject: [PATCH 4/5] Prefer executable GitHub CLI shim in Windows installer tests Keep native runtime acquisition query arguments out of cmd.exe by restoring executable-first PATHEXT ordering. The existing gh.exe fixture forwards arguments directly to bash without shell query-string splitting. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/unit/install-script.test.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/install-script.test.mjs b/tests/unit/install-script.test.mjs index e9976fa26f..fcecc085b5 100644 --- a/tests/unit/install-script.test.mjs +++ b/tests/unit/install-script.test.mjs @@ -433,7 +433,8 @@ async function createConsumer(t, ghAwVersion, { repository = "alpha-org/control" ...inheritedEnv, PATH: `${bin}${path.delimiter}${systemPath}`, CI: "", - ...(process.platform === "win32" ? { PATHEXT: ".CMD;.COM;.EXE;.BAT" } : {}), + // Native callers must use gh.exe so cmd.exe cannot split API query strings at '&'. + ...(process.platform === "win32" ? { PATHEXT: ".COM;.EXE;.BAT;.CMD" } : {}), NODE_OPTIONS: [process.env.NODE_OPTIONS, `--import=${pathToFileURL(mockFetch).href}`].filter(Boolean).join(" "), FAKE_CATALOG: catalog, FAKE_GH_SCRIPT: path.join(bin, "gh"), From 5b7a5842f0922b875a683855410c37ef225aab2f Mon Sep 17 00:00:00 2001 From: Peli de Halleux <4175913+pelikhan@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:08:21 -0800 Subject: [PATCH 5/5] Fix Windows dashboard startup paths and recovery test timing Normalize filesystem paths before resolving esbuild output dependencies and emitting web asset paths. Cover Windows separators, lazy/external imports, cyclic dependencies, and fail-closed missing chunks. Keep the recovery test lease valid through live-container assertions, then shorten it explicitly for the restart phase. Allow database and container setup before exercising the command timeout so all recovery scenarios reach their intended boundary. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- dashboard/site/scripts/build.mjs | 30 ++++++++++++---------- server/internal/hostedcli/recovery_test.go | 10 ++++++-- tests/unit/dashboard-site-build.test.mjs | 22 ++++++++++++++++ 3 files changed, 47 insertions(+), 15 deletions(-) diff --git a/dashboard/site/scripts/build.mjs b/dashboard/site/scripts/build.mjs index 168599a332..a4ebd6179d 100644 --- a/dashboard/site/scripts/build.mjs +++ b/dashboard/site/scripts/build.mjs @@ -222,6 +222,20 @@ async function findCampaignDashboards(repositoryPath, controlSettings) { return campaignDashboards; } +export function collectStaticModules(outputs, outputPath, modules = new Set()) { + outputPath = outputPath.replaceAll("\\", "/"); + if (modules.has(outputPath)) return modules; + const output = outputs[outputPath]; + if (!output) throw new Error(`Dashboard startup dependency is missing: ${outputPath}`); + modules.add(outputPath); + for (const dependency of output.imports) { + if (dependency.kind === "import-statement" && !dependency.external) { + collectStaticModules(outputs, dependency.path, modules); + } + } + return modules; +} + async function bundleSiteJavascript(destinationPath) { const bundlePath = join(destinationPath, ".bundle"); try { @@ -289,19 +303,9 @@ async function bundleSiteJavascript(destinationPath) { const startupOutput = Object.entries(result.metafile.outputs) .find(([, output]) => output.entryPoint === "src/dashboard-app.js")?.[0]; if (!startupOutput) throw new Error("Dashboard application startup chunk is missing."); - const collectStaticModules = (outputPath, modules = new Set()) => { - if (modules.has(outputPath)) return modules; - const output = result.metafile.outputs[outputPath]; - if (!output) throw new Error(`Dashboard startup dependency is missing: ${outputPath}`); - modules.add(outputPath); - for (const dependency of output.imports) { - if (dependency.kind === "import-statement" && !dependency.external) collectStaticModules(dependency.path, modules); - } - return modules; - }; - const startupModules = collectStaticModules(startupOutput); + const startupModules = collectStaticModules(result.metafile.outputs, startupOutput); const mainOutput = join(bundlePath, "main.js"); - const mainModules = collectStaticModules(relative(destinationPath, mainOutput)); + const mainModules = collectStaticModules(result.metafile.outputs, relative(destinationPath, mainOutput)); const coreModules = [...startupModules].filter((path) => !mainModules.has(path)); const corePaths = new Set(coreModules.map((path) => resolve(destinationPath, path))); const coalescedPath = join(destinationPath, ".coalesced"); @@ -407,7 +411,7 @@ async function listFiles(directory, root = directory) { for (const entry of await readdir(directory, { withFileTypes: true })) { const path = join(directory, entry.name); if (entry.isDirectory()) files.push(...await listFiles(path, root)); - else if (entry.isFile()) files.push(relative(root, path)); + else if (entry.isFile()) files.push(relative(root, path).replaceAll("\\", "/")); } return files; } diff --git a/server/internal/hostedcli/recovery_test.go b/server/internal/hostedcli/recovery_test.go index 40c90c49c7..3a89f6d4b0 100644 --- a/server/internal/hostedcli/recovery_test.go +++ b/server/internal/hostedcli/recovery_test.go @@ -33,13 +33,14 @@ func TestExecutorKeepsUnconfirmedContainerClaimAcrossRestart(t *testing.T) { defer func() { _ = store.Close() }() options := operational.CLICommandQueueOptions{MaxConcurrency: 1} if scenario == "timeout" { - options.Timeout = 20 * time.Millisecond + // Allow database/container setup before timing out the running workload. + options.Timeout = time.Second } queue, err := store.OpenCLICommandQueue(ctx, QueueName, options) if err != nil { t.Fatal(err) } - lease, err := queue.Acquire(ctx, "first-executor", 200*time.Millisecond) + lease, err := queue.Acquire(ctx, "first-executor", time.Minute) if err != nil { t.Fatal(err) } @@ -143,6 +144,11 @@ func TestExecutorKeepsUnconfirmedContainerClaimAcrossRestart(t *testing.T) { if err := queue.Release(ctx, lease); !errors.Is(err, operational.ErrConflict) { t.Fatalf("unsettled lease released: %v", err) } + // Expire only after asserting capacity under a valid executor lease. + lease, err = queue.Renew(ctx, lease, time.Millisecond) + if err != nil { + t.Fatal(err) + } if err := store.Close(); err != nil { t.Fatal(err) } diff --git a/tests/unit/dashboard-site-build.test.mjs b/tests/unit/dashboard-site-build.test.mjs index 8348187917..914875a4e4 100644 --- a/tests/unit/dashboard-site-build.test.mjs +++ b/tests/unit/dashboard-site-build.test.mjs @@ -8,6 +8,7 @@ import test from "node:test"; import { promisify } from "node:util"; import { buildDashboardSite, + collectStaticModules, embedDashboardVersion, embedDashboardVersions, filterExperimentalDashboardViews, @@ -19,6 +20,27 @@ import { agentCatalog } from "../../dashboard/site/src/agent/catalog.js"; const execFileAsync = promisify(execFile); +test("startup dependencies resolve Windows paths without preloading lazy or external imports", () => { + const outputs = { + ".bundle/main.js": { + imports: [ + { path: ".bundle/shared.js", kind: "import-statement" }, + { path: ".bundle/lazy.js", kind: "dynamic-import" }, + { path: "external", kind: "import-statement", external: true }, + ], + }, + ".bundle/shared.js": { + imports: [{ path: ".bundle/main.js", kind: "import-statement" }], + }, + }; + assert.deepEqual( + [...collectStaticModules(outputs, ".bundle\\main.js")], + [".bundle/main.js", ".bundle/shared.js"], + ); + assert.throws(() => collectStaticModules(outputs, ".bundle\\missing.js"), + /Dashboard startup dependency is missing: \.bundle\/missing\.js/); +}); + async function builtSiteSha(destination) { const index = await readFile(new URL("index.html", destination), "utf8"); return index.match(/script\.src = '\.\/src\/main\.js\?sha=([a-f0-9]{64})'/)?.[1];