From 64d85a2ca94ef80c55c16de90c5bdff6c26250ec Mon Sep 17 00:00:00 2001 From: Julian Coy Date: Thu, 23 Jul 2026 10:03:41 -0400 Subject: [PATCH] Emit `0.2` build manifests with supplementary file support, plan-driven archive assembly, and filesystem-identity validation --- .../changes/support-non-asset-files/tasks.md | 22 +- .../__tests__/candidate-archive.e2e.test.ts | 148 +++++++ .../src/__tests__/create-build.e2e.test.ts | 7 +- packages/cli/src/commands/build.ts | 29 +- .../cli/src/tui/views/build/build-view.tsx | 8 +- .../src/__tests__/build-pipeline.test.ts | 231 ++++++++++- .../load-supplementary-sources.test.ts | 187 +++++++++ .../src/build/load-supplementary-sources.ts | 385 ++++++++++++++++++ packages/engine/src/build/pipeline.ts | 74 +++- .../src/install/__tests__/run-add.test.ts | 4 +- .../__tests__/run-install.chain.test.ts | 8 +- .../__tests__/run-install.receipt.test.ts | 8 +- .../src/install/__tests__/run-install.test.ts | 11 +- .../src/install/__tests__/run-remove.test.ts | 4 +- 14 files changed, 1069 insertions(+), 57 deletions(-) create mode 100644 packages/cli/src/__tests__/candidate-archive.e2e.test.ts create mode 100644 packages/engine/src/build/__tests__/load-supplementary-sources.test.ts create mode 100644 packages/engine/src/build/load-supplementary-sources.ts diff --git a/openspec/changes/support-non-asset-files/tasks.md b/openspec/changes/support-non-asset-files/tasks.md index 540bc829..4044d344 100644 --- a/openspec/changes/support-non-asset-files/tasks.md +++ b/openspec/changes/support-non-asset-files/tasks.md @@ -99,20 +99,20 @@ ## 10. Current Producer and Build Pipeline — Research -- [ ] 10.1 Explore: Trace source-file loading, build validation stages, archive assembly, output cleanup, and build-result rendering -- [ ] 10.2 Explore: Inspect source filesystem APIs needed to reject missing files, links, resolved aliases, and non-regular declarations before output mutation -- [ ] 10.3 Explore: Inspect Changesets and CLI packaging to confirm the complete `0.2` producer may be implemented and merged without publishing `agent-facets`, while protocol, registry, adapter, and final CLI activation remain independently controlled release gates -- [ ] 10.4 Propose: Define the producer implementation that reuses the archive plan, preserves deterministic bytes, validates before cleanup, emits only current-format output in the unreleased candidate, and requires no long-lived runtime dual-format flag +- [x] 10.1 Explore: Trace source-file loading, build validation stages, archive assembly, output cleanup, and build-result rendering +- [x] 10.2 Explore: Inspect source filesystem APIs needed to reject missing files, links, resolved aliases, and non-regular declarations before output mutation +- [x] 10.3 Explore: Inspect Changesets and CLI packaging to confirm the complete `0.2` producer may be implemented and merged without publishing `agent-facets`, while protocol, registry, adapter, and final CLI activation remain independently controlled release gates +- [x] 10.4 Propose: Define the producer implementation that reuses the archive plan, preserves deterministic bytes, validates before cleanup, emits only current-format output in the unreleased candidate, and requires no long-lived runtime dual-format flag ## 11. Current Producer and Build Pipeline — Implementation -- [ ] 11.1 Implement: Load declared supplementary files as exact bytes, validate their resolved regular-file identities, and preserve previous `dist/` output on every input failure -- [ ] 11.2 Implement: Drive archive collection and all-entry hashing from the shared archive plan, preserving deterministic ordering and opaque binary or empty supplementary content -- [ ] 11.3 Implement: Switch every build in the unreleased source candidate, including asset-only facets, to flat build-manifest `0.2` output with a complete `files` map while retaining legacy consumer support -- [ ] 11.4 Implement: Update build results and CLI output to show the emitted format, complete entry listing, integrity, and archive-assembly stage -- [ ] 11.5 Implement: Add the build failure-class matrix for traversal, absolute/drive/URL prefixes, backslashes, NUL, empty/`.`/`..` segments, Unicode-normalization and portable-case aliases, Windows-reserved device names, forbidden portable characters, trailing dot/space segments, file/directory prefix collisions, symlinks, hard links, duplicate paths, reserved root `facet.json`, conventional-primary-path collisions, missing declarations, undeclared entries, and tampered bytes, plus success tests for top-level files, nested companions, binary/empty bytes, exact manifest-byte hashing, canonical-tar determinism, scoped output paths, and validation-before-cleanup -- [ ] 11.6 Implement: Add a reproducible candidate archive/interop path that can produce a representative `0.2` artifact for registry stage acceptance without publishing or releasing the CLI -- [ ] 11.7 Verify: Run focused build pipeline and CLI build tests and inspect representative `0.2` archives for exact deterministic membership while confirming no `agent-facets` release changeset is present +- [x] 11.1 Implement: Load declared supplementary files as exact bytes, validate their resolved regular-file identities, and preserve previous `dist/` output on every input failure +- [x] 11.2 Implement: Drive archive collection and all-entry hashing from the shared archive plan, preserving deterministic ordering and opaque binary or empty supplementary content +- [x] 11.3 Implement: Switch every build in the unreleased source candidate, including asset-only facets, to flat build-manifest `0.2` output with a complete `files` map while retaining legacy consumer support +- [x] 11.4 Implement: Update build results and CLI output to show the emitted format, complete entry listing, integrity, and archive-assembly stage +- [x] 11.5 Implement: Add the build failure-class matrix for traversal, absolute/drive/URL prefixes, backslashes, NUL, empty/`.`/`..` segments, Unicode-normalization and portable-case aliases, Windows-reserved device names, forbidden portable characters, trailing dot/space segments, file/directory prefix collisions, symlinks, hard links, duplicate paths, reserved root `facet.json`, conventional-primary-path collisions, missing declarations, undeclared entries, and tampered bytes, plus success tests for top-level files, nested companions, binary/empty bytes, exact manifest-byte hashing, canonical-tar determinism, scoped output paths, and validation-before-cleanup +- [x] 11.6 Implement: Add a reproducible candidate archive/interop path that can produce a representative `0.2` artifact for registry stage acceptance without publishing or releasing the CLI +- [x] 11.7 Verify: Run focused build pipeline and CLI build tests and inspect representative `0.2` archives for exact deterministic membership while confirming no `agent-facets` release changeset is present ## 12. Create and Edit Authoring — Research diff --git a/packages/cli/src/__tests__/candidate-archive.e2e.test.ts b/packages/cli/src/__tests__/candidate-archive.e2e.test.ts new file mode 100644 index 00000000..6670172f --- /dev/null +++ b/packages/cli/src/__tests__/candidate-archive.e2e.test.ts @@ -0,0 +1,148 @@ +import { afterAll, beforeAll, describe, expect, test } from 'bun:test' +import { existsSync } from 'node:fs' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { gunzipSync } from 'node:zlib' +import { type GunzipFn, parseFacetArchive, validateFacetArchive } from '@agent-facets/protocol' + +/** + * Reproducible candidate `0.2` archive / interop path (task 11.6). + * + * Builds a representative facet — a skill with a text companion and a binary + * companion, plus an archive-only README — using the freshly compiled + * candidate CLI (`dist/facet`, produced by `test:e2e`), then verifies the + * emitted `.facet` through the SAME protocol verifier a registry stage uses + * (`validateFacetArchive`). This proves the candidate producer emits a + * verifiable `0.2` artifact with exact deterministic membership, WITHOUT + * publishing or releasing the CLI — no Changeset is involved, and the binary + * is the local compile. + * + * A registry stage acceptance run can reproduce this exact archive by running + * `facet build` on the same source tree; the deterministic tar layout makes + * the bytes stable across machines. + */ + +let testDir: string + +beforeAll(async () => { + testDir = await mkdtemp(join(tmpdir(), 'cli-candidate-archive-')) +}) + +afterAll(async () => { + await rm(testDir, { recursive: true, force: true }) +}) + +const CLI_PATH = resolve(import.meta.dir, '../../dist/facet') + +if (!existsSync(CLI_PATH)) { + throw new Error(`[e2e] dist/facet not found at ${CLI_PATH}.\nBuild the CLI first: bun run --cwd packages/cli build`) +} + +async function runCli(cwd: string, ...args: string[]) { + const facetDir = await mkdtemp(join(testDir, 'facet-dir-')) + const proc = Bun.spawn([CLI_PATH, ...args], { + cwd, + stdout: 'pipe', + stderr: 'pipe', + env: { ...process.env, NO_COLOR: '1', FACET_DIR: facetDir }, + }) + const stdout = await new Response(proc.stdout).text() + const stderr = await new Response(proc.stderr).text() + const exitCode = await proc.exited + return { stdout: stdout.trim(), stderr: stderr.trim(), exitCode } +} + +const REPRESENTATIVE_BINARY = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x00, 0x01, 0xff, 0xfe]) + +/** Write the representative facet source tree into `dir`. */ +async function writeRepresentativeFacet(dir: string): Promise { + await Bun.write(join(dir, 'skills/planning/SKILL.md'), '# planning\n\nPlan things.\n') + await Bun.write(join(dir, 'skills/planning/references/api.md'), '# API reference\n') + await Bun.write(join(dir, 'skills/planning/assets/logo.bin'), REPRESENTATIVE_BINARY) + await Bun.write(join(dir, 'README.md'), '# Representative facet\n\nShips a README.\n') + await Bun.write( + join(dir, 'facet.json'), + JSON.stringify( + { + name: 'representative', + version: '1.0.0', + description: 'A representative 0.2 facet for stage interop', + files: ['README.md'], + skills: { + planning: { description: 'Planning skill', files: ['references/api.md', 'assets/logo.bin'] }, + }, + }, + null, + 2, + ), + ) +} + +const EXPECTED_MEMBERSHIP = [ + 'README.md', + 'facet.json', + 'skills/planning/SKILL.md', + 'skills/planning/assets/logo.bin', + 'skills/planning/references/api.md', +].sort() + +const gunzip: GunzipFn = async (bytes) => { + try { + return { ok: true, bytes: new Uint8Array(gunzipSync(bytes)) } + } catch { + return { ok: false, reason: 'corrupt' } + } +} + +describe('candidate 0.2 archive interop', () => { + test('the candidate CLI builds a verifiable 0.2 archive with exact membership', async () => { + const dir = await mkdtemp(join(testDir, 'build-')) + await writeRepresentativeFacet(dir) + + const built = await runCli(dir, 'build') + expect(built.exitCode).toBe(0) + + const archivePath = join(dir, 'dist/representative-1.0.0.facet') + expect(existsSync(archivePath)).toBe(true) + const outerBytes = new Uint8Array(await Bun.file(archivePath).arrayBuffer()) + + // Parse the outer container: the build manifest must be current 0.2. + const parsed = parseFacetArchive(outerBytes) + if (!parsed.ok) expect.unreachable() + expect(parsed.data.manifest.facetVersion).toBe(0.2) + + // Verify through the shared registry-grade verifier. + const verified = await validateFacetArchive(outerBytes, { gunzip }) + if (!verified.ok) expect.unreachable() + if (verified.data.archiveVersion !== 0.2) expect.unreachable() + + // Exact 0.2 membership, including the archive-only README and both + // skill companions (text + binary). + const observed = verified.data.entries.map((e) => e.path).sort() + expect(observed).toEqual(EXPECTED_MEMBERSHIP) + + // The archive-only README is classified as such (never a primary asset), + // and the binary companion is grouped with its owning skill. + const readme = verified.data.entries.find((e) => e.path === 'README.md') + expect(readme?.kind).toBe('archive-only') + const logo = verified.data.entries.find((e) => e.path === 'skills/planning/assets/logo.bin') + if (logo?.kind !== 'skill-companion') expect.unreachable() + expect(logo.skill).toBe('planning') + expect(logo.bytes).toEqual(REPRESENTATIVE_BINARY) + }) + + test('two candidate builds of the same source are byte-identical (deterministic)', async () => { + const build = async (name: string): Promise => { + const dir = await mkdtemp(join(testDir, `${name}-`)) + await writeRepresentativeFacet(dir) + const built = await runCli(dir, 'build') + expect(built.exitCode).toBe(0) + const archivePath = join(dir, 'dist/representative-1.0.0.facet') + return new Uint8Array(await Bun.file(archivePath).arrayBuffer()) + } + const a = await build('det-a') + const b = await build('det-b') + expect(Array.from(a)).toEqual(Array.from(b)) + }) +}) diff --git a/packages/cli/src/__tests__/create-build.e2e.test.ts b/packages/cli/src/__tests__/create-build.e2e.test.ts index 88590946..c7bc87a3 100644 --- a/packages/cli/src/__tests__/create-build.e2e.test.ts +++ b/packages/cli/src/__tests__/create-build.e2e.test.ts @@ -348,11 +348,14 @@ describe('facet build --verify', () => { const result = await runCli('build', dir, '--verify', '--json') expect(result.exitCode).toBe(0) const doc = JSON.parse(result.stdout) - expect(doc.schemaVersion).toBe('1') + expect(doc.schemaVersion).toBe('2') expect(doc.ok).toBe(true) expect(doc.verified).toBe(true) expect(doc.name).toBe('verifiable') - expect(Array.isArray(doc.assets)).toBe(true) + expect(doc.facetVersion).toBe(0.2) + // Complete inner-archive entry listing (includes facet.json + primaries). + expect(Array.isArray(doc.files)).toBe(true) + expect(doc.files).toContain('facet.json') expect(existsSync(join(dir, 'dist'))).toBe(false) }) diff --git a/packages/cli/src/commands/build.ts b/packages/cli/src/commands/build.ts index c019cc03..3ab88ffc 100644 --- a/packages/cli/src/commands/build.ts +++ b/packages/cli/src/commands/build.ts @@ -17,8 +17,15 @@ import { import { writeCliError } from '../util/errors.ts' import { resolveTargetDir } from './resolve-dir.ts' -/** Version tag for the machine-readable `--json` output document. */ -const BUILD_JSON_SCHEMA_VERSION = '1' +/** + * Version tag for the machine-readable `--json` output document. + * + * `2` replaces the `assets` array (primary-asset paths only) with a complete + * `files` array (every inner-archive entry — manifest, primaries, and + * supplementary files) and adds `facetVersion`. Consumers pinned to schema + * `1` must migrate rather than silently receive a different `assets` set. + */ +const BUILD_JSON_SCHEMA_VERSION = '2' export const buildCommand: Command = { name: 'build', @@ -108,9 +115,8 @@ export const buildCommand: Command = { await instance.waitUntilExit() // Ink has unmounted — print stdout summary for scroll-back const shortHash = integrity.length > 20 ? `${integrity.slice(0, 20)}...` : integrity - process.stdout.write( - `✓ Built ${buildName} v${buildVersion} → ${displayDir}/dist/ (${artifactCount} assets, ${shortHash})\n`, - ) + const entries = `${artifactCount} entr${artifactCount !== 1 ? 'ies' : 'y'}` + process.stdout.write(`✓ Built ${buildName} v${buildVersion} → ${displayDir}/dist/ (${entries}, ${shortHash})\n`) return 0 } catch { process.stdout.write( @@ -133,7 +139,10 @@ function printBuildJson(result: BuildResult | BuildFailure, verified: boolean): verified, name: result.data.name, version: result.data.version, - assets: Object.keys(result.assetHashes).sort(), + facetVersion: result.facetVersion, + // Complete inner-archive entry listing: facet.json, every primary + // asset, and every supplementary file (skill companions + archive-only). + files: Object.keys(result.fileHashes).sort(), integrity: result.integrity, warnings: result.warnings, } @@ -159,14 +168,16 @@ function printBuildPlain(result: BuildResult | BuildFailure, verified: boolean, process.stderr.write(`⚠ ${warning}\n`) } if (result.ok) { - const assetCount = Object.keys(result.assetHashes).length + // Count every inner-archive entry, not just primary assets. + const entryCount = Object.keys(result.fileHashes).length + const entries = `${entryCount} entr${entryCount !== 1 ? 'ies' : 'y'}` if (verified) { process.stdout.write( - `✓ Verified ${result.data.name} v${result.data.version} (${assetCount} asset${assetCount !== 1 ? 's' : ''}, no output written)\n`, + `✓ Verified ${result.data.name} v${result.data.version} (facetVersion ${result.facetVersion}, ${entries}, no output written)\n`, ) } else { process.stdout.write( - `✓ Built ${result.data.name} v${result.data.version} → ${displayDir}/dist/ (${assetCount} asset${assetCount !== 1 ? 's' : ''}, ${result.integrity})\n`, + `✓ Built ${result.data.name} v${result.data.version} → ${displayDir}/dist/ (facetVersion ${result.facetVersion}, ${entries}, ${result.integrity})\n`, ) } return diff --git a/packages/cli/src/tui/views/build/build-view.tsx b/packages/cli/src/tui/views/build/build-view.tsx index 31e00bfb..c4ffafb2 100644 --- a/packages/cli/src/tui/views/build/build-view.tsx +++ b/packages/cli/src/tui/views/build/build-view.tsx @@ -17,6 +17,8 @@ import { THEME } from '../../theme.ts' interface BuildViewResult { name: string version: string + facetVersion: number + /** Complete inner-archive entry listing (manifest, primaries, supplementary). */ files: string[] archiveFilename: string integrity: string @@ -98,12 +100,13 @@ export function BuildView({ try { await writeBuildOutput(pipelineResult, rootDir, { emitManifest }) - const files = Object.keys(pipelineResult.assetHashes).sort() + const files = Object.keys(pipelineResult.fileHashes).sort() updateStage('Writing output', { status: 'done' }) setResult({ name: pipelineResult.data.name, version: pipelineResult.data.version, + facetVersion: pipelineResult.facetVersion, files, archiveFilename: pipelineResult.archiveFilename, integrity: pipelineResult.integrity, @@ -183,13 +186,14 @@ export function BuildView({ Built successfully → dist/ {result.archiveFilename} + facetVersion {result.facetVersion} Archive contents: {result.files.map((f) => ( {f} ))} - {result.files.length} asset{result.files.length !== 1 ? 's' : ''} · {result.integrity} + {result.files.length} entr{result.files.length !== 1 ? 'ies' : 'y'} · {result.integrity} diff --git a/packages/engine/src/__tests__/build-pipeline.test.ts b/packages/engine/src/__tests__/build-pipeline.test.ts index 7f71e97f..64883528 100644 --- a/packages/engine/src/__tests__/build-pipeline.test.ts +++ b/packages/engine/src/__tests__/build-pipeline.test.ts @@ -319,9 +319,9 @@ describe('runBuildPipeline', () => { // Content hashing fields expect(result.archiveFilename).toBe('test-facet-1.0.0.facet') expect(result.archiveBytes.length).toBeGreaterThan(0) - expect(Object.keys(result.assetHashes)).toContain('facet.json') - expect(Object.keys(result.assetHashes)).toContain('skills/example/SKILL.md') - expect(result.assetHashes['skills/example/SKILL.md']).toMatchInlineSnapshot( + expect(Object.keys(result.fileHashes)).toContain('facet.json') + expect(Object.keys(result.fileHashes)).toContain('skills/example/SKILL.md') + expect(result.fileHashes['skills/example/SKILL.md']).toMatchInlineSnapshot( `"sha256:ded8057927e03783371d0d929e4a6e92da66eb9dd164377ad6845a5a1c0cb5ba"`, ) expect(result.integrity).toMatch(/^sha256:[a-f0-9]{64}$/) @@ -432,7 +432,7 @@ describe('runBuildPipeline', () => { expect(result.ok).toBe(true) if (result.ok) { expect(result.archiveFilename).toBe('multi-facet-2.0.0.facet') - const assetPaths = Object.keys(result.assetHashes).sort() + const assetPaths = Object.keys(result.fileHashes).sort() expect(assetPaths).toEqual([ 'agents/helper.md', 'commands/deploy.md', @@ -666,7 +666,7 @@ describe('content validation', () => { // The asset hash is computed over the verbatim file contents, so the // entry exists in the archive's per-asset hash map. - expect(result.assetHashes['skills/review/SKILL.md']).toMatch(/^sha256:/) + expect(result.fileHashes['skills/review/SKILL.md']).toMatch(/^sha256:/) }) test('build fails on empty content file', async () => { @@ -759,11 +759,14 @@ describe('writeBuildOutput', () => { const manifestEntry = outerEntries.find((e) => e.name === 'build-manifest.json') if (!manifestEntry) throw new Error('build-manifest.json not found in outer tar') const manifest = JSON.parse(manifestEntry.text) - expect(manifest.facetVersion).toBe(0.1) + // Producers now emit the current `0.2` flat build manifest with a + // complete `files` map; `0.1`/`assets` is a legacy consumer input only. + expect(manifest.facetVersion).toBe(0.2) expect(manifest.archive).toBe('archive.tar.gz') expect(manifest.integrity).toMatch(/^sha256:[a-f0-9]{64}$/) - expect(manifest.assets['facet.json']).toMatch(/^sha256:[a-f0-9]{64}$/) - expect(manifest.assets['skills/example/SKILL.md']).toMatch(/^sha256:[a-f0-9]{64}$/) + expect(manifest.assets).toBeUndefined() + expect(manifest.files['facet.json']).toMatch(/^sha256:[a-f0-9]{64}$/) + expect(manifest.files['skills/example/SKILL.md']).toMatch(/^sha256:[a-f0-9]{64}$/) // Inner archive contains expected assets const innerEntry = outerEntries.find((e) => e.name === 'archive.tar.gz') @@ -1112,3 +1115,215 @@ describe('runBuildPipeline — adapter API preflight', () => { expect(result.warnings).toEqual([]) }) }) + +// --- Producer: 0.2 supplementary files (task 11.5) --- +// +// The pure path-grammar/collision failure classes are exhaustively tested in +// packages/protocol/src/__tests__/archive-plan.test.ts, and the +// filesystem-identity classes in +// packages/engine/src/build/__tests__/load-supplementary-sources.test.ts. +// These pipeline-level tests prove the producer emits the current 0.2 archive +// with supplementary membership, hashes every entry, stays deterministic, and +// preserves prior dist/ on input failure. + +describe('runBuildPipeline — 0.2 supplementary files', () => { + async function writeFacet(dir: string, manifest: Record): Promise { + await Bun.write(join(dir, 'facet.json'), JSON.stringify(manifest)) + } + + test('archives a top-level README, a nested companion, and binary + empty bytes', async () => { + const dir = await createFixtureDir('supp-success') + await Bun.write(join(dir, 'skills/review/SKILL.md'), '# review\n') + await Bun.write(join(dir, 'README.md'), '# my facet\n') + await Bun.write(join(dir, 'skills/review/references/api.md'), 'api docs\n') + await Bun.write(join(dir, 'skills/review/assets/logo.bin'), new Uint8Array([0, 1, 2, 255])) + await Bun.write(join(dir, 'skills/review/EMPTY'), '') + await writeFacet(dir, { + name: 'supp', + version: '1.0.0', + files: ['README.md'], + skills: { + review: { description: 'r', files: ['references/api.md', 'assets/logo.bin', 'EMPTY'] }, + }, + }) + + const result = await runBuildPipeline(dir) + if (!result.ok) expect.unreachable() + + // Complete file map covers manifest + primary + every supplementary entry. + expect(Object.keys(result.fileHashes).sort()).toEqual([ + 'README.md', + 'facet.json', + 'skills/review/EMPTY', + 'skills/review/SKILL.md', + 'skills/review/assets/logo.bin', + 'skills/review/references/api.md', + ]) + // Empty file is hashed (SHA-256 of zero bytes). + expect(result.fileHashes['skills/review/EMPTY']).toBe(computeContentHash(new Uint8Array(0))) + // Binary bytes hashed verbatim. + expect(result.fileHashes['skills/review/assets/logo.bin']).toBe(computeContentHash(new Uint8Array([0, 1, 2, 255]))) + expect(result.facetVersion).toBe(0.2) + }) + + test('inner archive contains every supplementary entry byte-for-byte', async () => { + const dir = await createFixtureDir('supp-inner') + await Bun.write(join(dir, 'skills/review/SKILL.md'), '# review\n') + const binary = new Uint8Array([9, 8, 7, 0, 255]) + await Bun.write(join(dir, 'skills/review/logo.bin'), binary) + await Bun.write(join(dir, 'README.md'), '# readme\n') + await writeFacet(dir, { + name: 'supp', + version: '1.0.0', + files: ['README.md'], + skills: { review: { description: 'r', files: ['logo.bin'] } }, + }) + + const result = await runBuildPipeline(dir) + if (!result.ok) expect.unreachable() + + const outer = parseTar(result.archiveBytes) + const inner = outer.find((e) => e.name === 'archive.tar.gz') + if (!inner?.data) throw new Error('inner archive missing') + const innerFiles = await parseTarGzip(inner.data) + const logo = innerFiles.find((f) => f.name === 'skills/review/logo.bin') + expect(logo?.data ? new Uint8Array(logo.data) : undefined).toEqual(binary) + }) + + test('the embedded facet.json is hashed as its exact source bytes', async () => { + const dir = await createFixtureDir('supp-manifest-bytes') + await Bun.write(join(dir, 'skills/review/SKILL.md'), '# review\n') + // Deliberately non-canonical spacing — the hash must cover these exact bytes. + const manifestBytes = + '{\n "name": "supp",\n "version": "1.0.0",\n "skills": { "review": { "description": "r" } }\n}\n' + await Bun.write(join(dir, 'facet.json'), manifestBytes) + + const result = await runBuildPipeline(dir) + if (!result.ok) expect.unreachable() + expect(result.fileHashes['facet.json']).toBe(computeContentHash(manifestBytes)) + }) + + test('canonical tar output is byte-identical across two builds', async () => { + const build = async (name: string) => { + const dir = await createFixtureDir(name) + await Bun.write(join(dir, 'skills/review/SKILL.md'), '# review\n') + await Bun.write(join(dir, 'README.md'), '# readme\n') + await writeFacet(dir, { + name: 'supp', + version: '1.0.0', + files: ['README.md'], + skills: { review: { description: 'r' } }, + }) + const result = await runBuildPipeline(dir) + if (!result.ok) expect.unreachable() + return result + } + const a = await build('supp-determinism-a') + const b = await build('supp-determinism-b') + expect(a.integrity).toBe(b.integrity) + expect(Array.from(a.archiveBytes)).toEqual(Array.from(b.archiveBytes)) + }) + + test('a scoped facet identity writes under a nested dist path', async () => { + const dir = await createFixtureDir('supp-scoped') + await Bun.write(join(dir, 'skills/review/SKILL.md'), '# review\n') + await writeFacet(dir, { + name: '@acme/supp', + version: '2.0.0', + skills: { review: { description: 'r' } }, + }) + const result = await runBuildPipeline(dir) + if (!result.ok) expect.unreachable() + expect(result.archiveFilename).toBe('@acme/supp-2.0.0.facet') + await writeBuildOutput(result, dir) + expect(await Bun.file(join(dir, 'dist/@acme/supp-2.0.0.facet')).exists()).toBe(true) + }) + + test('a missing declared supplementary file fails and preserves prior dist output', async () => { + const dir = await createFixtureDir('supp-preserve-dist') + await Bun.write(join(dir, 'skills/review/SKILL.md'), '# review\n') + await writeFacet(dir, { + name: 'supp', + version: '1.0.0', + files: ['README.md'], + skills: { review: { description: 'r' } }, + }) + // README.md is declared but does NOT exist. Seed a prior dist/ artifact. + await Bun.write(join(dir, 'dist/prior.txt'), 'keep me') + + const result = await runBuildPipeline(dir) + if (result.ok) expect.unreachable() + if (result.kind !== 'validation') expect.unreachable() + expect(result.errors.some((e) => e.message.includes('README.md'))).toBe(true) + // Prior dist/ output is untouched — writeBuildOutput never ran. + expect(await Bun.file(join(dir, 'dist/prior.txt')).text()).toBe('keep me') + }) + + test('a traversal path in a declaration is rejected at manifest validation, preserving dist', async () => { + const dir = await createFixtureDir('supp-traversal') + await Bun.write(join(dir, 'skills/review/SKILL.md'), '# review\n') + await writeFacet(dir, { + name: 'supp', + version: '1.0.0', + files: ['../secret'], + skills: { review: { description: 'r' } }, + }) + await Bun.write(join(dir, 'dist/prior.txt'), 'keep me') + + const result = await runBuildPipeline(dir) + if (result.ok) expect.unreachable() + if (result.kind !== 'validation') expect.unreachable() + // The unsafe path is rejected before any output is touched. + expect(await Bun.file(join(dir, 'dist/prior.txt')).text()).toBe('keep me') + }) + + test('a supplementary/primary path collision is rejected at manifest validation', async () => { + const dir = await createFixtureDir('supp-collision') + await Bun.write(join(dir, 'skills/review/SKILL.md'), '# review\n') + await Bun.write(join(dir, 'agents/reviewer.md'), '# reviewer\n') + await writeFacet(dir, { + name: 'supp', + version: '1.0.0', + // Declares an archive-only file that collides with the agent primary path. + files: ['agents/reviewer.md'], + skills: { review: { description: 'r' } }, + agents: { reviewer: { description: 'a' } }, + }) + const result = await runBuildPipeline(dir) + if (result.ok) expect.unreachable() + expect(result.kind).toBe('validation') + }) + + test('undeclared source-tree files are never packaged', async () => { + const dir = await createFixtureDir('supp-undeclared') + await Bun.write(join(dir, 'skills/review/SKILL.md'), '# review\n') + // An undeclared stray file next to the manifest. + await Bun.write(join(dir, 'notes.txt'), 'private notes') + await writeFacet(dir, { + name: 'supp', + version: '1.0.0', + skills: { review: { description: 'r' } }, + }) + const result = await runBuildPipeline(dir) + if (!result.ok) expect.unreachable() + // Only declared entries are in the archive; the stray file is absent. + expect(Object.keys(result.fileHashes)).not.toContain('notes.txt') + }) + + test('an asset-only facet still emits 0.2 with a complete files map', async () => { + const dir = await createFixtureDir('supp-asset-only') + await Bun.write(join(dir, 'skills/review/SKILL.md'), '# review\n') + await writeFacet(dir, { + name: 'supp', + version: '1.0.0', + skills: { review: { description: 'r' } }, + }) + const result = await runBuildPipeline(dir) + if (!result.ok) expect.unreachable() + expect(result.facetVersion).toBe(0.2) + expect(Object.keys(result.fileHashes).sort()).toEqual(['facet.json', 'skills/review/SKILL.md']) + const manifest = JSON.parse(result.manifestJson) + expect(manifest.files).toBeDefined() + expect(manifest.assets).toBeUndefined() + }) +}) diff --git a/packages/engine/src/build/__tests__/load-supplementary-sources.test.ts b/packages/engine/src/build/__tests__/load-supplementary-sources.test.ts new file mode 100644 index 00000000..e2a7a64c --- /dev/null +++ b/packages/engine/src/build/__tests__/load-supplementary-sources.test.ts @@ -0,0 +1,187 @@ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test' +import { linkSync, mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { ArchivePlanEntry } from '@agent-facets/protocol' +import { collectArchiveEntriesFromPlan, loadSupplementarySources } from '../load-supplementary-sources.ts' + +/** + * Filesystem-identity failure matrix for the producer's supplementary-source + * loader (task 11.5). The pure path-grammar/collision classes are exercised in + * `packages/protocol/src/__tests__/archive-plan.test.ts`; this file owns only + * the disk-identity classes the archive plan cannot see: missing, symlink + * (target + parent), hard link, non-regular, out-of-tree escape, and + * resolved-source aliasing — plus the happy path and byte fidelity. + */ + +let root: string + +beforeEach(() => { + // realpath the root so macOS /var → /private/var doesn't cause spurious + // containment failures. + root = realpathSync(mkdtempSync(join(tmpdir(), 'supp-src-'))) +}) + +afterEach(() => { + rmSync(root, { recursive: true, force: true }) +}) + +/** An archive-only plan entry for a top-level path. */ +function archiveOnly(path: string): ArchivePlanEntry { + return { kind: 'archive-only', path } +} + +/** A skill-companion plan entry (path already prefixed with skills//). */ +function companion(skill: string, path: string): ArchivePlanEntry { + return { kind: 'skill-companion', path, skill } +} + +describe('loadSupplementarySources — happy path', () => { + test('loads top-level and nested companion bytes verbatim', async () => { + writeFileSync(join(root, 'README.md'), '# hi\n') + mkdirSync(join(root, 'skills/review/references'), { recursive: true }) + const binary = new Uint8Array([0, 1, 2, 255, 254]) + writeFileSync(join(root, 'skills/review/references/logo.bin'), binary) + + const result = await loadSupplementarySources(root, [ + archiveOnly('README.md'), + companion('review', 'skills/review/references/logo.bin'), + ]) + if (!result.ok) expect.unreachable() + + const readme = result.files.find((f) => f.archivePath === 'README.md') + const logo = result.files.find((f) => f.archivePath === 'skills/review/references/logo.bin') + expect(new TextDecoder().decode(readme?.content)).toBe('# hi\n') + expect(logo?.content).toEqual(binary) + }) + + test('an empty supplementary file loads as zero bytes', async () => { + writeFileSync(join(root, 'EMPTY'), '') + const result = await loadSupplementarySources(root, [archiveOnly('EMPTY')]) + if (!result.ok) expect.unreachable() + expect(result.files[0]?.content.length).toBe(0) + }) + + test('a plan with no supplementary entries loads nothing', async () => { + const result = await loadSupplementarySources(root, [{ kind: 'manifest', path: 'facet.json' }]) + if (!result.ok) expect.unreachable() + expect(result.files).toEqual([]) + }) +}) + +describe('loadSupplementarySources — filesystem-identity failures', () => { + test('missing declared file', async () => { + const result = await loadSupplementarySources(root, [archiveOnly('LICENSE')]) + if (result.ok) expect.unreachable() + expect(result.failures[0]?.code).toBe('missing') + }) + + test('declared path is a directory', async () => { + mkdirSync(join(root, 'docs'), { recursive: true }) + const result = await loadSupplementarySources(root, [archiveOnly('docs')]) + if (result.ok) expect.unreachable() + if (result.failures[0]?.code !== 'not-regular-file') expect.unreachable() + expect(result.failures[0].kind).toBe('directory') + }) + + test('declared path is a symlink to a regular file', async () => { + writeFileSync(join(root, 'real.md'), 'x') + symlinkSync(join(root, 'real.md'), join(root, 'link.md')) + const result = await loadSupplementarySources(root, [archiveOnly('link.md')]) + if (result.ok) expect.unreachable() + if (result.failures[0]?.code !== 'not-regular-file') expect.unreachable() + expect(result.failures[0].kind).toBe('symlink') + }) + + test('a symlinked parent component is rejected', async () => { + mkdirSync(join(root, 'realdir'), { recursive: true }) + writeFileSync(join(root, 'realdir/note.md'), 'x') + symlinkSync(join(root, 'realdir'), join(root, 'linkdir')) + const result = await loadSupplementarySources(root, [archiveOnly('linkdir/note.md')]) + if (result.ok) expect.unreachable() + if (result.failures[0]?.code !== 'symlinked-parent') expect.unreachable() + expect(result.failures[0].component).toBe('linkdir') + }) + + test('a hard link is rejected', async () => { + writeFileSync(join(root, 'original.md'), 'x') + linkSync(join(root, 'original.md'), join(root, 'hard.md')) + // Only declare the hard link; both files now have nlink === 2. + const result = await loadSupplementarySources(root, [archiveOnly('hard.md')]) + if (result.ok) expect.unreachable() + expect(result.failures[0]?.code).toBe('hard-link') + }) + + test('a symlink escaping the facet root is rejected', async () => { + const outside = realpathSync(mkdtempSync(join(tmpdir(), 'outside-'))) + writeFileSync(join(outside, 'secret.md'), 'x') + symlinkSync(outside, join(root, 'escape')) + const result = await loadSupplementarySources(root, [archiveOnly('escape/secret.md')]) + if (result.ok) expect.unreachable() + const code = result.failures[0]?.code + if (code === undefined) expect.unreachable() + // Caught as a symlinked parent before realpath containment even runs. + expect(['symlinked-parent', 'escapes-root']).toContain(code) + rmSync(outside, { recursive: true, force: true }) + }) + + test('two hard-linked declarations are each rejected as hard links', async () => { + // Two distinct spellings backed by one inode is the shape the + // resolved-source-alias guard exists for, but hard links carry nlink>1 and + // are rejected by the earlier hard-link check before aliasing is reached. + // This documents that the earlier, stricter check fires first: both + // declarations fail, and neither smuggles duplicate bytes into the archive. + writeFileSync(join(root, 'a.md'), 'x') + linkSync(join(root, 'a.md'), join(root, 'b.md')) + const result = await loadSupplementarySources(root, [archiveOnly('a.md'), archiveOnly('b.md')]) + if (result.ok) expect.unreachable() + expect(result.failures.map((f) => f.code)).toEqual(['hard-link', 'hard-link']) + }) + + test('a distinct single-link file is not flagged as an alias', async () => { + writeFileSync(join(root, 'one.md'), 'x') + writeFileSync(join(root, 'two.md'), 'y') + const result = await loadSupplementarySources(root, [archiveOnly('one.md'), archiveOnly('two.md')]) + if (!result.ok) expect.unreachable() + expect(result.files).toHaveLength(2) + }) +}) + +describe('collectArchiveEntriesFromPlan', () => { + test('assembles entries in the plan order with resolved content by kind', () => { + const plan: ArchivePlanEntry[] = [ + { kind: 'archive-only', path: 'README.md' }, + { kind: 'manifest', path: 'facet.json' }, + { kind: 'primary-asset', path: 'skills/review/SKILL.md', assetType: 'skill', name: 'review' }, + { kind: 'skill-companion', path: 'skills/review/api.md', skill: 'review' }, + ] + const resolved = { + name: 'x', + version: '1.0.0', + skills: { review: { description: 'd', prompt: '# review\n' } }, + } + const entries = collectArchiveEntriesFromPlan(plan, '{"name":"x"}', resolved, [ + { archivePath: 'README.md', content: new TextEncoder().encode('# readme\n') }, + { archivePath: 'skills/review/api.md', content: new TextEncoder().encode('api') }, + ]) + + // Order preserved from the (already-sorted) plan. + expect(entries.map((e) => e.path)).toEqual([ + 'README.md', + 'facet.json', + 'skills/review/SKILL.md', + 'skills/review/api.md', + ]) + // Manifest and primary carry their strings; supplementary carry bytes. + expect(entries[1]?.content).toBe('{"name":"x"}') + expect(entries[2]?.content).toBe('# review\n') + expect(entries[3]?.content).toBeInstanceOf(Uint8Array) + }) + + test('throws if a supplementary entry has no loaded bytes (pipeline bug)', () => { + const plan: ArchivePlanEntry[] = [{ kind: 'archive-only', path: 'README.md' }] + expect(() => collectArchiveEntriesFromPlan(plan, '{}', { name: 'x', version: '1.0.0' }, [])).toThrow( + /no loaded bytes/, + ) + }) +}) diff --git a/packages/engine/src/build/load-supplementary-sources.ts b/packages/engine/src/build/load-supplementary-sources.ts new file mode 100644 index 00000000..0049aed4 --- /dev/null +++ b/packages/engine/src/build/load-supplementary-sources.ts @@ -0,0 +1,385 @@ +import { lstat, readFile, realpath } from 'node:fs/promises' +import { dirname, join, sep } from 'node:path' +import type { ValidationError } from '@agent-facets/common' +import type { ArchiveEntry, ArchivePlanEntry, ResolvedFacetManifest } from '@agent-facets/protocol' + +/** + * Loads and validates the on-disk source of every declared supplementary + * file (skill companions and archive-only files) before the build produces + * any output. This is the filesystem-identity layer that sits on top of + * protocol's pure path-grammar layer (design D7): the archive plan has + * already proven the *spelling* of every path is safe and collision-free; + * this module proves the *resolved source* is a real, contained, regular + * file — symlinks (target or parent), hard links, non-regular files, and + * out-of-tree escapes are all rejected. + * + * Primary assets (`manifest`, `primary-asset`) are NOT handled here — the + * existing `resolvePrompts` path owns those. Only `skill-companion` and + * `archive-only` plan entries carry supplementary bytes. + * + * The check-then-read sequence has an irreducible TOCTOU window: `lstat`, + * `realpath`, and the final `readFile` are separate syscalls, so a path + * could in principle be swapped between validation and read. Build is a + * single-process operation over the author's own trusted source tree, so + * the threat model is malformed or accidental inputs (a stray symlink, a + * `README` that is actually a directory), not an active attacker racing the + * builder. We mitigate what we can — we read the same resolved absolute path + * we validated, and the bytes we hash are always the bytes we actually read — + * and accept the residual window rather than reaching for non-portable + * `openat`/`O_NOFOLLOW` primitives Node/Bun do not expose. + */ + +/** One resolved, verified supplementary source ready for archiving. */ +export interface LoadedSupplementaryFile { + /** Canonical inner-archive path (from the archive plan). */ + archivePath: string + /** Exact source bytes, read verbatim — binary and empty permitted (D6). */ + content: Uint8Array +} + +/** + * Pure-data failure for a single supplementary source. Discriminated by + * `code` so every failure class is distinguishable without parsing + * messages, and so the 11.5 test matrix gets one case per class. + */ +export type SupplementarySourceFailure = + | { code: 'missing'; archivePath: string; declarationSite: string; sourcePath: string } + | { + code: 'not-regular-file' + archivePath: string + declarationSite: string + sourcePath: string + kind: 'directory' | 'symlink' | 'other' + } + | { code: 'symlinked-parent'; archivePath: string; declarationSite: string; sourcePath: string; component: string } + | { code: 'hard-link'; archivePath: string; declarationSite: string; sourcePath: string; links: number } + | { code: 'escapes-root'; archivePath: string; declarationSite: string; sourcePath: string; resolved: string } + | { code: 'resolved-source-alias'; archivePath: string; declarationSite: string; collidesWith: string } + | { code: 'unreadable'; archivePath: string; declarationSite: string; sourcePath: string } + +export type LoadSupplementarySourcesResult = + | { ok: true; files: LoadedSupplementaryFile[] } + | { ok: false; failures: SupplementarySourceFailure[] } + +/** + * The declaration site an archive-plan entry originated from, for failure + * attribution. Skill companions are declared on their owning skill's `files` + * array; archive-only files are declared in top-level `files`. + */ +function declarationSiteFor(entry: Extract): string { + return entry.kind === 'skill-companion' ? `skills.${entry.skill}.files` : 'files' +} + +/** + * Load and validate every supplementary source declared by the archive plan. + * + * `rootDir` is the facet root; `plan` is the full tagged entry list from + * `planArchiveEntries`. Manifest and primary-asset entries are ignored here. + * Returns loaded bytes for every supplementary entry, or the complete set of + * structured failures. Collects all failures (does not stop at the first) so + * a build reports every bad declaration in one pass. + */ +export async function loadSupplementarySources( + rootDir: string, + plan: readonly ArchivePlanEntry[], +): Promise { + const supplementary = plan.filter( + (e): e is Extract => + e.kind === 'skill-companion' || e.kind === 'archive-only', + ) + + const failures: SupplementarySourceFailure[] = [] + const loaded: LoadedSupplementaryFile[] = [] + + // Realpath the facet root once so containment comparisons are stable under + // symlinked temp dirs (macOS `/var` → `/private/var`). + let rootReal: string + try { + rootReal = await realpath(rootDir) + } catch { + // A build cannot proceed without a resolvable root; surface every + // declared supplementary path as unreadable rather than throwing. + for (const entry of supplementary) { + failures.push({ + code: 'unreadable', + archivePath: entry.path, + declarationSite: declarationSiteFor(entry), + sourcePath: join(rootDir, entry.path), + }) + } + return { ok: false, failures } + } + const rootPrefix = rootReal.endsWith(sep) ? rootReal : `${rootReal}${sep}` + + /** Maps `${dev}:${ino}` → the first archivePath that resolved to it. */ + const identityByKey = new Map() + + for (const entry of supplementary) { + const declarationSite = declarationSiteFor(entry) + const sourcePath = join(rootDir, entry.path) + + // 1. Reject a symlink anywhere in the parent chain from the facet root + // down to the file's parent. `realpath` alone would silently *follow* a + // parent symlink that stays inside the root; walking with `lstat` rejects + // it. Only existing components are checked — a missing parent surfaces as + // a `missing` target below. + const parentSymlink = await firstSymlinkedParent(rootReal, dirname(sourcePath)) + if (parentSymlink !== null) { + failures.push({ + code: 'symlinked-parent', + archivePath: entry.path, + declarationSite, + sourcePath, + component: parentSymlink, + }) + continue + } + + // 2. lstat the final target (does NOT follow a final symlink). + let stats: Awaited> + try { + stats = await lstat(sourcePath) + } catch { + failures.push({ code: 'missing', archivePath: entry.path, declarationSite, sourcePath }) + continue + } + if (stats.isSymbolicLink()) { + failures.push({ code: 'not-regular-file', archivePath: entry.path, declarationSite, sourcePath, kind: 'symlink' }) + continue + } + if (stats.isDirectory()) { + failures.push({ + code: 'not-regular-file', + archivePath: entry.path, + declarationSite, + sourcePath, + kind: 'directory', + }) + continue + } + if (!stats.isFile()) { + failures.push({ code: 'not-regular-file', archivePath: entry.path, declarationSite, sourcePath, kind: 'other' }) + continue + } + if (stats.nlink > 1) { + failures.push({ code: 'hard-link', archivePath: entry.path, declarationSite, sourcePath, links: stats.nlink }) + continue + } + + // 3. realpath containment — belt-and-suspenders after the parent walk, + // catching a multi-hop escape the parent walk's single-level lstat cannot. + let real: string + try { + real = await realpath(sourcePath) + } catch { + failures.push({ code: 'unreadable', archivePath: entry.path, declarationSite, sourcePath }) + continue + } + if (real !== rootReal && !real.startsWith(rootPrefix)) { + failures.push({ code: 'escapes-root', archivePath: entry.path, declarationSite, sourcePath, resolved: real }) + continue + } + + // 4. Resolved-source identity: two declarations pointing at one inode + // are aliases the pure spelling-collision check cannot see. Keyed by + // (dev, ino); the `hard-link` check above already rejects the common + // in-tree case, but this catches a single file declared twice via + // distinct spellings that resolve to the same source. + const identityKey = `${stats.dev}:${stats.ino}` + const collidesWith = identityByKey.get(identityKey) + if (collidesWith !== undefined) { + failures.push({ code: 'resolved-source-alias', archivePath: entry.path, declarationSite, collidesWith }) + continue + } + identityByKey.set(identityKey, entry.path) + + // 5. Read the same resolved path we validated. The bytes we hash are the + // bytes we read, so a swap between validation and read produces a hash + // the reviewer sees rather than a validated-but-unhashed file. + let content: Uint8Array + try { + content = new Uint8Array(await readFile(real)) + } catch { + failures.push({ code: 'unreadable', archivePath: entry.path, declarationSite, sourcePath }) + continue + } + + loaded.push({ archivePath: entry.path, content }) + } + + if (failures.length > 0) { + return { ok: false, failures } + } + return { ok: true, files: loaded } +} + +/** + * Assemble the complete, deterministically ordered archive-entry list + * directly from the shared archive plan (design D3) — the single source of + * truth for archive membership and ordering. Every planned entry is resolved + * to its content by kind: + * + * - `manifest` → the exact embedded `facet.json` bytes, + * - `primary-asset` → the resolved prompt text for that asset, + * - `skill-companion` → the loaded supplementary bytes, + * - `archive-only` → the loaded supplementary bytes. + * + * The plan is already sorted lexicographically by path, so the resulting + * entry list — and therefore the assembled tar — is deterministic regardless + * of manifest declaration order. Binary and empty supplementary content pass + * through verbatim as `Uint8Array`; primary content is passed through as-is. + * + * Preconditions (guaranteed by earlier pipeline stages): every primary asset + * in the plan has a resolved prompt, and every supplementary entry has loaded + * bytes. A missing lookup indicates a pipeline bug and is surfaced as a + * thrown error rather than silently dropping an entry. + */ +export function collectArchiveEntriesFromPlan( + plan: readonly ArchivePlanEntry[], + manifestBytes: string, + resolved: ResolvedFacetManifest, + supplementaryFiles: readonly LoadedSupplementaryFile[], +): ArchiveEntry[] { + const supplementaryByPath = new Map(supplementaryFiles.map((f) => [f.archivePath, f.content])) + + return plan.map((entry): ArchiveEntry => ({ path: entry.path, content: contentForEntry(entry) })) + + function contentForEntry(entry: ArchivePlanEntry): string | Uint8Array { + switch (entry.kind) { + case 'manifest': + return manifestBytes + case 'primary-asset': { + const prompt = primaryPromptFor(resolved, entry) + if (prompt === undefined) { + throw new Error(`Archive plan references primary asset ${entry.path} with no resolved prompt`) + } + return prompt + } + case 'skill-companion': + case 'archive-only': { + const bytes = supplementaryByPath.get(entry.path) + if (bytes === undefined) { + throw new Error(`Archive plan references supplementary file ${entry.path} with no loaded bytes`) + } + return bytes + } + default: { + const unreachable: never = entry + return unreachable + } + } + } +} + +/** Look up a primary asset's resolved prompt text by its plan entry. */ +function primaryPromptFor( + resolved: ResolvedFacetManifest, + entry: Extract, +): string | undefined { + switch (entry.assetType) { + case 'skill': + return resolved.skills?.[entry.name]?.prompt + case 'agent': + return resolved.agents?.[entry.name]?.prompt + case 'command': + return resolved.commands?.[entry.name]?.prompt + } +} + +/** + * Translate a structured supplementary-source failure into the project-wide + * `ValidationError` shape the build pipeline reports. The `code` is preserved + * in the message so the class remains identifiable in rendered output, while + * `path` carries the declaration site for field attribution. + */ +export function supplementarySourceFailureToValidationError(failure: SupplementarySourceFailure): ValidationError { + switch (failure.code) { + case 'missing': + return { + path: failure.declarationSite, + message: `Declared file not found: ${failure.archivePath} (resolved to ${failure.sourcePath}).`, + expected: 'an existing regular file', + actual: 'file not found', + } + case 'not-regular-file': + return { + path: failure.declarationSite, + message: `Declared file ${failure.archivePath} is a ${failure.kind}, not a regular file.`, + expected: 'a regular file', + actual: failure.kind, + } + case 'symlinked-parent': + return { + path: failure.declarationSite, + message: `Declared file ${failure.archivePath} resolves through a symlinked parent directory "${failure.component}". Links are not permitted in supplementary source paths.`, + expected: 'no symlinked path components', + actual: `symlinked component "${failure.component}"`, + } + case 'hard-link': + return { + path: failure.declarationSite, + message: `Declared file ${failure.archivePath} is a hard link (${failure.links} links). Links are not permitted in supplementary source paths.`, + expected: 'a regular file with a single link', + actual: `hard link (${failure.links} links)`, + } + case 'escapes-root': + return { + path: failure.declarationSite, + message: `Declared file ${failure.archivePath} resolves to ${failure.resolved}, outside the facet root.`, + expected: 'a source inside the facet root', + actual: 'a source outside the facet root', + } + case 'resolved-source-alias': + return { + path: failure.declarationSite, + message: `Declared file ${failure.archivePath} resolves to the same source as ${failure.collidesWith}. Each declaration must reference a distinct file.`, + expected: 'distinct source files', + actual: 'two declarations sharing one source', + } + case 'unreadable': + return { + path: failure.declarationSite, + message: `Declared file ${failure.archivePath} could not be read from ${failure.sourcePath}.`, + expected: 'a readable regular file', + actual: 'unreadable file', + } + } +} + +/** + * Walk every existing directory component from `rootReal` (exclusive) down to + * and including `dir`, returning the first component that is a symlink, or + * `null` if none are. Non-existent components are not symlinks (the missing + * target is reported separately), so a lstat failure ends the walk cleanly. + */ +async function firstSymlinkedParent(rootReal: string, dir: string): Promise { + // Only inspect components at or below the root. `dir` may equal the root + // (a top-level file's parent), in which case there is nothing to walk. + const rootPrefix = rootReal.endsWith(sep) ? rootReal : `${rootReal}${sep}` + if (dir === rootReal) return null + if (!dir.startsWith(rootPrefix)) { + // The parent isn't under the (realpathed) root at all — containment will + // reject the target; nothing symlink-specific to report here. + return null + } + + const relative = dir.slice(rootPrefix.length) + const components = relative.split(sep).filter((c) => c.length > 0) + + let current = rootReal + for (const component of components) { + current = join(current, component) + let stats: Awaited> + try { + stats = await lstat(current) + } catch { + // A missing intermediate component isn't a symlink; stop walking and + // let the target lstat report `missing`. + return null + } + if (stats.isSymbolicLink()) { + return component + } + } + return null +} diff --git a/packages/engine/src/build/pipeline.ts b/packages/engine/src/build/pipeline.ts index 00d66f5f..f0363287 100644 --- a/packages/engine/src/build/pipeline.ts +++ b/packages/engine/src/build/pipeline.ts @@ -4,12 +4,13 @@ import type { ValidationError } from '@agent-facets/common' import { assembleOuterTar, assembleTar, - collectArchiveEntries, computeAssetHashes, computeContentHash, detectNamingCollisions, + FACET_ARCHIVE_VERSION, FACET_MANIFEST_FILE, INNER_ARCHIVE_NAME, + planArchiveEntries, type ResolvedFacetManifest, validateCompactFacets, validateContentFiles, @@ -19,6 +20,12 @@ import { jsonFileText } from '../json-file-text.ts' import { loadManifest, resolvePrompts } from '../loaders/facet.ts' import { buildArtifactFilename } from '../registry/artifact-path.ts' import { compressArchive } from './compress.ts' +import { + collectArchiveEntriesFromPlan, + type LoadedSupplementaryFile, + loadSupplementarySources, + supplementarySourceFailureToValidationError, +} from './load-supplementary-sources.ts' import { validateAdapterMetadata } from './validate-adapters.ts' export interface BuildProgress { @@ -30,11 +37,20 @@ export interface BuildResult { ok: true data: ResolvedFacetManifest warnings: string[] + /** The emitted archive-format version (currently always `0.2`). */ + facetVersion: number /** The complete .facet file bytes (outer uncompressed tar containing manifest + inner archive) */ archiveBytes: Uint8Array integrity: string archiveFilename: string - assetHashes: Record + /** + * Complete per-entry hash map for every inner-archive entry: `facet.json`, + * every primary asset, and every supplementary file (skill companions and + * archive-only files). Keyed by canonical inner-archive path. This is the + * full `files` map embedded in the `0.2` build manifest, not a + * primary-asset-only subset. + */ + fileHashes: Record /** Serialized build-manifest.json content (for --emit-manifest and test verification) */ manifestJson: string } @@ -56,6 +72,7 @@ export type BuildFailure = export const BUILD_STAGES = [ 'Parsing manifest', 'Resolving prompts', + 'Loading files', 'Validating assets', 'Checking collisions', 'Validating adapters', @@ -124,6 +141,36 @@ export async function runBuildPipeline( onProgress?.({ stage: 'Resolving prompts', status: 'done' }) + // Stage 2b: Derive the shared archive plan and load every declared + // supplementary file's exact bytes, validating its resolved source + // identity (regular file, no links, contained in the tree) BEFORE any + // output is touched. `planArchiveEntries` re-runs the pure path-grammar + // and collision checks the manifest schema already narrowed on; a + // validated manifest always yields `ok: true`, so a plan failure here is + // defensive. All disk-identity failures preserve previous `dist/` output + // because the caller only writes on `ok: true`. + onProgress?.({ stage: 'Loading files', status: 'running' }) + + const plan = planArchiveEntries(manifest) + if (!plan.ok) { + onProgress?.({ stage: 'Loading files', status: 'failed' }) + return { ok: false, kind: 'validation', errors: plan.errors, warnings } + } + + const supplementaryResult = await loadSupplementarySources(rootDir, plan.data) + if (!supplementaryResult.ok) { + onProgress?.({ stage: 'Loading files', status: 'failed' }) + return { + ok: false, + kind: 'validation', + errors: supplementaryResult.failures.map(supplementarySourceFailureToValidationError), + warnings, + } + } + const supplementaryFiles: LoadedSupplementaryFile[] = supplementaryResult.files + + onProgress?.({ stage: 'Loading files', status: 'done' }) + // Stage 3: Validate assets (no empty files; author front matter is OK) onProgress?.({ stage: 'Validating assets', status: 'running' }) @@ -170,19 +217,29 @@ export async function runBuildPipeline( const resolved = resolveResult.data const manifestContent = await Bun.file(join(rootDir, FACET_MANIFEST_FILE)).text() - const entries = collectArchiveEntries(resolved, manifestContent) - const assetHashes = computeAssetHashes(entries) + // Membership and ordering come from the one shared archive plan (design + // D3) — the same derivation verification consumes — so producer and + // verifier can never disagree about which paths an archive contains. The + // plan is pre-sorted, so tar assembly is deterministic. + const entries = collectArchiveEntriesFromPlan(plan.data, manifestContent, resolved, supplementaryFiles) + const fileHashes = computeAssetHashes(entries) const tarBytes = assembleTar(entries) const integrity = computeContentHash(tarBytes) const innerArchiveBytes = compressArchive(tarBytes) const archiveFilename = buildArtifactFilename(resolved.name, resolved.version) - // Build the build manifest and wrap into the outer tar + // Build the current `0.2` build manifest and wrap into the outer tar. + // Every build — asset-only or with supplementary files — emits the + // current flat shape: `facetVersion: 0.2`, the canonical `archive` name, + // `integrity`, and a complete `files` map (manifest + primaries + + // supplementary), derived from the shared plan above. `0.1`/`assets` + // remains a legacy *consumer* input only; producers never emit it. There + // is no runtime flag or conditional dual-format mode. const buildManifest = { - facetVersion: 0.1, + facetVersion: FACET_ARCHIVE_VERSION, archive: INNER_ARCHIVE_NAME, integrity, - assets: assetHashes, + files: fileHashes, } const manifestJson = jsonFileText(buildManifest) const archiveBytes = assembleOuterTar(manifestJson, innerArchiveBytes) @@ -193,10 +250,11 @@ export async function runBuildPipeline( ok: true, data: resolved, warnings, + facetVersion: FACET_ARCHIVE_VERSION, archiveBytes, integrity, archiveFilename, - assetHashes, + fileHashes, manifestJson, } } diff --git a/packages/engine/src/install/__tests__/run-add.test.ts b/packages/engine/src/install/__tests__/run-add.test.ts index 4cd84330..96ed0241 100644 --- a/packages/engine/src/install/__tests__/run-add.test.ts +++ b/packages/engine/src/install/__tests__/run-add.test.ts @@ -29,7 +29,7 @@ async function manifestFor(fixtureDir: string) { const { runBuildPipeline } = await import('../../build/pipeline.ts') const built = await runBuildPipeline(fixtureDir, []) if (!built.ok) throw new Error('test bug: fixture failed to build') - return JSON.parse(built.manifestJson) as import('@agent-facets/protocol').BuildManifest + return JSON.parse(built.manifestJson) as import('@agent-facets/protocol').CurrentBuildManifest } mock.module('../../registry/resolve-metadata.ts', () => ({ @@ -55,7 +55,7 @@ mock.module('../../registry/download.ts', () => ({ } cpSync(registryFixtureDir, dest, { recursive: true }) const manifest = await manifestFor(registryFixtureDir) - return { ok: true, value: { integrity: manifest.integrity, fileHashes: manifest.assets } } + return { ok: true, value: { integrity: manifest.integrity, fileHashes: manifest.files } } }, })) diff --git a/packages/engine/src/install/__tests__/run-install.chain.test.ts b/packages/engine/src/install/__tests__/run-install.chain.test.ts index 514e06be..648e6af3 100644 --- a/packages/engine/src/install/__tests__/run-install.chain.test.ts +++ b/packages/engine/src/install/__tests__/run-install.chain.test.ts @@ -3,7 +3,7 @@ import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, import { tmpdir } from 'node:os' import { join } from 'node:path' import { ADAPTER_API_VERSION } from '@agent-facets/adapter/api-version' -import type { BuildManifest, LockfileFacet } from '@agent-facets/protocol' +import type { BuildManifest, CurrentBuildManifest, LockfileFacet } from '@agent-facets/protocol' import { LOCKFILE_VERSION } from '@agent-facets/protocol' import type { Addition } from '../types.ts' @@ -45,11 +45,11 @@ function describeSpec(spec: { kind: string; major?: number; minor?: number; patc } } -async function manifestFor(fixtureDir: string): Promise { +async function manifestFor(fixtureDir: string): Promise { const { runBuildPipeline } = await import('../../build/pipeline.ts') const built = await runBuildPipeline(fixtureDir, []) if (!built.ok) throw new Error('test bug: fixture failed to build') - return JSON.parse(built.manifestJson) as BuildManifest + return JSON.parse(built.manifestJson) as CurrentBuildManifest } mock.module('../../registry/resolve-metadata.ts', () => ({ @@ -83,7 +83,7 @@ mock.module('../../registry/download.ts', () => ({ } cpSync(fixture, dest, { recursive: true }) const manifest = await manifestFor(fixture) - return { ok: true, value: { integrity: manifest.integrity, fileHashes: manifest.assets } } + return { ok: true, value: { integrity: manifest.integrity, fileHashes: manifest.files } } }, })) diff --git a/packages/engine/src/install/__tests__/run-install.receipt.test.ts b/packages/engine/src/install/__tests__/run-install.receipt.test.ts index b61e46f5..c82d261e 100644 --- a/packages/engine/src/install/__tests__/run-install.receipt.test.ts +++ b/packages/engine/src/install/__tests__/run-install.receipt.test.ts @@ -3,7 +3,7 @@ import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, import { tmpdir } from 'node:os' import { join } from 'node:path' import { ADAPTER_API_VERSION } from '@agent-facets/adapter/api-version' -import type { BuildManifest } from '@agent-facets/protocol' +import type { CurrentBuildManifest } from '@agent-facets/protocol' import { CURRENT_RECEIPT_VERSION, type Receipt } from '../receipt.ts' import type { Addition, StageEvent } from '../types.ts' @@ -25,11 +25,11 @@ let fixtureForVersion: FixtureForVersion = () => null let resolveRequests: Array<{ name: string; version: string }> = [] let metadataOffline = false -async function manifestFor(fixtureDir: string): Promise { +async function manifestFor(fixtureDir: string): Promise { const { runBuildPipeline } = await import('../../build/pipeline.ts') const built = await runBuildPipeline(fixtureDir, []) if (!built.ok) throw new Error('test bug: fixture failed to build') - return JSON.parse(built.manifestJson) as BuildManifest + return JSON.parse(built.manifestJson) as CurrentBuildManifest } mock.module('../../registry/resolve-metadata.ts', () => ({ @@ -63,7 +63,7 @@ mock.module('../../registry/download.ts', () => ({ } cpSync(fixture, dest, { recursive: true }) const manifest = await manifestFor(fixture) - return { ok: true, value: { integrity: manifest.integrity, fileHashes: manifest.assets } } + return { ok: true, value: { integrity: manifest.integrity, fileHashes: manifest.files } } }, })) diff --git a/packages/engine/src/install/__tests__/run-install.test.ts b/packages/engine/src/install/__tests__/run-install.test.ts index f16e4939..5b5ad3f5 100644 --- a/packages/engine/src/install/__tests__/run-install.test.ts +++ b/packages/engine/src/install/__tests__/run-install.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import type { Adapter } from '@agent-facets/adapter' import { ADAPTER_API_VERSION } from '@agent-facets/adapter/api-version' -import type { BuildManifest } from '@agent-facets/protocol' +import type { CurrentBuildManifest } from '@agent-facets/protocol' /** * Tests for `runInstall`'s manifest-vs-lockfile reconciliation. @@ -46,12 +46,13 @@ function describeSpec(spec: { kind: string; major?: number; minor?: number; patc } /** Build the fixture's genuine build manifest — the same artifact a real - * registry would serve in the outer tar. */ -async function manifestFor(fixtureDir: string): Promise { + * registry would serve in the outer tar. Producers emit the current `0.2` + * flat shape (`files` map). */ +async function manifestFor(fixtureDir: string): Promise { const { runBuildPipeline } = await import('../../build/pipeline.ts') const built = await runBuildPipeline(fixtureDir, []) if (!built.ok) throw new Error('test bug: fixture failed to build') - return JSON.parse(built.manifestJson) as BuildManifest + return JSON.parse(built.manifestJson) as CurrentBuildManifest } mock.module('../../registry/resolve-metadata.ts', () => ({ @@ -90,7 +91,7 @@ mock.module('../../registry/download.ts', () => ({ } cpSync(fixture, dest, { recursive: true }) const manifest = await manifestFor(fixture) - return { ok: true, value: { integrity: manifest.integrity, fileHashes: manifest.assets } } + return { ok: true, value: { integrity: manifest.integrity, fileHashes: manifest.files } } }, })) diff --git a/packages/engine/src/install/__tests__/run-remove.test.ts b/packages/engine/src/install/__tests__/run-remove.test.ts index 31723c93..2106ba42 100644 --- a/packages/engine/src/install/__tests__/run-remove.test.ts +++ b/packages/engine/src/install/__tests__/run-remove.test.ts @@ -29,7 +29,7 @@ async function manifestFor(fixtureDir: string) { const { runBuildPipeline } = await import('../../build/pipeline.ts') const built = await runBuildPipeline(fixtureDir, []) if (!built.ok) throw new Error('test bug: fixture failed to build') - return JSON.parse(built.manifestJson) as import('@agent-facets/protocol').BuildManifest + return JSON.parse(built.manifestJson) as import('@agent-facets/protocol').CurrentBuildManifest } mock.module('../../registry/resolve-metadata.ts', () => ({ @@ -55,7 +55,7 @@ mock.module('../../registry/download.ts', () => ({ } cpSync(registryFixtureDir, dest, { recursive: true }) const manifest = await manifestFor(registryFixtureDir) - return { ok: true, value: { integrity: manifest.integrity, fileHashes: manifest.assets } } + return { ok: true, value: { integrity: manifest.integrity, fileHashes: manifest.files } } }, }))