From 789450c40bc01917f3cf9a3ade6ab2ec19963516 Mon Sep 17 00:00:00 2001 From: Chumaniac <121109854+Chumaniac@users.noreply.github.com> Date: Wed, 7 Oct 2026 01:21:01 +0800 Subject: [PATCH] feat: add verified artifact delivery and acceptance --- CHANGELOG.md | 4 + README.md | 3 + docs/artifact-delivery.md | 72 +++++++++ fixtures/product/order-summary/README.md | 9 ++ .../order-summary/artifacts/metrics.csv | 3 + .../order-summary/artifacts/summary.json | 6 + fixtures/product/order-summary/contract.json | 78 ++++++++++ src/cli/commands/artifacts.ts | 142 ++++++++++++++++++ src/cli/index.ts | 14 ++ src/domain/artifact-content.ts | 129 ++++++++++++++++ src/domain/artifact-contract.ts | 81 ++++++++++ src/domain/strict-json.ts | 30 ++++ src/sandbox/workspace-tree.ts | 132 ++++++++++++---- tests/domain/artifact-content.test.ts | 78 ++++++++++ tests/integration/artifacts.test.ts | 106 +++++++++++++ 15 files changed, 858 insertions(+), 29 deletions(-) create mode 100644 docs/artifact-delivery.md create mode 100644 fixtures/product/order-summary/README.md create mode 100644 fixtures/product/order-summary/artifacts/metrics.csv create mode 100644 fixtures/product/order-summary/artifacts/summary.json create mode 100644 fixtures/product/order-summary/contract.json create mode 100644 src/cli/commands/artifacts.ts create mode 100644 src/domain/artifact-content.ts create mode 100644 src/domain/artifact-contract.ts create mode 100644 src/domain/strict-json.ts create mode 100644 tests/domain/artifact-content.test.ts create mode 100644 tests/integration/artifacts.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 2067bcf..a99ec23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- Candidate `artifacts` command validates actual local files with strict capacity, + CSV/JSON contracts, digest binding and cross-file summaries. Reports omit values + and local paths. Shared scans use bounded reads, metadata allowlists, streamed + enumeration, no-follow file opens and snapshot-change checks. - `report` now requires matching plan digests when both an ActionPlan and an ApplyReceipt are supplied. Mismatched evidence fails before rendering any Markdown, JSON, or SARIF output, with an error that does not expose local diff --git a/README.md b/README.md index c125b42..38e21ed 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,9 @@ The command above verifies the included sample Skill. Replace the fixture path w ## What you get +- The local source candidate `artifacts` command independently checks actual + CSV/JSON delivery files, Receipt/file hashes and cross-file count/integer sums. + See [actual artifact checks](docs/artifact-delivery.md); it runs no Skill code. - `verify` reviews one local Skill for provenance, target compatibility, and changes without running its scripts. - `scan` inventories local Skills, while `compat` checks their declared features against agent profiles. - `diff` shows the meaningful changes between two Skill versions before you accept them. diff --git a/docs/artifact-delivery.md b/docs/artifact-delivery.md new file mode 100644 index 0000000..74d8c89 --- /dev/null +++ b/docs/artifact-delivery.md @@ -0,0 +1,72 @@ +# Actual artifact delivery checks + +The candidate source command `artifacts` checks local output files against a +versioned contract. It opens actual bytes; it does not execute a Skill, Replay, +Docker, provider or remote service. This command is not in the published npm0.1.0 +package. Existing `test` preflight and Replay event evidence keep their semantics. + +## Run a physical check + +```bash +npm ci +npm run build +node dist/cli/index.js artifacts \ + --contract fixtures/product/order-summary/contract.json \ + --path fixtures/product/order-summary/artifacts --format json +``` + +Use `--delivery /path/to/bundle` instead of `--path` for a SkillTape source bundle +created by `verify --delivery-dir`. The checker validates its layout, exact +Receipt hash, artifact hashes/sizes, and deterministic artifact-set digest before +accepting a complete result. A consistently rewritten manifest is not signed or +authenticated evidence; the report marks provenance as not authenticated. + +## Declared requirements + +The [synthetic contract](../fixtures/product/order-summary/contract.json) declares: + +- capacity ceilings and a complete allowlist of required files; +- exact CSV headers, scalar types, real calendar dates, row limits, unique keys, + optional expected hashes, and rejection of formula-risk text prefixes; +- JSON top-level field types, required values and optional rejection of extras; +- independently recomputed CSV record counts and safe integer sums compared with + declared JSON fields. Integer sums avoid unspecified floating-point tolerances. + +`bytes` rules check integrity and capacity without interpreting content. JSON +checks cover declared top-level fields, not full JSON Schema or arbitrary +expressions. Contract files are limited to 64 KiB. Limits must be explicit and +within the shared 10,000-file/16 MiB-file/64 MiB-total workspace ceilings. +Duplicate JSON keys are rejected, including escaped-key aliases in metadata and +payloads. JSON depth is capped at 64 and object keys at 100,000; individual keys +are at most 1,024 characters. CSV rows stop at the declared field count instead +of allocating an array for millions of separators. Manifest order is checked +by UTF-8 path bytes. Directory allowlists use precomputed prefix sets. + +The inventory is streamed and checked before payload reads. Extra paths, links, +special files, excess depth/entries and capacity overruns fail. Single-file reads +are bounded and no-follow where supported; metadata and content changes during +inspection fail. This is inspection of a stable caller-selected snapshot, not an +OS sandbox against a same-privilege process actively replacing directory parents. + +CSV parsing preserves spaces and supports quoted commas, multiline values, +escaped quotes and CRLF, plus the common LF extension. The format basis is +[RFC 4180](https://www.rfc-editor.org/rfc/rfc4180.html). Text-prefix checks include +ASCII/full-width formula starters and leading controls discussed by +[OWASP CSV Injection](https://community.owasp.org/attacks/CSV_Injection). +The checker rejects risky parsed cells; it neither rewrites files nor guarantees +safe interpretation in every spreadsheet application or locale. + +## Reports and exit codes + +Exit0 means declared checks passed, exit1 an artifact finding, and exit2 invalid +input/contract. JSON/text reports contain relative paths, digests, sizes, record +counts and stable findings, not raw cells, JSON values, process output or host +absolute paths. Cross-file totals remain internal. Findings identify the file, +declared field and data-record number where appropriate. + +The [fixture](../fixtures/product/order-summary/README.md) proves physical file +checks with fictional outputs. The complete cross-project example separately +runs a local summarizer, preserves a SkillTape bundle, passes this checker and +rejects corruption and an internally inconsistent summary in copies. None of +those checks establishes customer-data truth, model/Agent acceptance or a live +production/business integration. diff --git a/fixtures/product/order-summary/README.md b/fixtures/product/order-summary/README.md new file mode 100644 index 0000000..0687b10 --- /dev/null +++ b/fixtures/product/order-summary/README.md @@ -0,0 +1,9 @@ +# Synthetic order summary artifacts + +These fictional outputs exercise actual-file inspection, not a provider or database. +Use the source CLI with `artifacts --contract fixtures/product/order-summary/contract.json +--path fixtures/product/order-summary/artifacts --format json`. + +The contract verifies exact headers, valid dates, nonnegative safe integers, unique +days, required JSON fields and schema_version1, then independently compares CSV +record counts and integer sums with the summary. Reports omit cell and JSON values. diff --git a/fixtures/product/order-summary/artifacts/metrics.csv b/fixtures/product/order-summary/artifacts/metrics.csv new file mode 100644 index 0000000..7615643 --- /dev/null +++ b/fixtures/product/order-summary/artifacts/metrics.csv @@ -0,0 +1,3 @@ +day,orders,total_cents +2026-01-01,2,2000 +2026-01-02,1,1600 diff --git a/fixtures/product/order-summary/artifacts/summary.json b/fixtures/product/order-summary/artifacts/summary.json new file mode 100644 index 0000000..1dca429 --- /dev/null +++ b/fixtures/product/order-summary/artifacts/summary.json @@ -0,0 +1,6 @@ +{ + "schema_version": 1, + "row_count": 2, + "total_orders": 3, + "total_cents": 3600 +} diff --git a/fixtures/product/order-summary/contract.json b/fixtures/product/order-summary/contract.json new file mode 100644 index 0000000..5d9fb95 --- /dev/null +++ b/fixtures/product/order-summary/contract.json @@ -0,0 +1,78 @@ +{ + "schema": "skillsync.artifacts/v1", + "limits": { + "max_files": 8, + "max_file_bytes": 1048576, + "max_total_bytes": 8388608 + }, + "files": [ + { + "path": "metrics.csv", + "format": "csv", + "max_rows": 1000, + "unique_by": [ + "day" + ], + "columns": [ + { + "name": "day", + "type": "date" + }, + { + "name": "orders", + "type": "integer", + "min": 0 + }, + { + "name": "total_cents", + "type": "integer", + "min": 0 + } + ] + }, + { + "path": "summary.json", + "format": "json", + "fields": [ + { + "name": "schema_version", + "type": "integer", + "equals": 1 + }, + { + "name": "row_count", + "type": "integer", + "min": 0 + }, + { + "name": "total_orders", + "type": "integer", + "min": 0 + }, + { + "name": "total_cents", + "type": "integer", + "min": 0 + } + ] + } + ], + "checks": [ + { + "type": "csv_summary", + "csv": "metrics.csv", + "json": "summary.json", + "row_count_field": "row_count", + "sums": [ + { + "column": "orders", + "field": "total_orders" + }, + { + "column": "total_cents", + "field": "total_cents" + } + ] + } + ] +} diff --git a/src/cli/commands/artifacts.ts b/src/cli/commands/artifacts.ts new file mode 100644 index 0000000..9bc7fcf --- /dev/null +++ b/src/cli/commands/artifacts.ts @@ -0,0 +1,142 @@ +import { createHash } from "node:crypto"; +import { lstat, readdir } from "node:fs/promises"; +import { join } from "node:path"; +import { parse as parseYaml } from "yaml"; +import { z } from "zod"; + +import { parseArtifactContract } from "../../domain/artifact-contract.js"; +import { parseStrictJson } from "../../domain/strict-json.js"; +import { inspectArtifactContent, type ArtifactFacts, type ArtifactFinding } from "../../domain/artifact-content.js"; +import { redactLocalPaths } from "../../reporters/local-paths.js"; +import { readWorkspaceFile, scanStagedWorkspace, WorkspaceTreeError } from "../../sandbox/workspace-tree.js"; + +const hash = (bytes: Uint8Array | string) => createHash("sha256").update(bytes).digest("hex"); +const sha = z.string().regex(/^[0-9a-f]{64}$/); +const manifestSchema = z.object({ schema: z.literal("skilltape.dev/delivery/v1"), + run_id: sha, skill_hash: sha, receipt_sha256: sha, artifact_set_sha256: sha, + files: z.array(z.object({ path: z.string().min(1).max(1024), bytes: z.number().int().min(0).max(16 * 1024 * 1024), sha256: sha }).strict()).max(10_000), +}).strict(); + +export type ArtifactsOptions = { contract: string; delivery?: string; path?: string }; +export type ArtifactReport = { + schema: "skillsync.artifacts-report/v1"; status: "passed" | "failed"; exitCode: number; + evidence: "physical-files"; execution: "not-run"; provenance: "not-authenticated"; + receipt_binding: "verified" | "not-provided" | "failed"; + contract_sha256?: string; artifact_set_sha256?: string; + files: Array<{ path: string; bytes: number; sha256: string; rows?: number }>; + findings: ArtifactFinding[]; +}; + +function setHash(files: ArtifactReport["files"]): string { + const digest = createHash("sha256"); + for (const file of [...files].sort((left, right) => Buffer.compare(Buffer.from(left.path), Buffer.from(right.path)))) { + const size = Buffer.alloc(8); size.writeBigUInt64BE(BigInt(file.bytes)); + digest.update(file.path); digest.update(Buffer.from([0])); digest.update(size); digest.update(file.sha256); + } + return digest.digest("hex"); +} + +function readJson(bytes: Uint8Array): unknown { + return parseStrictJson(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); +} + +export async function runArtifacts(options: ArtifactsOptions): Promise { + const report: ArtifactReport = { schema: "skillsync.artifacts-report/v1", status: "failed", exitCode: 1, + evidence: "physical-files", execution: "not-run", provenance: "not-authenticated", + receipt_binding: options.delivery ? "failed" : "not-provided", files: [], findings: [] }; + const issue = (code: string, path?: string, field?: string) => { + if (report.findings.length < 256) report.findings.push({ code, ...(path ? { path } : {}), ...(field ? { field } : {}) }); + }; + let contract; + try { + if (Boolean(options.delivery) === Boolean(options.path)) throw new Error("select one directory"); + const bytes = await readWorkspaceFile(options.contract, 64 * 1024); + contract = parseArtifactContract(parseYaml(new TextDecoder("utf-8", { fatal: true }).decode(bytes), { maxAliasCount: 100, prettyErrors: false, logLevel: "silent" })); + report.contract_sha256 = hash(JSON.stringify(contract)); + } catch { issue("artifact.contract-invalid"); report.exitCode = 2; return report; } + let root = options.path ?? join(options.delivery!, "artifacts"); + let manifest: z.infer | undefined; + let metadataHashes: { receipt: string; manifest: string } | undefined; + if (options.delivery) { + try { + const metadata = await lstat(options.delivery); + if (!metadata.isDirectory() || metadata.isSymbolicLink()) throw new Error("directory invalid"); + const entries = await readdir(options.delivery); + if (entries.length !== 3 || !["artifacts", "receipt.json", "delivery.json"].every(name => entries.includes(name))) { + issue("delivery.layout"); return report; + } + const manifestBytes = await readWorkspaceFile(join(options.delivery, "delivery.json"), 1024 * 1024); + const result = manifestSchema.safeParse(readJson(manifestBytes)); + if (!result.success) { issue("delivery.manifest-invalid"); return report; } + manifest = result.data; + const expected = new Set(contract.files.map(file => file.path)); + if (manifest.files.length !== expected.size || new Set(manifest.files.map(file => file.path)).size !== manifest.files.length || + manifest.files.some((file, index) => index > 0 && Buffer.compare(Buffer.from(manifest!.files[index - 1].path), Buffer.from(file.path)) >= 0) || + manifest.files.some(file => !expected.has(file.path))) { issue("delivery.file-set"); return report; } + const receiptBytes = await readWorkspaceFile(join(options.delivery, "receipt.json"), 1024 * 1024); + const receipt = readJson(receiptBytes) as Record; + if (!receipt || receipt.schema !== "skilltape.dev/receipt/v1" || receipt.status !== "succeeded" || + receipt.run_id !== manifest.run_id || receipt.skill_hash !== manifest.skill_hash || hash(receiptBytes) !== manifest.receipt_sha256) { + issue("delivery.receipt-binding"); return report; + } + report.receipt_binding = "verified"; + metadataHashes = { receipt: hash(receiptBytes), manifest: hash(manifestBytes) }; + } catch { issue("delivery.metadata-invalid"); return report; } + root = join(options.delivery, "artifacts"); + } + const facts = new Map(); + try { + const rules = new Map(contract.files.map(file => [file.path, file])); + const tree = await scanStagedWorkspace(root, { + limits: { maxFiles: contract.limits.max_files, maxFileBytes: contract.limits.max_file_bytes, maxTotalBytes: contract.limits.max_total_bytes }, + allowedFiles: new Set(rules.keys()), + inspectFile(path, content, observation) { + const rule = rules.get(path)!; + const sha256 = observation.digest.slice("sha256:".length); + if (rule.sha256 && rule.sha256 !== sha256) issue("artifact.digest-mismatch", path); + const inspected = inspectArtifactContent(rule, content); + facts.set(path, inspected.facts); + for (const finding of inspected.findings) if (report.findings.length < 256) report.findings.push(finding); + report.files.push({ path, bytes: observation.bytes, sha256, ...(inspected.facts.rows !== undefined ? { rows: inspected.facts.rows } : {}) }); + }, + }); + const found = new Set(tree.files.map(file => file.path.slice("workspace/".length))); + for (const file of contract.files) if (!found.has(file.path)) issue("artifact.missing", file.path); + } catch (error) { + report.files = []; + report.findings = []; + issue(error instanceof WorkspaceTreeError && error.code === "workspace.tree-too-large" ? "artifact.capacity" : "artifact.structure-or-snapshot"); + if (error instanceof WorkspaceTreeError) report.findings.at(-1)!.message = error.message; + return report; + } + report.files.sort((left, right) => Buffer.compare(Buffer.from(left.path), Buffer.from(right.path))); + report.artifact_set_sha256 = setHash(report.files); + if (manifest) { + if (manifest.artifact_set_sha256 !== report.artifact_set_sha256) issue("delivery.artifact-set-digest"); + for (const file of report.files) { + const declared = manifest.files.find(entry => entry.path === file.path)!; + if (declared.bytes !== file.bytes || declared.sha256 !== file.sha256) issue("delivery.artifact-digest", file.path); + } + try { + const entries = await readdir(options.delivery!); + if (entries.length !== 3 || !["artifacts", "receipt.json", "delivery.json"].every(name => entries.includes(name)) || + hash(await readWorkspaceFile(join(options.delivery!, "receipt.json"), 1024 * 1024)) !== metadataHashes!.receipt || + hash(await readWorkspaceFile(join(options.delivery!, "delivery.json"), 1024 * 1024)) !== metadataHashes!.manifest) issue("delivery.changed-during-check"); + } catch { issue("delivery.changed-during-check"); } + } + for (const check of contract.checks) { + const csv = facts.get(check.csv), json = facts.get(check.json); + if (!csv || !json) { issue("artifact.comparison-missing", check.csv); continue; } + if (csv.rows !== json.integers.get(check.row_count_field)) issue("artifact.summary-row-count", check.json, check.row_count_field); + for (const sum of check.sums) if (csv.integers.get(sum.column) !== json.integers.get(sum.field)) issue("artifact.summary-sum", check.json, sum.field); + } + if (!report.findings.length) { report.status = "passed"; report.exitCode = 0; } + return report; +} + +export function renderArtifacts(report: ArtifactReport, format = "text"): string { + const safe = redactLocalPaths(report); + if (format === "json") return JSON.stringify(safe, null, 2) + "\n"; + return [`Artifact validation: ${safe.status}`, `Evidence: physical files; execution: not-run; receipt binding: ${safe.receipt_binding}`, + `Files: ${safe.files.length}`, ...safe.findings.map(finding => `${finding.code}${finding.path ? ` file=${JSON.stringify(finding.path)}` : ""}${finding.field ? ` field=${JSON.stringify(finding.field)}` : ""}${finding.row !== undefined ? ` row=${finding.row}` : ""}${finding.message ? ` ${finding.message}` : ""}`)].join("\n") + "\n"; +} diff --git a/src/cli/index.ts b/src/cli/index.ts index 591a29b..a3e5e22 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -38,6 +38,7 @@ import { type FixPlanOptions, } from "./commands/fix.js"; import { ActionPlanApplyError } from "../domain/patch-application.js"; +import { runArtifacts, renderArtifacts, type ArtifactsOptions } from "./commands/artifacts.js"; import { listBehaviorFixtures, renderBehaviorTest, @@ -563,6 +564,19 @@ export function createCli(io: CliIO = defaultCliIO): Command { } }); + program.command("artifacts") + .description("Inspect actual local delivery artifacts against a bounded contract.") + .requiredOption("--contract ", "JSON or YAML artifact contract") + .option("--delivery ", "SkillTape delivery bundle directory") + .option("--path ", "Standalone artifact directory; choose this or --delivery") + .option("--format ", "Output format: text or json", "text") + .action(async (options: ArtifactsOptions & { format?: string }) => { + if (options.format !== "text" && options.format !== "json") throw new Error("artifacts format must be text or json"); + const report = await runArtifacts(options); + io.writeOut(renderArtifacts(report, options.format)); + if (report.exitCode) io.setExitCode?.(report.exitCode); + }); + const runner = program .command("runner") .description("Validate a local Runner image contract."); diff --git a/src/domain/artifact-content.ts b/src/domain/artifact-content.ts new file mode 100644 index 0000000..dc09631 --- /dev/null +++ b/src/domain/artifact-content.ts @@ -0,0 +1,129 @@ +import type { ArtifactFileRule, ArtifactScalarRule } from "./artifact-contract.js"; +import { parseStrictJson } from "./strict-json.js"; + +export type ArtifactFinding = { code: string; path?: string; field?: string; row?: number; message?: string }; +export type ArtifactFacts = { rows?: number; integers: Map }; + +function* csvRows(text: string, maxFields: number): Generator { + let field = "", row: string[] = [], quoted = false, afterQuote = false, active = false; + for (let index = 0; index < text.length; index += 1) { + const char = text[index]; + active = true; + if (quoted) { + if (char === '"') { + if (text[index + 1] === '"') { field += '"'; index += 1; } + else { quoted = false; afterQuote = true; } + } else field += char; + continue; + } + if (afterQuote && char !== "," && char !== "\r" && char !== "\n") throw new Error("invalid csv"); + if (char === '"') { + if (field || afterQuote) throw new Error("invalid csv"); + quoted = true; + } else if (char === ",") { + row.push(field); + if (row.length >= maxFields) throw new Error("csv-column-count"); + field = ""; afterQuote = false; + } + else if (char === "\r" || char === "\n") { + if (char === "\r") { + if (text[index + 1] !== "\n") throw new Error("invalid csv"); + index += 1; + } + row.push(field); yield row; row = []; field = ""; afterQuote = false; active = false; + } else field += char; + } + if (quoted) throw new Error("invalid csv"); + if (active || row.length || field || afterQuote) { row.push(field); yield row; } +} + +function validDate(value: string): boolean { + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return false; + const time = Date.parse(value); + return Number.isFinite(time) && new Date(time).toISOString().slice(0, 10) === value; +} + +function scalarValid(value: unknown, rule: ArtifactScalarRule): boolean { + if (rule.type === "integer" && (typeof value !== "number" || !Number.isSafeInteger(value))) return false; + if (rule.type === "number" && (typeof value !== "number" || !Number.isFinite(value))) return false; + if (rule.type === "string" && (typeof value !== "string" || (!rule.allow_empty && !value.length))) return false; + if (rule.type === "date" && (typeof value !== "string" || !validDate(value))) return false; + if (rule.type === "boolean" && typeof value !== "boolean") return false; + if (rule.type === "array" && !Array.isArray(value)) return false; + if (rule.type === "object" && (!value || typeof value !== "object" || Array.isArray(value))) return false; + if (rule.equals !== undefined && value !== rule.equals) return false; + if (typeof value === "number" && ((rule.min !== undefined && value < rule.min) || (rule.max !== undefined && value > rule.max))) return false; + return true; +} + +function csvValue(value: string, rule: ArtifactScalarRule): unknown { + if (rule.type === "integer" || rule.type === "number") { + const pattern = rule.type === "integer" ? /^-?\d+$/ : /^-?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$/; + return pattern.test(value) ? Number(value) : undefined; + } + if (rule.type === "boolean") return value === "true" ? true : value === "false" ? false : undefined; + return value; +} + +function formula(value: string): boolean { + return /^[\t\r\n]/.test(value) || /^[=+@\-\uFF1D\uFF0B\uFF0D\uFF20]/.test(value.trimStart()); +} + +export function inspectArtifactContent(rule: ArtifactFileRule, bytes: Uint8Array): { facts: ArtifactFacts; findings: ArtifactFinding[] } { + const facts: ArtifactFacts = { integers: new Map() }, findings: ArtifactFinding[] = []; + const issue = (code: string, field?: string, row?: number) => { + if (findings.length < 64) findings.push({ code, path: rule.path, ...(field ? { field } : {}), ...(row !== undefined ? { row } : {}) }); + }; + if (rule.format === "bytes") return { facts, findings }; + let text: string; + try { text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); } + catch { issue("artifact.invalid-utf8"); return { facts, findings }; } + if (rule.format === "json") { + let value: unknown; + try { value = parseStrictJson(text); } catch { issue("artifact.json-invalid"); return { facts, findings }; } + if (!value || typeof value !== "object" || Array.isArray(value)) { issue("artifact.json-root"); return { facts, findings }; } + const object = value as Record; + if (!rule.allow_extra_fields && Object.keys(object).some(key => !rule.fields.some(field => field.name === key))) issue("artifact.json-extra-field"); + for (const field of rule.fields) { + if (!Object.hasOwn(object, field.name)) { if (field.required) issue("artifact.json-missing-field", field.name); continue; } + if (!scalarValid(object[field.name], field)) issue("artifact.json-value", field.name); + else if (field.type === "integer") facts.integers.set(field.name, object[field.name] as number); + } + return { facts, findings }; + } + let count = 0; + for (const column of rule.columns) if (column.type === "integer") facts.integers.set(column.name, 0); + const keys = new Set(); + const uniqueIndexes = rule.unique_by.map(name => rule.columns.findIndex(column => column.name === name)); + try { + const records = csvRows(text, rule.columns.length); + const header = records.next().value as string[] | undefined; + if (!header || header.length !== rule.columns.length || header.some((name, index) => name !== rule.columns[index].name)) { + issue("artifact.csv-header"); return { facts, findings }; + } + if (rule.reject_formulas && header.some(formula)) issue("artifact.csv-formula", undefined, 0); + for (const row of records) { + count += 1; + if (count > rule.max_rows) { issue("artifact.csv-row-limit"); break; } + if (row.length !== rule.columns.length) { issue("artifact.csv-column-count", undefined, count); continue; } + if (uniqueIndexes.length) { + const key = JSON.stringify(uniqueIndexes.map(index => row[index])); + if (keys.has(key)) issue("artifact.csv-duplicate", undefined, count); + keys.add(key); + } + for (const [index, column] of rule.columns.entries()) { + const value = csvValue(row[index], column); + if (rule.reject_formulas && ["string", "date"].includes(column.type) && formula(row[index])) issue("artifact.csv-formula", column.name, count); + if (!scalarValid(value, column)) { issue("artifact.csv-value", column.name, count); continue; } + if (column.type === "integer") { + const total = (facts.integers.get(column.name) ?? 0) + (value as number); + if (!Number.isSafeInteger(total)) issue("artifact.csv-unsafe-sum", column.name, count); + else facts.integers.set(column.name, total); + } + } + } + facts.rows = count; + if (count < rule.min_rows) issue("artifact.csv-too-few-rows"); + } catch (error) { issue(error instanceof Error && error.message === "csv-column-count" ? "artifact.csv-column-count" : "artifact.csv-invalid"); } + return { facts, findings }; +} diff --git a/src/domain/artifact-contract.ts b/src/domain/artifact-contract.ts new file mode 100644 index 0000000..1ffe996 --- /dev/null +++ b/src/domain/artifact-contract.ts @@ -0,0 +1,81 @@ +import { z } from "zod"; + +import { WORKSPACE_TREE_LIMITS } from "../sandbox/workspace-tree.js"; + +const safePath = z.string().min(1).max(1024).refine(path => + !path.startsWith("/") && !path.includes("\\") && !/\p{Cc}/u.test(path) && + !/^[A-Za-z]:/.test(path) && path.split("/").every(part => part && part !== "." && part !== ".."), +"artifact paths must be safe and relative"); +const hash = z.string().regex(/^[0-9a-f]{64}$/); +const scalar = z.object({ + name: z.string().min(1).max(128), + type: z.enum(["string", "integer", "number", "boolean", "date", "array", "object"]), + required: z.boolean().default(true), + equals: z.union([z.string(), z.number().finite(), z.boolean()]).optional(), + min: z.number().finite().optional(), max: z.number().finite().optional(), + allow_empty: z.boolean().default(false), +}).strict().refine(rule => rule.min === undefined || rule.max === undefined || rule.min <= rule.max, +"minimum must not exceed maximum"); +const base = { path: safePath, sha256: hash.optional() }; +const csv = z.object({ ...base, format: z.literal("csv"), columns: z.array(scalar).min(1).max(128), + min_rows: z.number().int().min(0).default(1), max_rows: z.number().int().min(1).max(100_000), + unique_by: z.array(z.string().min(1)).max(128).default([]), + reject_formulas: z.boolean().default(true), +}).strict(); +const json = z.object({ ...base, format: z.literal("json"), fields: z.array(scalar).max(128).default([]), + allow_extra_fields: z.boolean().default(false), +}).strict(); +const bytes = z.object({ ...base, format: z.literal("bytes") }).strict(); +const comparison = z.object({ type: z.literal("csv_summary"), csv: safePath, json: safePath, + row_count_field: z.string().min(1).max(128), + sums: z.array(z.object({ column: z.string().min(1).max(128), field: z.string().min(1).max(128) }).strict()).max(128), +}).strict(); + +export const artifactContractSchema = z.object({ schema: z.literal("skillsync.artifacts/v1"), + limits: z.object({ + max_files: z.number().int().positive().max(WORKSPACE_TREE_LIMITS.maxFiles), + max_file_bytes: z.number().int().positive().max(WORKSPACE_TREE_LIMITS.maxFileBytes), + max_total_bytes: z.number().int().positive().max(WORKSPACE_TREE_LIMITS.maxTotalBytes), + }).strict(), + files: z.array(z.discriminatedUnion("format", [bytes, csv, json])).max(WORKSPACE_TREE_LIMITS.maxFiles), + checks: z.array(comparison).max(128).default([]), +}).strict().superRefine((contract, context) => { + const fail = () => context.addIssue({ code: z.ZodIssueCode.custom, message: "inconsistent artifact declarations" }); + const files = new Map(contract.files.map(file => [file.path, file])); + if (files.size !== contract.files.length || contract.files.length > contract.limits.max_files) fail(); + for (const path of files.keys()) { + const parts = path.split("/"); + for (let index = 1; index < parts.length; index += 1) if (files.has(parts.slice(0, index).join("/"))) fail(); + } + for (const file of contract.files) { + const rules = file.format === "csv" ? file.columns : file.format === "json" ? file.fields : []; + if (new Set(rules.map(rule => rule.name)).size !== rules.length) fail(); + for (const rule of rules) { + if ((rule.min !== undefined || rule.max !== undefined) && !["integer", "number"].includes(rule.type)) fail(); + if (rule.equals !== undefined && ((rule.type === "integer" && !Number.isSafeInteger(rule.equals)) || + (rule.type === "number" && typeof rule.equals !== "number") || + (["string", "date"].includes(rule.type) && typeof rule.equals !== "string") || + (rule.type === "boolean" && typeof rule.equals !== "boolean") || ["array", "object"].includes(rule.type))) fail(); + } + if (file.format === "csv" && (file.min_rows > file.max_rows || + file.columns.some(rule => ["array", "object"].includes(rule.type) || !rule.required) || + file.unique_by.some(name => !file.columns.some(column => column.name === name)))) fail(); + } + for (const check of contract.checks) { + const csv = files.get(check.csv), json = files.get(check.json); + if (csv?.format !== "csv" || json?.format !== "json") { fail(); continue; } + const integerField = (name: string) => json.fields.some(field => field.name === name && field.type === "integer" && field.required); + if (!integerField(check.row_count_field) || check.sums.some(sum => + !csv.columns.some(column => column.name === sum.column && column.type === "integer") || !integerField(sum.field))) fail(); + } +}); + +export type ArtifactContract = z.infer; +export type ArtifactFileRule = ArtifactContract["files"][number]; +export type ArtifactScalarRule = z.infer; + +export function parseArtifactContract(value: unknown): ArtifactContract { + const result = artifactContractSchema.safeParse(value); + if (!result.success) throw new Error("artifact contract is invalid"); + return result.data; +} diff --git a/src/domain/strict-json.ts b/src/domain/strict-json.ts new file mode 100644 index 0000000..e014279 --- /dev/null +++ b/src/domain/strict-json.ts @@ -0,0 +1,30 @@ +/** Parse JSON without ambiguous duplicate keys; never include input in errors. */ +export function parseStrictJson(text: string): unknown { + const stack: Array<{ keys?: Set; expectKey: boolean }> = []; + let keysSeen = 0; + for (let index = 0; index < text.length; index += 1) { + const char = text[index]; + if (char === '"') { + const start = index; + index += 1; + while (index < text.length && text[index] !== '"') { + if (text[index] === "\\") index += 1; + index += 1; + } + if (index >= text.length) throw new Error("invalid JSON"); + const frame = stack.at(-1); + if (frame?.keys && frame.expectKey) { + const token = text.slice(start, index + 1); + if (token.length > 6146 || ++keysSeen > 100_000) throw new Error("JSON key budget exceeded"); + const key = JSON.parse(token) as string; + if (key.length > 1024 || frame.keys.has(key)) throw new Error("ambiguous JSON key"); + frame.keys.add(key); frame.expectKey = false; + } + } else if (char === "{" || char === "[") { + if (stack.length >= 64) throw new Error("JSON depth limit exceeded"); + stack.push({ ...(char === "{" ? { keys: new Set() } : {}), expectKey: char === "{" }); + } else if (char === "}" || char === "]") stack.pop(); + else if (char === "," && stack.at(-1)?.keys) stack.at(-1)!.expectKey = true; + } + return JSON.parse(text) as unknown; +} diff --git a/src/sandbox/workspace-tree.ts b/src/sandbox/workspace-tree.ts index 99bff71..a46f9c1 100644 --- a/src/sandbox/workspace-tree.ts +++ b/src/sandbox/workspace-tree.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; -import { lstat, readFile, readdir } from "node:fs/promises"; +import { constants, type Stats } from "node:fs"; +import { lstat, open, opendir } from "node:fs/promises"; import { isAbsolute, join, relative, resolve } from "node:path"; import { normalizeSandboxPath } from "../domain/behavior-v2.js"; @@ -42,51 +43,124 @@ function workspacePath(root: string, filePath: string): string { return `workspace/${normalizeSandboxPath(relativePath, "workspace tree path", "workspace")}`; } -export async function scanStagedWorkspace(rootPath: string): Promise { +export type WorkspaceTreeScanOptions = { + limits?: Partial<{ maxFiles: number; maxFileBytes: number; maxTotalBytes: number }>; + allowedFiles?: ReadonlySet; + inspectFile?: (path: string, content: Buffer, observation: VirtualFileObservation) => void | Promise; +}; + +function unchanged(left: Stats, right: Stats): boolean { + return left.dev === right.dev && left.ino === right.ino && left.size === right.size && + left.mtimeMs === right.mtimeMs && left.ctimeMs === right.ctimeMs; +} + +export async function readWorkspaceFile(path: string, maxBytes: number, expected?: Stats): Promise { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 1 || maxBytes > WORKSPACE_TREE_LIMITS.maxFileBytes) { + throw new WorkspaceTreeLimitError("invalid file read limit"); + } + try { + const metadata = expected ?? await lstat(path); + if (!metadata.isFile() || metadata.isSymbolicLink()) throw new WorkspaceTreeError("only regular files are readable"); + if (!Number.isSafeInteger(metadata.size) || metadata.size > maxBytes) throw new WorkspaceTreeLimitError("file exceeds the read limit"); + const handle = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + try { + const before = await handle.stat(); + if (!before.isFile() || !unchanged(metadata, before)) throw new WorkspaceTreeError("workspace changed during inspection"); + const buffer = Buffer.alloc(metadata.size + 1); + let offset = 0; + while (offset < buffer.length) { + const { bytesRead } = await handle.read(buffer, offset, buffer.length - offset, offset); + if (!bytesRead) break; + offset += bytesRead; + } + if (offset !== metadata.size || !unchanged(before, await handle.stat()) || !unchanged(metadata, await lstat(path))) { + throw new WorkspaceTreeError("workspace changed during inspection"); + } + return buffer.subarray(0, offset); + } finally { await handle.close(); } + } catch (error) { + if (error instanceof WorkspaceTreeError) throw error; + throw new WorkspaceTreeError("workspace file is inaccessible or changed"); + } +} + +export async function scanStagedWorkspace(rootPath: string, options: WorkspaceTreeScanOptions = {}): Promise { const root = resolve(rootPath); + const limits = { ...WORKSPACE_TREE_LIMITS, ...options.limits }; + for (const key of ["maxFiles", "maxFileBytes", "maxTotalBytes"] as const) { + if (!Number.isSafeInteger(limits[key]) || limits[key] < 1 || limits[key] > WORKSPACE_TREE_LIMITS[key]) { + throw new WorkspaceTreeLimitError("requested limits must be positive and within workspace ceilings"); + } + } const rootMetadata = await lstat(root).catch(() => undefined); if (!rootMetadata || !rootMetadata.isDirectory() || rootMetadata.isSymbolicLink()) { throw new WorkspaceTreeError("staged workspace must be a regular directory"); } - - const files: VirtualFileObservation[] = []; + const inventory: Array<{ absolute: string; path: string; metadata: Stats }> = []; + const directories: Array<{ absolute: string; metadata: Stats }> = []; let totalBytes = 0; - - async function visit(current: string): Promise { + let entriesSeen = 0; + const allowedDirectories = new Set(); + for (const path of options.allowedFiles ?? []) { + const parts = path.split("/"); + for (let index = 1; index < parts.length; index += 1) allowedDirectories.add(parts.slice(0, index).join("/")); + } + async function visit(current: string, depth: number): Promise { + if (depth > 64) throw new WorkspaceTreeLimitError("workspace directory depth exceeds the limit"); const metadata = await lstat(current); - if (metadata.isSymbolicLink()) { - throw new WorkspaceTreeError("symlinks are not allowed in the staged workspace"); - } + if (metadata.isSymbolicLink()) throw new WorkspaceTreeError("symlinks are not allowed in the staged workspace"); + const path = current === root ? "" : workspacePath(root, current).slice("workspace/".length); if (metadata.isDirectory()) { - const entries = await readdir(current, { withFileTypes: true }); - for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { - await visit(join(current, entry.name)); + if (path && options.allowedFiles && !allowedDirectories.has(path)) { + throw new WorkspaceTreeError("directory is not declared by the artifact contract"); + } + directories.push({ absolute: current, metadata }); + // Stream directory entries so the inventory cap also bounds enumeration. + for await (const entry of await opendir(current)) { + entriesSeen += 1; + if (entriesSeen > 20_000) throw new WorkspaceTreeLimitError("workspace entry count exceeds the limit"); + if (entry.name.includes("\\") || /\p{Cc}/u.test(entry.name)) { + throw new WorkspaceTreeError("unsafe workspace file name"); + } + await visit(join(current, entry.name), depth + 1); } return; } - if (!metadata.isFile()) { - throw new WorkspaceTreeError("only regular files are allowed in the staged workspace"); + if (!metadata.isFile()) throw new WorkspaceTreeError("only regular files are allowed in the staged workspace"); + if (options.allowedFiles && !options.allowedFiles.has(path)) { + throw new WorkspaceTreeError("file is not declared by the artifact contract"); } - if (metadata.size > WORKSPACE_TREE_LIMITS.maxFileBytes) { + if (!Number.isSafeInteger(metadata.size) || metadata.size > limits.maxFileBytes) { throw new WorkspaceTreeLimitError("a staged workspace file exceeds the size limit"); } - if (files.length >= WORKSPACE_TREE_LIMITS.maxFiles || totalBytes + metadata.size > WORKSPACE_TREE_LIMITS.maxTotalBytes) { + if (inventory.length >= limits.maxFiles || totalBytes + metadata.size > limits.maxTotalBytes) { throw new WorkspaceTreeLimitError("staged workspace exceeds the tree size limit"); } - - const content = await readFile(current); - const digest = createHash("sha256").update(content).digest("hex"); - files.push({ - path: workspacePath(root, current), - bytes: content.byteLength, - digest: `sha256:${digest}`, - }); - totalBytes += content.byteLength; + totalBytes += metadata.size; + inventory.push({ absolute: current, path, metadata }); + } + // Validate the complete metadata inventory before reading any payload, so an + // undeclared private file or oversized tree never reaches the content hook. + try { + await visit(root, 0); + const files: VirtualFileObservation[] = []; + for (const entry of inventory) { + const content = await readWorkspaceFile(entry.absolute, limits.maxFileBytes, entry.metadata); + const observation = { path: `workspace/${entry.path}`, bytes: content.byteLength, + digest: `sha256:${createHash("sha256").update(content).digest("hex")}` }; + files.push(observation); + await options.inspectFile?.(entry.path, content, observation); + } + for (const entry of [...inventory, ...directories]) { + const after = await lstat(entry.absolute); + if (after.isSymbolicLink() || !unchanged(entry.metadata, after)) throw new WorkspaceTreeError("workspace changed during inspection"); + } + files.sort((left, right) => left.path.localeCompare(right.path)); + return { files, digest: treeDigest(files) }; + } catch (error) { + if (error instanceof WorkspaceTreeError) throw error; + throw new WorkspaceTreeError("workspace file is inaccessible or changed"); } - - await visit(root); - files.sort((left, right) => left.path.localeCompare(right.path)); - return { files, digest: treeDigest(files) }; } export function workspaceTreeDelta( diff --git a/tests/domain/artifact-content.test.ts b/tests/domain/artifact-content.test.ts new file mode 100644 index 0000000..9fe070f --- /dev/null +++ b/tests/domain/artifact-content.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; +import { parseArtifactContract } from "../../src/domain/artifact-contract"; +import { inspectArtifactContent } from "../../src/domain/artifact-content"; + +function rule(options: Record = {}) { + return parseArtifactContract({ schema: "skillsync.artifacts/v1", + limits: { max_files: 8, max_file_bytes: 10000, max_total_bytes: 20000 }, + files: [{ path: "data.csv", format: "csv", columns: [{ name: "label", type: "string" }, { name: "count", type: "integer" }], + unique_by: ["label"], max_rows: 10, ...options }] }).files[0]; +} + +describe("actual artifact formats", () => { + it("rejects duplicate JSON keys rather than choosing a conflicting last value", () => { + const json = parseArtifactContract({ schema: "skillsync.artifacts/v1", + limits: { max_files: 8, max_file_bytes: 10000, max_total_bytes: 20000 }, + files: [{ path: "data.json", format: "json", fields: [{ name: "version", type: "integer", equals: 1 }] }] }).files[0]; + expect(inspectArtifactContent(json, Buffer.from('{"version":2,"version":1}')).findings[0]?.code).toBe("artifact.json-invalid"); + expect(inspectArtifactContent(json, Buffer.from('{"version":2,"\\u0076ersion":1}')).findings[0]?.code).toBe("artifact.json-invalid"); + }); + it("treats an explicitly allowed empty CSV as zero integer totals", () => { + const result = inspectArtifactContent(rule({ min_rows: 0 }), Buffer.from("label,count\n")); + expect(result.findings).toEqual([]); + expect(result.facts.rows).toBe(0); + expect(result.facts.integers.get("count")).toBe(0); + }); + it("parses quotes, comma fields, multiline fields and CRLF without retaining records", () => { + const result = inspectArtifactContent(rule(), Buffer.from('label,count\r\n"first, second",1\r\n"multi\nline ""quote""",2\r\n')); + expect(result.findings).toEqual([]); + expect(result.facts.rows).toBe(2); + expect(result.facts.integers.get("count")).toBe(3); + }); + it("accepts safe negative integers without treating them as text formulas", () => { + expect(inspectArtifactContent(rule(), Buffer.from("label,count\na,-2\n")).findings).toEqual([]); + }); + it.each([ + ["wrong,count\na,1\n", "artifact.csv-header"], + ["label,count\na,1,extra\n", "artifact.csv-column-count"], + ['label,count\n"unfinished,1\n', "artifact.csv-invalid"], + ['label,count\na"b,1\n', "artifact.csv-invalid"], + ["label,count\na,1\na,2\n", "artifact.csv-duplicate"], + ["label,count\n=2+2,1\n", "artifact.csv-formula"], + ['label,count\n" @example",1\n', "artifact.csv-formula"], + ["label,count\n\uff1d2+2,1\n", "artifact.csv-formula"], + ["label,count\na,9007199254740992\n", "artifact.csv-value"], + ["label,count\na,9007199254740991\nb,1\n", "artifact.csv-unsafe-sum"], + ["label,count\na,NaN\n", "artifact.csv-value"], + ])("rejects invalid CSV with stable codes: %s", (input, code) => { + expect(inspectArtifactContent(rule(), Buffer.from(input)).findings.some(finding => finding.code === code)).toBe(true); + }); + it("enforces record limits and UTF-8", () => { + expect(inspectArtifactContent(rule({ max_rows: 1 }), Buffer.from("label,count\na,1\nb,2\n")).findings[0].code).toBe("artifact.csv-row-limit"); + expect(inspectArtifactContent(rule(), Buffer.from([0xff])).findings[0].code).toBe("artifact.invalid-utf8"); + }); + it("checks actual JSON types and values without exposing private input values", () => { + const json = parseArtifactContract({ schema: "skillsync.artifacts/v1", + limits: { max_files: 8, max_file_bytes: 10000, max_total_bytes: 20000 }, + files: [{ path: "data.json", format: "json", fields: [ + { name: "version", type: "integer", equals: 1 }, { name: "count", type: "integer" }] }] }).files[0]; + expect(inspectArtifactContent(json, Buffer.from('{"version":1,"count":2}')).findings).toEqual([]); + const bad = inspectArtifactContent(json, Buffer.from('{"version":2,"count":"NEVER_DISCLOSE_VALUE"}')); + expect(bad.findings.some(finding => finding.code === "artifact.json-value")).toBe(true); + expect(JSON.stringify(bad.findings)).not.toContain("NEVER_DISCLOSE_VALUE"); + }); + it("requires own JSON properties and checks real calendar dates", () => { + const json = parseArtifactContract({ schema: "skillsync.artifacts/v1", + limits: { max_files: 8, max_file_bytes: 10000, max_total_bytes: 20000 }, + files: [{ path: "data.json", format: "json", fields: [{ name: "constructor", type: "object" }] }] }).files[0]; + expect(inspectArtifactContent(json, Buffer.from("{}")).findings[0].code).toBe("artifact.json-missing-field"); + const date = rule({ columns: [{ name: "label", type: "date" }, { name: "count", type: "integer" }] }); + expect(inspectArtifactContent(date, Buffer.from("label,count\n2026-02-29,1\n")).findings[0].code).toBe("artifact.csv-value"); + expect(inspectArtifactContent(date, Buffer.from("label,count\n2024-02-29,1\n")).findings).toEqual([]); + }); + it("rejects unsupported or unsafe contract declarations", () => { + expect(() => rule({ path: "../outside.csv" })).toThrow("artifact contract is invalid"); + expect(() => rule({ columns: [{ name: "x", type: "string", min: 1 }] })).toThrow("artifact contract is invalid"); + expect(() => rule({ columns: [{ name: "x", type: "string" }, { name: "x", type: "string" }] })).toThrow("artifact contract is invalid"); + }); +}); diff --git a/tests/integration/artifacts.test.ts b/tests/integration/artifacts.test.ts new file mode 100644 index 0000000..1dc390c --- /dev/null +++ b/tests/integration/artifacts.test.ts @@ -0,0 +1,106 @@ +import { createHash } from "node:crypto"; +import { cp, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { describe, expect, it } from "vitest"; +import { runArtifacts, renderArtifacts } from "../../src/cli/commands/artifacts"; +import { createCli } from "../../src/cli/index"; +import { scanStagedWorkspace } from "../../src/sandbox/workspace-tree"; + +const source = resolve("fixtures/product/order-summary"); +const sha = (bytes: Uint8Array | string) => createHash("sha256").update(bytes).digest("hex"); + +async function fixture(run: (root: string, contract: string, data: string) => Promise) { + const root = await mkdtemp(join(tmpdir(), "skillsync-artifacts-")); + try { + await cp(join(source, "artifacts"), join(root, "data"), { recursive: true }); + await cp(join(source, "contract.json"), join(root, "contract.json")); + await run(root, join(root, "contract.json"), join(root, "data")); + } finally { await rm(root, { recursive: true, force: true }); } +} + +async function bind(bundle: string) { + const receipt = { schema: "skilltape.dev/receipt/v1", status: "succeeded", run_id: "a".repeat(64), skill_hash: "b".repeat(64), steps: [] }; + const receiptBytes = Buffer.from(JSON.stringify(receipt)); + await writeFile(join(bundle, "receipt.json"), receiptBytes); + const files = await Promise.all(["metrics.csv", "summary.json"].map(async path => { + const bytes = await readFile(join(bundle, "artifacts", path)); + return { path, bytes: bytes.length, sha256: sha(bytes) }; + })); + const digest = createHash("sha256"); + for (const file of files) { const size = Buffer.alloc(8); size.writeBigUInt64BE(BigInt(file.bytes)); + digest.update(file.path); digest.update(Buffer.from([0])); digest.update(size); digest.update(file.sha256); } + await writeFile(join(bundle, "delivery.json"), JSON.stringify({ schema: "skilltape.dev/delivery/v1", + run_id: receipt.run_id, skill_hash: receipt.skill_hash, receipt_sha256: sha(receiptBytes), artifact_set_sha256: digest.digest("hex"), files })); +} + +describe("physical artifact validation", () => { + it("rejects an ambiguous receipt and an unsorted manifest", async () => fixture(async (root, contract, data) => { + const bundle = join(root, "bundle"); await mkdir(bundle); await cp(data, join(bundle, "artifacts"), { recursive: true }); await bind(bundle); + const manifest = JSON.parse(await readFile(join(bundle, "delivery.json"), "utf8")); + manifest.files.reverse(); await writeFile(join(bundle, "delivery.json"), JSON.stringify(manifest)); + expect((await runArtifacts({ contract, delivery: bundle })).status).toBe("failed"); + await bind(bundle); + const bytes = (await readFile(join(bundle, "receipt.json"), "utf8")).replace('"status":"succeeded"', '"status":"failed","status":"succeeded"'); + await writeFile(join(bundle, "receipt.json"), bytes); + const changed = JSON.parse(await readFile(join(bundle, "delivery.json"), "utf8")); changed.receipt_sha256 = sha(bytes); + await writeFile(join(bundle, "delivery.json"), JSON.stringify(changed)); + expect((await runArtifacts({ contract, delivery: bundle })).receipt_binding).toBe("failed"); + })); + it("checks actual files and independently compares CSV and JSON totals", async () => fixture(async (_root, contract, data) => { + const report = await runArtifacts({ contract, path: data }); + expect(report.status).toBe("passed"); expect(report.evidence).toBe("physical-files"); + expect(report.execution).toBe("not-run"); expect(report.receipt_binding).toBe("not-provided"); + expect(report.files.find(file => file.path === "metrics.csv")?.rows).toBe(2); + expect(report.files).toHaveLength(2); + })); + it("rejects an internally inconsistent summary even with updated bundle hashes", async () => fixture(async (root, contract, data) => { + const bundle = join(root, "bundle"); await mkdir(bundle); await cp(data, join(bundle, "artifacts"), { recursive: true }); + await writeFile(join(bundle, "artifacts", "summary.json"), JSON.stringify({ schema_version: 1, row_count: 2, total_orders: 3, total_cents: 9999 })); + await bind(bundle); + const report = await runArtifacts({ contract, delivery: bundle }); + expect(report.receipt_binding).toBe("verified"); expect(report.status).toBe("failed"); + expect(report.findings).toContainEqual({ code: "artifact.summary-sum", path: "summary.json", field: "total_cents" }); + })); + it("rejects actual corruption against the retained manifest", async () => fixture(async (root, contract, data) => { + const bundle = join(root, "bundle"); await mkdir(bundle); await cp(data, join(bundle, "artifacts"), { recursive: true }); await bind(bundle); + expect((await runArtifacts({ contract, delivery: bundle })).status).toBe("passed"); + await writeFile(join(bundle, "artifacts", "metrics.csv"), "day,orders,total_cents\n2026-01-01,1,1\n"); + const report = await runArtifacts({ contract, delivery: bundle }); + expect(report.status).toBe("failed"); expect(report.findings.some(finding => finding.code === "delivery.artifact-digest")).toBe(true); + })); + it("rejects undeclared private files before any payload callback", async () => fixture(async (_root, _contract, data) => { + await writeFile(join(data, "private.txt"), "NEVER_READ_PRIVATE_FILE"); + let reads = 0; + await expect(scanStagedWorkspace(data, { allowedFiles: new Set(["metrics.csv", "summary.json"]), inspectFile: () => { reads += 1; } })).rejects.toThrow("not declared"); + expect(reads).toBe(0); + })); + it("rejects symlinks, missing files and capacity overruns", async () => fixture(async (root, contract, data) => { + await rm(join(data, "metrics.csv")); + expect((await runArtifacts({ contract, path: data })).findings).toContainEqual({ code: "artifact.missing", path: "metrics.csv" }); + await symlink(join(source, "artifacts", "metrics.csv"), join(data, "metrics.csv")); + expect((await runArtifacts({ contract, path: data })).status).toBe("failed"); + await rm(join(data, "metrics.csv")); await cp(join(source, "artifacts", "metrics.csv"), join(data, "metrics.csv")); + const value = JSON.parse(await readFile(contract, "utf8")); value.limits.max_file_bytes = 1; + await writeFile(contract, JSON.stringify(value)); + expect((await runArtifacts({ contract, path: data })).findings[0].code).toBe("artifact.capacity"); + const alias = join(root, "alias"); await symlink(data, alias); + expect((await runArtifacts({ contract, path: alias })).status).toBe("failed"); + })); + it("rejects a file modified during inspection", async () => fixture(async (_root, _contract, data) => { + await expect(scanStagedWorkspace(data, { inspectFile: async path => { + if (path === "metrics.csv") await writeFile(join(data, path), "changed during inspection"); + } })).rejects.toThrow("changed during inspection"); + })); + it("keeps local paths and private values out of CLI reports", async () => fixture(async (_root, contract, data) => { + await writeFile(join(data, "summary.json"), '{"schema_version":1,"row_count":2,"total_orders":3,"total_cents":"NEVER_EXPOSE_PRIVATE_VALUE"}'); + const report = await runArtifacts({ contract, path: data }); + expect(report.status).toBe("failed"); + for (const format of ["json", "text"]) { const output = renderArtifacts(report, format); + expect(output).not.toContain(data); expect(output).not.toContain("NEVER_EXPOSE_PRIVATE_VALUE"); } + let output = "", exitCode = 0; + const program = createCli({ writeOut: text => { output += text; }, writeErr: () => undefined, setExitCode: code => { exitCode = code; } }); + await program.parseAsync(["node", "skillsync", "artifacts", "--contract", contract, "--path", data, "--format", "json"]); + expect(exitCode).toBe(1); expect(JSON.parse(output).status).toBe("failed"); + })); +});