Validate bounded cross-file artifact references

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+- Source artifact contracts support bounded CSV `reference_exists` checks with
+  exact string IDs and declared unique target keys. An offline citation example
+  rejects dangling references without exposing values; the aggregate index is
+  capped at 100,000 referenced cells, including repeats.
 - Source string artifact fields support bounded exact `one_of` options for
   classification labels, with a physical CSV/JSON material-intake example.
   Unknown labels fail without value disclosure; classifications do not prove approval.

README.md

@@ -39,6 +39,11 @@ The command above verifies the included sample Skill. Replace the fixture path w
   See [actual artifact checks](docs/artifact-delivery.md); it runs no Skill code.
   The [knowledge source inventory](fixtures/product/source-index/README.md) adds
   exact declared HTTPS hosts to physical CSV/JSON checks without fetching sources.
+  CSV `reference_exists` checks also bind exact string IDs to a declared unique
+  CSV target, with a 100,000-cell reference limit and no raw IDs in findings.
+  The [reference-integrity example](fixtures/product/reference-integrity/README.md)
+  demonstrates missing citations using synthetic files; it does not fetch or
+  authenticate sources.
   The [material intake catalog](fixtures/product/review-intake/README.md) rejects
   unknown classification labels using bounded `one_of` strings; labels do not prove approval.
 - `verify` reviews one local Skill for provenance, target compatibility, and changes without running its scripts.

docs/artifact-delivery.md

@@ -67,6 +67,22 @@ the actual string values.
 checks cover declared top-level fields, not full JSON Schema or arbitrary
 expressions. Contract files are limited to 64 KiB. Limits must be explicit and
 within the shared 10,000-file/16 MiB-file/64 MiB-total workspace ceilings.
+
+`reference_exists` checks an exact nonempty string ID from one CSV against a
+declared single-column `unique_by` key in another CSV. Both columns must be
+declared required strings with `allow_empty: false`. File traversal order does
+not affect references. IDs are not trimmed, normalized or folded; numeric IDs,
+JSON references and composite keys are unsupported. Existing scalar, formula,
+duplicate-key and physical inventory checks continue to fail independently.
+The [synthetic citation catalog](../fixtures/product/reference-integrity/README.md)
+includes a runnable contract and explains adaptations for review, knowledge and
+operations data. Only declared reference fields are retained as SHA-256
+fingerprints with source row numbers. At most 100,000 referenced cells are
+indexed per invocation, counting repeated IDs and deduplicating field declarations.
+Exceeding this ceiling releases indexes and fails with `artifact.reference-capacity`.
+Missing references report relative file, field and data-row number, never the ID
+or fingerprint. This checks internal consistency without authenticating records.
+
 Duplicate JSON keys are rejected, including escaped-key aliases in metadata and
 payloads. JSON depth is capped at 64 and object keys at 100,000; individual keys
 are at most 1,024 characters. CSV rows stop at the declared field count instead

fixtures/product/reference-integrity/README.md

@@ -0,0 +1,28 @@
+# Offline reference integrity
+
+This source-only synthetic example checks that three citation records reference
+two existing, uniquely identified source records. Repeated citations are allowed.
+
+```bash
+npm run build
+node dist/cli/index.js artifacts \
+  --contract fixtures/product/reference-integrity/contract.json \
+  --path fixtures/product/reference-integrity/artifacts --format json
+```
+
+In a disposable copy, replacing `SOURCE2` in `citations.csv` with `UNKNOWN`
+produces `artifact.reference-missing` for data row 2. Reports omit that value.
+Duplicate source IDs fail the existing CSV uniqueness check. File names may be
+ordered either way; all physical checks finish before references are compared.
+
+The same contract can bind review findings to a changed-file catalog, knowledge
+claims to a source catalog, or orders to a customer catalog. IDs are exact
+nonempty strings. The referenced target must declare its single-column unique
+key. Composite keys, JSON references and numeric/coerced IDs are unsupported.
+Only declared reference fields are indexed, using SHA-256 fingerprints; the
+100,000-cell aggregate ceiling counts repeated IDs too. Indexes are released on
+overflow, which fails with `artifact.reference-capacity`.
+
+This verifies local file consistency, not source truth, reviewer identity,
+customer ownership or permission to perform business actions. It does not run a
+Skill, fetch sources or activate a provider. Published npm0.1.0 lacks this command.

fixtures/product/reference-integrity/artifacts/citations.csv

@@ -0,0 +1,4 @@
+claim_id,source_id
+CLAIM1,SOURCE1
+CLAIM2,SOURCE2
+CLAIM3,SOURCE1

fixtures/product/reference-integrity/artifacts/sources.csv

@@ -0,0 +1,3 @@
+source_id,title
+SOURCE1,Fictional engineering note
+SOURCE2,Fictional operations report

fixtures/product/reference-integrity/contract.json

@@ -0,0 +1,13 @@
+{
+  "schema": "skillsync.artifacts/v1",
+  "limits": { "max_files": 2, "max_file_bytes": 4096, "max_total_bytes": 8192 },
+  "files": [
+    { "path": "citations.csv", "format": "csv", "max_rows": 100,
+      "columns": [{ "name": "claim_id", "type": "string" }, { "name": "source_id", "type": "string" }] },
+    { "path": "sources.csv", "format": "csv", "max_rows": 100, "unique_by": ["source_id"],
+      "columns": [{ "name": "source_id", "type": "string" }, { "name": "title", "type": "string" }] }
+  ],
+  "checks": [{ "type": "reference_exists",
+    "source": { "path": "citations.csv", "field": "source_id" },
+    "target": { "path": "sources.csv", "field": "source_id" } }]
+}

src/cli/commands/artifacts.ts

@@ -7,6 +7,7 @@ import { z } from "zod";
 import { parseArtifactContract } from "../../domain/artifact-contract.js";
 import { parseStrictJson } from "../../domain/strict-json.js";
 import { inspectArtifactContent, type ArtifactFacts, type ArtifactFinding } from "../../domain/artifact-content.js";
+import { createArtifactReferenceIndex } from "../../domain/artifact-references.js";
 import { redactLocalPaths } from "../../reporters/local-paths.js";
 import { readWorkspaceFile, scanStagedWorkspace, WorkspaceTreeError } from "../../sandbox/workspace-tree.js";
 
@@ -85,6 +86,7 @@ export async function runArtifacts(options: ArtifactsOptions): Promise<ArtifactR
     root = join(options.delivery, "artifacts");
   }
   const facts = new Map<string, ArtifactFacts>();
+  const references = createArtifactReferenceIndex(contract);
   try {
     const rules = new Map(contract.files.map(file => [file.path, file]));
     const tree = await scanStagedWorkspace(root, {
@@ -94,7 +96,7 @@ export async function runArtifacts(options: ArtifactsOptions): Promise<ArtifactR
         const rule = rules.get(path)!;
         const sha256 = observation.digest.slice("sha256:".length);
         if (rule.sha256 && rule.sha256 !== sha256) issue("artifact.digest-mismatch", path);
-        const inspected = inspectArtifactContent(rule, content);
+        const inspected = inspectArtifactContent(rule, content, references.observer(path));
         facts.set(path, inspected.facts);
         for (const finding of inspected.findings) if (report.findings.length < 256) report.findings.push(finding);
         report.files.push({ path, bytes: observation.bytes, sha256, ...(inspected.facts.rows !== undefined ? { rows: inspected.facts.rows } : {}) });
@@ -125,11 +127,13 @@ export async function runArtifacts(options: ArtifactsOptions): Promise<ArtifactR
     } catch { issue("delivery.changed-during-check"); }
   }
   for (const check of contract.checks) {
+    if (check.type !== "csv_summary") continue;
     const csv = facts.get(check.csv), json = facts.get(check.json);
     if (!csv || !json) { issue("artifact.comparison-missing", check.csv); continue; }
     if (csv.rows !== json.integers.get(check.row_count_field)) issue("artifact.summary-row-count", check.json, check.row_count_field);
     for (const sum of check.sums) if (csv.integers.get(sum.column) !== json.integers.get(sum.field)) issue("artifact.summary-sum", check.json, sum.field);
   }
+  for (const finding of references.findings()) if (report.findings.length < 256) report.findings.push(finding);
   if (!report.findings.length) { report.status = "passed"; report.exitCode = 0; }
   return report;
 }

src/domain/artifact-content.ts

@@ -3,6 +3,7 @@ import { parseStrictJson } from "./strict-json.js";
 
 export type ArtifactFinding = { code: string; path?: string; field?: string; row?: number; message?: string };
 export type ArtifactFacts = { rows?: number; integers: Map<string, number> };
+export type ArtifactCellObserver = (field: string, value: string, row: number) => void;
 
 function* csvRows(text: string, maxFields: number): Generator<string[]> {
   let field = "", row: string[] = [], quoted = false, afterQuote = false, active = false;
@@ -83,7 +84,7 @@ function formula(value: string): boolean {
   return /^[\t\r\n]/.test(value) || /^[=+@\-\uFF1D\uFF0B\uFF0D\uFF20]/.test(value.trimStart());
 }
 
-export function inspectArtifactContent(rule: ArtifactFileRule, bytes: Uint8Array): { facts: ArtifactFacts; findings: ArtifactFinding[] } {
+export function inspectArtifactContent(rule: ArtifactFileRule, bytes: Uint8Array, observe?: ArtifactCellObserver): { facts: ArtifactFacts; findings: ArtifactFinding[] } {
   const facts: ArtifactFacts = { integers: new Map() }, findings: ArtifactFinding[] = [];
   const issue = (code: string, field?: string, row?: number) => {
     if (findings.length < 64) findings.push({ code, path: rule.path, ...(field ? { field } : {}), ...(row !== undefined ? { row } : {}) });
@@ -127,8 +128,10 @@ export function inspectArtifactContent(rule: ArtifactFileRule, bytes: Uint8Array
       }
       for (const [index, column] of rule.columns.entries()) {
         const value = csvValue(row[index], column);
-        if (rule.reject_formulas && ["string", "date"].includes(column.type) && formula(row[index])) issue("artifact.csv-formula", column.name, count);
+        const riskyFormula = rule.reject_formulas && ["string", "date"].includes(column.type) && formula(row[index]);
+        if (riskyFormula) issue("artifact.csv-formula", column.name, count);
         if (!scalarValid(value, column)) { issue("artifact.csv-value", column.name, count); continue; }
+        if (!riskyFormula && column.type === "string") observe?.(column.name, value as string, count);
         if (column.type === "integer") {
           const total = (facts.integers.get(column.name) ?? 0) + (value as number);
           if (!Number.isSafeInteger(total)) issue("artifact.csv-unsafe-sum", column.name, count);

src/domain/artifact-contract.ts

@@ -37,6 +37,8 @@ const comparison = z.object({ type: z.literal("csv_summary"), csv: safePath, jso
   row_count_field: z.string().min(1).max(128),
   sums: z.array(z.object({ column: z.string().min(1).max(128), field: z.string().min(1).max(128) }).strict()).max(128),
 }).strict();
+const referenceField = z.object({ path: safePath, field: z.string().min(1).max(128) }).strict();
+const reference = z.object({ type: z.literal("reference_exists"), source: referenceField, target: referenceField }).strict();
 
 export const artifactContractSchema = z.object({ schema: z.literal("skillsync.artifacts/v1"),
   limits: z.object({
@@ -45,7 +47,7 @@ export const artifactContractSchema = z.object({ schema: z.literal("skillsync.ar
     max_total_bytes: z.number().int().positive().max(WORKSPACE_TREE_LIMITS.maxTotalBytes),
   }).strict(),
   files: z.array(z.discriminatedUnion("format", [bytes, csv, json])).max(WORKSPACE_TREE_LIMITS.maxFiles),
-  checks: z.array(comparison).max(128).default([]),
+  checks: z.array(z.discriminatedUnion("type", [comparison, reference])).max(128).default([]),
 }).strict().superRefine((contract, context) => {
   const fail = () => context.addIssue({ code: z.ZodIssueCode.custom, message: "inconsistent artifact declarations" });
   const files = new Map(contract.files.map(file => [file.path, file]));
@@ -75,6 +77,17 @@ export const artifactContractSchema = z.object({ schema: z.literal("skillsync.ar
       file.unique_by.some(name => !file.columns.some(column => column.name === name)))) fail();
   }
   for (const check of contract.checks) {
+    if (check.type === "reference_exists") {
+      const source = files.get(check.source.path), target = files.get(check.target.path);
+      if (source?.format !== "csv" || target?.format !== "csv") { fail(); continue; }
+      const sourceColumn = source.columns.find(column => column.name === check.source.field);
+      const targetColumn = target.columns.find(column => column.name === check.target.field);
+      // Exact nonempty string IDs only; a component of a composite key is not unique.
+      if (sourceColumn?.type !== "string" || targetColumn?.type !== "string" ||
+        sourceColumn.allow_empty || targetColumn.allow_empty ||
+        target.unique_by.length !== 1 || target.unique_by[0] !== check.target.field) fail();
+      continue;
+    }
     const csv = files.get(check.csv), json = files.get(check.json);
     if (csv?.format !== "csv" || json?.format !== "json") { fail(); continue; }
     const integerField = (name: string) => json.fields.some(field => field.name === name && field.type === "integer" && field.required);

src/domain/artifact-references.ts

@@ -0,0 +1,59 @@
+import { createHash } from "node:crypto";
+import type { ArtifactContract } from "./artifact-contract.js";
+import type { ArtifactCellObserver, ArtifactFinding } from "./artifact-content.js";
+
+export const MAX_REFERENCE_CELLS = 100_000;
+type Index = { targets?: Set<string>; sources?: Array<{ digest: string; row: number }> };
+
+/** One invocation-local index: declared fields only, hashed IDs, fixed aggregate ceiling. */
+export function createArtifactReferenceIndex(contract: ArtifactContract) {
+  const checks = contract.checks.filter(check => check.type === "reference_exists");
+  const fields = new Map<string, Map<string, Index>>();
+  const index = (path: string, field: string): Index => {
+    let columns = fields.get(path);
+    if (!columns) { columns = new Map(); fields.set(path, columns); }
+    let result = columns.get(field);
+    if (!result) { result = {}; columns.set(field, result); }
+    return result;
+  };
+  for (const check of checks) {
+    index(check.source.path, check.source.field).sources ??= [];
+    index(check.target.path, check.target.field).targets ??= new Set();
+  }
+  let cells = 0, exhausted = false;
+  return {
+    observer(path: string): ArtifactCellObserver | undefined {
+      const columns = fields.get(path);
+      if (!columns) return undefined;
+      return (field, value, row) => {
+        const entry = columns.get(field);
+        if (!entry || exhausted) return;
+        // Repeated IDs count too. Stop hashing and release retained indexes at the ceiling.
+        cells += 1;
+        if (cells > MAX_REFERENCE_CELLS) {
+          exhausted = true;
+          for (const columns of fields.values()) for (const entry of columns.values()) {
+            entry.targets?.clear(); if (entry.sources) entry.sources.length = 0;
+          }
+          return;
+        }
+        const digest = createHash("sha256").update("skillsync.references/v1\0string\0").update(value).digest("hex");
+        entry.targets?.add(digest); entry.sources?.push({ digest, row });
+      };
+    },
+    findings(): ArtifactFinding[] {
+      if (exhausted) return [{ code: "artifact.reference-capacity" }];
+      const findings: ArtifactFinding[] = [];
+      for (const check of checks) {
+        const targets = index(check.target.path, check.target.field).targets!;
+        for (const source of index(check.source.path, check.source.field).sources!) {
+          if (!targets.has(source.digest)) {
+            findings.push({ code: "artifact.reference-missing", path: check.source.path, field: check.source.field, row: source.row });
+            if (findings.length >= 256) return findings;
+          }
+        }
+      }
+      return findings;
+    },
+  };
+}