From 666103e61a8544b86a2d2954d96ed5496102c986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Jacob=20Read=20IV=20=E2=80=94=20Creator=20of=20?= =?UTF-8?q?=C4=80ML=E2=84=A2?= Date: Mon, 14 Sep 2026 08:39:20 -0700 Subject: [PATCH 1/5] Harden independent verification, release discipline, and evidence contracts --- conformance/verifier-challenge-cases.json | 49 +++++++ conformance/verifier-challenge.json | 27 +--- docs/REAL_SCREEN_EVIDENCE_CONTRACT.md | 128 ++++++++++++++++++ docs/RELEASE_DISCIPLINE.md | 89 ++++++++++++ .../INDEPENDENT_IMPLEMENTATION_CHALLENGE.md | 115 ++++++++++++++++ scripts/build-external-verifier-kit.mjs | 5 +- scripts/run-verifier-conformance.mjs | 77 ++++++----- ...challenge-bound-conformance-result.test.js | 5 +- test/external-verifier-action.test.js | 31 ++++- 9 files changed, 460 insertions(+), 66 deletions(-) create mode 100644 conformance/verifier-challenge-cases.json create mode 100644 docs/REAL_SCREEN_EVIDENCE_CONTRACT.md create mode 100644 docs/RELEASE_DISCIPLINE.md create mode 100644 publications/INDEPENDENT_IMPLEMENTATION_CHALLENGE.md diff --git a/conformance/verifier-challenge-cases.json b/conformance/verifier-challenge-cases.json new file mode 100644 index 00000000..bfe8e4ed --- /dev/null +++ b/conformance/verifier-challenge-cases.json @@ -0,0 +1,49 @@ +{ + "schema": "aml-verifier-challenge-cases/1", + "bundle_source": "independent/python/witness-vector.json", + "mutation_language": { + "schema": "aml-json-pointer-replace/1", + "operation": "replace", + "path_semantics": "RFC 6901 JSON Pointer", + "supported_operations": ["replace"] + }, + "cases": [ + { + "id": "golden-valid", + "now": "2030-01-01T00:05:00Z", + "expected_valid": true, + "mutations": [] + }, + { + "id": "tampered-purpose", + "now": "2030-01-01T00:05:00Z", + "expected_valid": false, + "mutations": [ + { + "op": "replace", + "path": "/evidence/receipt/decisions/0/purpose", + "value": "tampered-by-conformance-harness" + } + ] + }, + { + "id": "tampered-challenge", + "now": "2030-01-01T00:05:00Z", + "expected_valid": false, + "mutations": [ + { + "op": "replace", + "path": "/challenge/nonce", + "value": "tampered-challenge-nonce-000000000000000000000" + } + ] + }, + { + "id": "expired-challenge", + "now": "2030-01-01T00:11:00Z", + "expected_valid": false, + "mutations": [] + } + ], + "evidence_boundary": "This file defines project challenge inputs and expected validity. Matching it is black-box interoperability evidence, not certification or proof of implementation independence." +} diff --git a/conformance/verifier-challenge.json b/conformance/verifier-challenge.json index 887ba951..bd025474 100644 --- a/conformance/verifier-challenge.json +++ b/conformance/verifier-challenge.json @@ -4,6 +4,7 @@ "canonical_repository": "https://github.com/aruintelligence/aml-core", "harness": "scripts/run-verifier-conformance.mjs", "witness_vector": "independent/python/witness-vector.json", + "cases_file": "conformance/verifier-challenge-cases.json", "command_contract": { "invocation": " --now ", "stdout": "single JSON object", @@ -17,32 +18,6 @@ "must_bind_exact_witness_vector_sha256": true, "verifier": "scripts/verify-verifier-conformance-result.mjs" }, - "cases": [ - { - "id": "golden-valid", - "expected_valid": true, - "now": "2030-01-01T00:05:00Z", - "mutation": null - }, - { - "id": "tampered-purpose", - "expected_valid": false, - "now": "2030-01-01T00:05:00Z", - "mutation": "evidence.receipt.decisions[0].purpose is changed" - }, - { - "id": "tampered-challenge", - "expected_valid": false, - "now": "2030-01-01T00:05:00Z", - "mutation": "challenge.nonce is changed" - }, - { - "id": "expired-challenge", - "expected_valid": false, - "now": "2030-01-01T00:11:00Z", - "mutation": null - } - ], "independence": { "required_for_external_witness_credit": true, "must_be_maintained_outside_canonical_repository": true, diff --git a/docs/REAL_SCREEN_EVIDENCE_CONTRACT.md b/docs/REAL_SCREEN_EVIDENCE_CONTRACT.md new file mode 100644 index 00000000..7d716fc7 --- /dev/null +++ b/docs/REAL_SCREEN_EVIDENCE_CONTRACT.md @@ -0,0 +1,128 @@ +# ĀML™ Real-Screen Evidence Contract + +This document defines a common evidence shape for ĀML experiments against actual rendered interfaces. + +The purpose is reproducibility and provenance. It does not convert subjective review labels into objective measurements of cognition, attention, wellbeing, accessibility, morality, conversion, or business performance. + +## Required separation + +A real-screen experiment should keep these layers distinct: + +1. **production observation** — what was actually rendered or captured; +2. **review labels** — explicit human or model-assisted judgments and their provenance; +3. **deterministic policy** — the rule/version applied to those declared inputs; +4. **receipt** — the resulting machine-readable decision evidence; +5. **remediation** — code or content changes made after review; +6. **after observation** — a new production capture proving what was actually deployed. + +Source-code remediation is not, by itself, production-after evidence. + +## Recommended experiment directory + +```text +evidence/real-screen/YYYY-MM-DD-slug/ + README.md + capture-metadata.json + screen.html + labels.json + receipt.json + REMEDIATION.md # when a change is made + after/ # when production after-capture exists + capture-metadata.json + screen.html + labels.json + receipt.json +``` + +Additional untouched screenshot, DOM, network, or browser artifacts may be preserved when the capture system supports them. + +## Capture metadata + +`capture-metadata.json` SHOULD record, where available: + +- source URL; +- capture date/time and timezone; +- capture agent/tool; +- viewport and device-pixel-ratio; +- scroll position; +- page title; +- hashes and byte sizes of screenshot, rendered HTML, DOM snapshot, or other preserved artifacts; +- a clear description of which artifact is passed to the ĀML harness; +- limitations of the capture. + +A hash proves byte identity. It does not prove that a label is correct or independent. + +## Labels and provenance + +`labels.json` SHOULD be separate from the captured screen material and SHOULD identify for each reviewed element: + +- stable element/review ID; +- declared purpose; +- `attention_cost`; +- `restoration_value`; +- provenance source kind; +- reviewer/author identity or role where appropriate; +- source reference tying the judgment to the capture; +- whether the review is project-directed; +- any disagreement or uncertainty worth preserving. + +The project MUST NOT describe these numeric labels as objective physiological, neurological, psychological, clinical, or scientific measurements unless independent evidence actually establishes that claim. + +## Policy binding + +The receipt SHOULD identify the exact policy/rule and software state used to evaluate the labels. For the minimal prototype gate: + +```text +render_allowed = restoration_value >= attention_cost +``` + +A deterministic result proves that the declared inputs and rule reproduce the same decision. It does not prove that the inputs are universally correct. + +## Deterministic rerun + +Where feasible: + +1. run the harness once; +2. preserve the receipt; +3. run it again from the same captured input and labels; +4. compare the resulting receipt bytes or documented deterministic fields; +5. record PASS / FAIL / MIXED. + +Any nondeterminism should be surfaced, not hidden. + +## `real_screen_evidence_eligible` + +A project-controlled experiment SHOULD remain `real_screen_evidence_eligible: false` unless the repository's stated evidence requirements for that status are actually met. + +The flag must not be upgraded merely because: + +- the source URL is public; +- the page belonged to ĀRU or a customer; +- the project generated a receipt; +- a build passed; +- the project author agrees with the labels. + +## Before/after work + +An after-state should be treated as proven only after a new production capture is obtained. Preserve the previous capture rather than overwriting it. + +A remediation report SHOULD state: + +- exact element(s) changed; +- what capability or restoration path was preserved; +- before checkpoint/commit where available; +- after checkpoint/commit where available; +- whether production publication is verified; +- what claims are *not* established by the intervention. + +## Capability preservation + +A SUPPRESS decision should not automatically mean deletion. For important controls, test whether the useful capability remains reachable through a lower-burden path. This project refers to that engineering pattern as **suppression without capability destruction**. + +## External sites + +Observational experiments on public third-party interfaces must not imply partnership, endorsement, audit authority, certification, or maintainer approval. Respect applicable terms, access controls, copyright, privacy, and security boundaries. + +## Claim boundary + +A real-screen evidence package can establish reproducibility of the captured bytes, declared labels, policy inputs, and deterministic output. On its own it does **not** establish conversion lift, user preference, cognitive benefit, accessibility conformance, safety, scientific validity, regulatory compliance, or independent adoption. diff --git a/docs/RELEASE_DISCIPLINE.md b/docs/RELEASE_DISCIPLINE.md new file mode 100644 index 00000000..51e5a3e6 --- /dev/null +++ b/docs/RELEASE_DISCIPLINE.md @@ -0,0 +1,89 @@ +# ĀML™ Release Discipline + +This document defines the minimum publication discipline for a release that is presented as a stable or externally testable ĀML™ software/protocol state. + +It is an engineering and evidence contract, not a claim of standards-body approval, certification, scientific validation, regulatory compliance, or universal production suitability. + +## Release identity + +A release intended for outside reproduction SHOULD identify all of the following: + +1. semantic version or prerelease identifier; +2. immutable Git commit SHA; +3. package version; +4. applicable verifier-contract snapshot; +5. applicable conformance/test-vector versions; +6. cryptographic hashes for published release artifacts where practical; +7. stable/preview/experimental status. + +A moving `main` branch is useful for development but is not a substitute for an immutable release identity. + +## Channels + +### Stable + +A stable release is the current supported package/API contract. Stable does not mean flawless, certified, scientifically validated, or suitable for every production system. + +### Preview / release candidate + +A preview may contain broader architecture intended for outside testing. Breaking changes remain possible and MUST be stated plainly. + +### Experimental + +Research artifacts, field experiments, draft RFCs, prototypes, and exploratory interfaces MUST remain distinguishable from the stable package contract. + +## Minimum release evidence + +Before calling a release ready for broad outside reproduction, the project SHOULD preserve: + +- the exact source commit; +- automated test result(s); +- public conformance result(s) where applicable; +- package/artifact manifest and checksums where available; +- external-verifier kit root and manifest where applicable; +- verifier-contract snapshot and migration lineage; +- known limitations and claim boundaries; +- rollback or previous-stable reference. + +Where signed release/provenance artifacts exist, they SHOULD bind the exact artifact bytes and the release identity they represent. + +## Independent verification boundary + +Project-authored CI proves that project-authored checks passed for a particular state. It does not become independent evidence merely because it runs on GitHub infrastructure. + +Independent evidence requires an outside implementation, outside reproducer, or other evidence source that satisfies the relevant independence rules. PASS, FAIL, and MIXED results are all valid evidence states. + +## Release candidate checklist + +Before advancing a new stable release, verify: + +- [ ] package/API version coherence; +- [ ] full automated test suite passes; +- [ ] conformance and interoperability checks pass or known failures are disclosed; +- [ ] security workflows have no unexplained blocking failure; +- [ ] external verifier/witness artifacts build deterministically where applicable; +- [ ] published hashes and manifests correspond to the final commit; +- [ ] documentation links resolve; +- [ ] licensing/trademark notices match the intended distribution model; +- [ ] experimental claims are not presented as independent adoption or validation; +- [ ] release notes state breaking changes, migrations, and known limitations. + +## Release receipts + +A release receipt SHOULD make it possible to answer, without trusting prose alone: + +- Which commit was released? +- Which package version was released? +- Which verifier contract applied? +- Which tests or checks were run? +- Which artifact hashes were published? +- Which signing/trust policy applied? +- What was stable versus experimental? + +## No retroactive evidence rewriting + +Historical releases, snapshots, receipts, and external witness records should remain append-only wherever practical. If a correction is necessary, preserve the original state and publish a correction or superseding record rather than silently rewriting the historical claim. + +## Commercial boundary + +Release availability under the repository software license does not itself grant rights to official ĀML™/ĀRU™ branding, certification-style identity, endorsement, OEM/co-branding, managed infrastructure, or separately licensed commercial offerings. See `LICENSING.md`, `TRADEMARKS.md`, and `COMMERCIAL.md`. diff --git a/publications/INDEPENDENT_IMPLEMENTATION_CHALLENGE.md b/publications/INDEPENDENT_IMPLEMENTATION_CHALLENGE.md new file mode 100644 index 00000000..870ac509 --- /dev/null +++ b/publications/INDEPENDENT_IMPLEMENTATION_CHALLENGE.md @@ -0,0 +1,115 @@ +# ĀML™ Independent Implementation Challenge + +The strongest next test of ĀML is not another project-authored implementation. It is an outside implementation that can reproduce a published contract without importing or wrapping the canonical runtime. + +This challenge invites implementers to test the public protocol as a falsifiable interoperability target. + +## Goal + +Using only the published protocol text, schemas, test vectors, verifier-contract snapshots, and the reference-code-free External Verifier Kit, build a verifier in another repository/runtime and report what happens. + +**PASS, FAIL, and MIXED are all useful outcomes.** + +A failure that identifies an ambiguous or underspecified contract can be more valuable than agreement. + +## Independence requirements for independent-implementation credit + +An implementation seeking independent evidence credit should: + +- be maintained outside `aruintelligence/aml-core`; +- not import the canonical JavaScript/Python/Go reference verifier; +- not shell out to the `aml-core` CLI or API as the actual decision engine; +- implement the published contract from the public specification/materials; +- disclose meaningful shared dependencies or copied implementation logic; +- publish enough source and reproduction detail for others to inspect the result. + +Using the reference-code-free kit, public JSON schemas, vectors, RFCs, and protocol documents is expected and does not by itself defeat independence. + +## Starting artifact + +Build or download the External Verifier Kit generated by: + +```bash +node scripts/build-external-verifier-kit.mjs +``` + +The kit intentionally excludes reference implementation source code and contains the public verifier contract, language-neutral challenge cases, schemas, vectors, snapshot/migration material, and witness-submission guidance. + +## Black-box command contract + +Your verifier should accept the published command shape: + +```text + --now +``` + +A valid bundle should emit a JSON object containing `valid: true` and exit `0`. Invalid challenge cases should be rejected with a nonzero exit. + +Run the canonical black-box harness against your verifier: + +```bash +node scripts/run-verifier-conformance.mjs -- [args...] +``` + +The exact challenge cases are published as language-neutral data in: + +`conformance/verifier-challenge-cases.json` + +That file defines evaluation times, expected validity, and exact mutation operations. An outside implementer should not need to read the canonical harness source to discover the challenge bytes. + +## Report the exact target + +An outside report should identify: + +- source repository URL and commit SHA of the outside implementation; +- language/runtime and version; +- ĀML repository commit or release tested; +- verifier-contract snapshot ID; +- challenge SHA-256; +- challenge-case-corpus SHA-256 where available; +- witness-vector SHA-256; +- PASS / FAIL / MIXED result; +- per-case observed outcomes; +- reproduction command; +- known deviations, ambiguity, or unsupported features. + +Do not report a moving branch name alone when an immutable commit is available. + +## What counts as success? + +A project-defined black-box PASS is evidence that the tested outside implementation produced the expected validity results for the exact challenge state. It is not, by itself: + +- certification; +- proof that the implementation is fully independent; +- proof the entire ĀML specification is correct; +- standards-body approval; +- scientific validation; +- regulatory compliance; +- security certification; +- production suitability; +- evidence of broad adoption. + +Independent status is a provenance claim and must be supported separately from the technical PASS result. + +## Negative evidence is welcome + +Please publish FAIL or MIXED when that is the result. Useful failure reports include: + +- ambiguous canonicalization behavior; +- schema/spec disagreement; +- incompatible edge cases; +- unexpected success on a tampered vector; +- unexpected rejection of the golden vector; +- divergent time handling; +- unclear migration semantics; +- behavior that cannot be reproduced from the published kit alone. + +The project should fix an underspecified contract rather than redefine a failure as success. + +## Public witness path + +Follow `publications/WITNESS_SUBMISSION.md` for the repository's public witness/evidence submission path. Do not place secrets, private keys, customer data, or confidential information in a public report. + +## Trademark boundary + +Independent technical implementation or conformance testing does not automatically grant official ĀML™ branding, certification-style marks, endorsement, partnership, OEM/co-branding, or other reserved trademark rights. See `TRADEMARKS.md` and `LICENSING.md`. diff --git a/scripts/build-external-verifier-kit.mjs b/scripts/build-external-verifier-kit.mjs index 3275b59f..11f53400 100644 --- a/scripts/build-external-verifier-kit.mjs +++ b/scripts/build-external-verifier-kit.mjs @@ -7,6 +7,7 @@ import { execFileSync } from 'node:child_process'; export const KIT_FILES = [ 'conformance/verifier-challenge.json', + 'conformance/verifier-challenge-cases.json', 'conformance/witness-record.example.json', 'independent/python/witness-vector.json', 'protocol/sorted-json-v1.md', @@ -67,6 +68,7 @@ export function buildExternalVerifierKit(outputDir = 'dist/external-verifier-kit if (!currentSnapshot?.manifest || !currentSnapshot?.source_commit) throw new Error('current verifier snapshot is not resolvable from catalog'); const challengeBytes = fs.readFileSync('conformance/verifier-challenge.json'); + const challengeCasesBytes = fs.readFileSync('conformance/verifier-challenge-cases.json'); const witnessVectorBytes = fs.readFileSync('independent/python/witness-vector.json'); fs.rmSync(outputDir, { recursive: true, force: true }); @@ -82,7 +84,7 @@ export function buildExternalVerifierKit(outputDir = 'dist/external-verifier-kit } entries.sort((a, b) => codeUnitCompare(a.path, b.path)); - const readme = `# ĀML External Verifier Kit\n\nThis artifact is intentionally **reference-code-free**. It contains the current and historical verifier-contract snapshots, explicit migration lineage, public protocol text, JSON Schemas, canonicalization/test vectors, the black-box verifier challenge, one JSON witness fixture, and witness-submission material. It does not contain the JavaScript, Python, Go, or other reference verifier implementations from aml-core.\n\nCurrent verifier snapshot: **${catalog.current_snapshot}**\nMigration count: **${(catalog.migrations || []).length}**\n\nImplement the published contract in your own runtime, then run the External Verifier Challenge from your own repository. PASS, FAIL, and MIXED results are all useful.\n\nCommand contract:\n\n\`\`\`text\n --now \n\`\`\`\n\nA valid bundle must emit JSON with \`valid: true\` and exit 0. Invalid challenge cases must be rejected with a nonzero exit. Snapshot 2 conformance results identify the exact challenge and golden witness-vector bytes by SHA-256.\n\nThis kit reduces accidental dependence on reference implementation code. Possessing or using the kit does not itself prove an implementation is independent.\n`; + const readme = `# ĀML External Verifier Kit\n\nThis artifact is intentionally **reference-code-free**. It contains the current and historical verifier-contract snapshots, explicit migration lineage, public protocol text, JSON Schemas, canonicalization/test vectors, the black-box verifier challenge, its language-neutral case corpus, one JSON witness fixture, and witness-submission material. It does not contain the JavaScript, Python, Go, or other reference verifier implementations from aml-core.\n\nCurrent verifier snapshot: **${catalog.current_snapshot}**\nMigration count: **${(catalog.migrations || []).length}**\n\nImplement the published contract in your own runtime, then run the External Verifier Challenge from your own repository. PASS, FAIL, and MIXED results are all useful. The exact challenge cases are data-defined in \`conformance/verifier-challenge-cases.json\` using a narrow RFC 6901 JSON Pointer replacement profile, so an implementer does not need to inspect the canonical JavaScript harness to discover the mutation bytes.\n\nCommand contract:\n\n\`\`\`text\n --now \n\`\`\`\n\nA valid bundle must emit JSON with \`valid: true\` and exit 0. Invalid challenge cases must be rejected with a nonzero exit. Snapshot 2 conformance results identify the exact challenge and golden witness-vector bytes by SHA-256; the kit manifest additionally identifies the exact language-neutral case corpus.\n\nThis kit reduces accidental dependence on reference implementation code. Possessing or using the kit does not itself prove an implementation is independent.\n`; const readmeBytes = Buffer.from(readme, 'utf8'); fs.writeFileSync(path.join(outputDir, 'README.md'), readmeBytes); entries.push({ path: 'README.md', bytes: readmeBytes.length, sha256: sha256(readmeBytes) }); @@ -99,6 +101,7 @@ export function buildExternalVerifierKit(outputDir = 'dist/external-verifier-kit contract_snapshot_count: (catalog.snapshots || []).length, contract_migration_count: (catalog.migrations || []).length, challenge_sha256: sha256(challengeBytes), + challenge_cases_sha256: sha256(challengeCasesBytes), witness_vector_sha256: sha256(witnessVectorBytes), file_count: entries.length, root_algorithm: 'SHA-256 over UTF-8 sorted SHA256SUMS material', diff --git a/scripts/run-verifier-conformance.mjs b/scripts/run-verifier-conformance.mjs index ddc1e88b..87b1871d 100644 --- a/scripts/run-verifier-conformance.mjs +++ b/scripts/run-verifier-conformance.mjs @@ -17,13 +17,25 @@ const baseArgs = process.argv.slice(split + 2); const scriptDir = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.resolve(scriptDir, '..'); const challengePath = path.join(repoRoot, 'conformance/verifier-challenge.json'); -const vectorPath = path.join(repoRoot, 'independent/python/witness-vector.json'); const challengeBytes = fs.readFileSync(challengePath); -const vectorBytes = fs.readFileSync(vectorPath); const challenge = JSON.parse(challengeBytes.toString('utf8')); +if (typeof challenge.cases_file !== 'string' || !challenge.cases_file) throw new Error('Verifier challenge must declare cases_file'); + +const casesPath = path.join(repoRoot, challenge.cases_file); +const casesBytes = fs.readFileSync(casesPath); +const casesContract = JSON.parse(casesBytes.toString('utf8')); +if (casesContract.schema !== 'aml-verifier-challenge-cases/1') throw new Error('Unsupported verifier challenge case schema'); +if (casesContract.mutation_language?.schema !== 'aml-json-pointer-replace/1') throw new Error('Unsupported verifier challenge mutation language'); +if (!Array.isArray(casesContract.cases) || !casesContract.cases.length) throw new Error('Verifier challenge requires cases'); +if (typeof casesContract.bundle_source !== 'string' || !casesContract.bundle_source) throw new Error('Verifier challenge cases require bundle_source'); +if (challenge.witness_vector !== casesContract.bundle_source) throw new Error('Challenge witness_vector and cases bundle_source must match'); + +const vectorPath = path.join(repoRoot, casesContract.bundle_source); +const vectorBytes = fs.readFileSync(vectorPath); const source = JSON.parse(vectorBytes.toString('utf8')); const sha256 = (bytes) => crypto.createHash('sha256').update(bytes).digest('hex'); const challengeSha256 = sha256(challengeBytes); +const challengeCasesSha256 = sha256(casesBytes); const witnessVectorSha256 = sha256(vectorBytes); const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'aml-verifier-conformance-')); @@ -49,39 +61,36 @@ function write(name, value) { return target; } -const purposeTamper = structuredClone(source); -purposeTamper.evidence.receipt.decisions[0].purpose = 'tampered-by-conformance-harness'; - -const challengeTamper = structuredClone(source); -challengeTamper.challenge.nonce = 'tampered-challenge-nonce-000000000000000000000'; +function decodePointerToken(token) { + return token.replace(/~1/g, '/').replace(/~0/g, '~'); +} -const cases = [ - { - id: 'golden-valid', - expected: true, - run: () => invoke(vectorPath, '2030-01-01T00:05:00Z') - }, - { - id: 'tampered-purpose', - expected: false, - run: () => invoke(write('tampered-purpose', purposeTamper), '2030-01-01T00:05:00Z') - }, - { - id: 'tampered-challenge', - expected: false, - run: () => invoke(write('tampered-challenge', challengeTamper), '2030-01-01T00:05:00Z') - }, - { - id: 'expired-challenge', - expected: false, - run: () => invoke(vectorPath, '2030-01-01T00:11:00Z') +function applyReplace(root, mutation) { + if (!mutation || mutation.op !== 'replace' || typeof mutation.path !== 'string' || !mutation.path.startsWith('/')) { + throw new Error('Unsupported verifier challenge mutation'); + } + const tokens = mutation.path.slice(1).split('/').map(decodePointerToken); + let parent = root; + for (const token of tokens.slice(0, -1)) { + if (parent === null || typeof parent !== 'object' || !(token in parent)) throw new Error(`Mutation path does not exist: ${mutation.path}`); + parent = parent[token]; } -]; + const leaf = tokens.at(-1); + if (parent === null || typeof parent !== 'object' || !(leaf in parent)) throw new Error(`Mutation path does not exist: ${mutation.path}`); + parent[leaf] = structuredClone(mutation.value); +} + +function materializeCase(testCase) { + const bundle = structuredClone(source); + for (const mutation of testCase.mutations || []) applyReplace(bundle, mutation); + if (!(testCase.mutations || []).length) return vectorPath; + return write(testCase.id, bundle); +} -const results = cases.map(test => { - const observed = test.run(); - const passed = observed.valid === test.expected && (test.expected ? observed.exit_code === 0 : observed.exit_code !== 0); - return { id: test.id, expected_valid: test.expected, passed, observed }; +const results = casesContract.cases.map(testCase => { + const observed = invoke(materializeCase(testCase), testCase.now); + const passed = observed.valid === testCase.expected_valid && (testCase.expected_valid ? observed.exit_code === 0 : observed.exit_code !== 0); + return { id: testCase.id, expected_valid: testCase.expected_valid, passed, observed }; }); const passed = results.every(r => r.passed); @@ -91,11 +100,13 @@ console.log(JSON.stringify({ challenge_schema: challenge.schema, challenge_sha256: challengeSha256, witness_vector_sha256: witnessVectorSha256, + challenge_cases: challenge.cases_file, + challenge_cases_sha256: challengeCasesSha256, harness_root: repoRoot, command: [command, ...baseArgs], passed, results, - claim_boundary: 'PASS is project-defined black-box compatibility evidence bound to the exact published challenge and witness-vector bytes; it is not certification or proof of verifier independence.' + claim_boundary: 'PASS is project-defined black-box compatibility evidence bound to the exact published challenge, language-neutral case corpus, and witness-vector bytes; it is not certification or proof of verifier independence.' }, null, 2)); process.exit(passed ? 0 : 1); diff --git a/test/challenge-bound-conformance-result.test.js b/test/challenge-bound-conformance-result.test.js index 07ccfb5c..4f375437 100644 --- a/test/challenge-bound-conformance-result.test.js +++ b/test/challenge-bound-conformance-result.test.js @@ -9,10 +9,11 @@ import { spawnSync } from 'node:child_process'; const challengeBytes = fs.readFileSync('conformance/verifier-challenge.json'); const vectorBytes = fs.readFileSync('independent/python/witness-vector.json'); const challenge = JSON.parse(challengeBytes.toString('utf8')); +const cases = JSON.parse(fs.readFileSync(challenge.cases_file, 'utf8')); const sha256 = (bytes) => crypto.createHash('sha256').update(bytes).digest('hex'); function goodResult() { - const results = challenge.cases.map((entry) => ({ + const results = cases.cases.map((entry) => ({ id: entry.id, expected_valid: entry.expected_valid, passed: true, @@ -52,6 +53,8 @@ test('challenge contract requires exact challenge and vector bindings', () => { assert.equal(challenge.result_contract.must_bind_exact_challenge_sha256, true); assert.equal(challenge.result_contract.must_bind_exact_witness_vector_sha256, true); assert.equal(challenge.result_contract.verifier, 'scripts/verify-verifier-conformance-result.mjs'); + assert.equal(challenge.cases_file, 'conformance/verifier-challenge-cases.json'); + assert.equal(cases.bundle_source, challenge.witness_vector); }); test('archived conformance result verifies against exact local challenge bytes', () => { diff --git a/test/external-verifier-action.test.js b/test/external-verifier-action.test.js index 8dba7847..6f014510 100644 --- a/test/external-verifier-action.test.js +++ b/test/external-verifier-action.test.js @@ -3,13 +3,18 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; const challenge = JSON.parse(fs.readFileSync('conformance/verifier-challenge.json', 'utf8')); +const cases = JSON.parse(fs.readFileSync(challenge.cases_file, 'utf8')); const harness = fs.readFileSync('scripts/run-verifier-conformance.mjs', 'utf8'); const action = fs.readFileSync('actions/verifier-conformance/action.yml', 'utf8'); -test('external verifier challenge publishes the exact harness cases', () => { +test('external verifier challenge publishes a language-neutral exact case corpus', () => { assert.equal(challenge.schema, 'aml-external-verifier-challenge/1'); + assert.equal(challenge.cases_file, 'conformance/verifier-challenge-cases.json'); + assert.equal(cases.schema, 'aml-verifier-challenge-cases/1'); + assert.equal(cases.bundle_source, challenge.witness_vector); + assert.equal(cases.mutation_language.schema, 'aml-json-pointer-replace/1'); assert.deepEqual( - challenge.cases.map((entry) => [entry.id, entry.expected_valid]), + cases.cases.map((entry) => [entry.id, entry.expected_valid]), [ ['golden-valid', true], ['tampered-purpose', false], @@ -17,9 +22,25 @@ test('external verifier challenge publishes the exact harness cases', () => { ['expired-challenge', false] ] ); - for (const entry of challenge.cases) { - assert.match(harness, new RegExp(`id: ['\"]${entry.id}['\"]`)); - } + assert.deepEqual(cases.cases[1].mutations, [{ + op: 'replace', + path: '/evidence/receipt/decisions/0/purpose', + value: 'tampered-by-conformance-harness' + }]); + assert.deepEqual(cases.cases[2].mutations, [{ + op: 'replace', + path: '/challenge/nonce', + value: 'tampered-challenge-nonce-000000000000000000000' + }]); +}); + +test('external verifier harness consumes the case corpus instead of hard-coding cases', () => { + assert.match(harness, /challenge\.cases_file/); + assert.match(harness, /casesContract\.cases\.map/); + assert.match(harness, /aml-json-pointer-replace\/1/); + assert.doesNotMatch(harness, /purposeTamper/); + assert.doesNotMatch(harness, /challengeTamper/); + assert.match(harness, /challenge_cases_sha256/); }); test('external verifier action drives the canonical black-box harness', () => { From c76ebf3590c206f8e348655625621953a8740d49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Jacob=20Read=20IV=20=E2=80=94=20Creator=20of=20?= =?UTF-8?q?=C4=80ML=E2=84=A2?= Date: Mon, 14 Sep 2026 08:41:29 -0700 Subject: [PATCH 2/5] Bind external verifier challenge to exact case corpus --- conformance/verifier-challenge.json | 1 + 1 file changed, 1 insertion(+) diff --git a/conformance/verifier-challenge.json b/conformance/verifier-challenge.json index bd025474..cdeff6d8 100644 --- a/conformance/verifier-challenge.json +++ b/conformance/verifier-challenge.json @@ -5,6 +5,7 @@ "harness": "scripts/run-verifier-conformance.mjs", "witness_vector": "independent/python/witness-vector.json", "cases_file": "conformance/verifier-challenge-cases.json", + "cases_sha256": "15409d9b921b8d8e16a54d9cb83760c102fa43ef4e9a349f195e4551e1beaa56", "command_contract": { "invocation": " --now ", "stdout": "single JSON object", From e991de5bf50872aa18b70fe6a164a63cd44de731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Jacob=20Read=20IV=20=E2=80=94=20Creator=20of=20?= =?UTF-8?q?=C4=80ML=E2=84=A2?= Date: Mon, 14 Sep 2026 08:41:53 -0700 Subject: [PATCH 3/5] Bind verifier case corpus through challenge without changing result schema --- scripts/run-verifier-conformance.mjs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/run-verifier-conformance.mjs b/scripts/run-verifier-conformance.mjs index 87b1871d..42ba6a94 100644 --- a/scripts/run-verifier-conformance.mjs +++ b/scripts/run-verifier-conformance.mjs @@ -19,10 +19,14 @@ const repoRoot = path.resolve(scriptDir, '..'); const challengePath = path.join(repoRoot, 'conformance/verifier-challenge.json'); const challengeBytes = fs.readFileSync(challengePath); const challenge = JSON.parse(challengeBytes.toString('utf8')); +const sha256 = (bytes) => crypto.createHash('sha256').update(bytes).digest('hex'); if (typeof challenge.cases_file !== 'string' || !challenge.cases_file) throw new Error('Verifier challenge must declare cases_file'); +if (!/^[0-9a-f]{64}$/.test(String(challenge.cases_sha256 || ''))) throw new Error('Verifier challenge must bind cases_sha256'); const casesPath = path.join(repoRoot, challenge.cases_file); const casesBytes = fs.readFileSync(casesPath); +const casesSha256 = sha256(casesBytes); +if (casesSha256 !== challenge.cases_sha256) throw new Error('Verifier challenge case corpus hash mismatch'); const casesContract = JSON.parse(casesBytes.toString('utf8')); if (casesContract.schema !== 'aml-verifier-challenge-cases/1') throw new Error('Unsupported verifier challenge case schema'); if (casesContract.mutation_language?.schema !== 'aml-json-pointer-replace/1') throw new Error('Unsupported verifier challenge mutation language'); @@ -33,9 +37,7 @@ if (challenge.witness_vector !== casesContract.bundle_source) throw new Error('C const vectorPath = path.join(repoRoot, casesContract.bundle_source); const vectorBytes = fs.readFileSync(vectorPath); const source = JSON.parse(vectorBytes.toString('utf8')); -const sha256 = (bytes) => crypto.createHash('sha256').update(bytes).digest('hex'); const challengeSha256 = sha256(challengeBytes); -const challengeCasesSha256 = sha256(casesBytes); const witnessVectorSha256 = sha256(vectorBytes); const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'aml-verifier-conformance-')); @@ -100,13 +102,11 @@ console.log(JSON.stringify({ challenge_schema: challenge.schema, challenge_sha256: challengeSha256, witness_vector_sha256: witnessVectorSha256, - challenge_cases: challenge.cases_file, - challenge_cases_sha256: challengeCasesSha256, harness_root: repoRoot, command: [command, ...baseArgs], passed, results, - claim_boundary: 'PASS is project-defined black-box compatibility evidence bound to the exact published challenge, language-neutral case corpus, and witness-vector bytes; it is not certification or proof of verifier independence.' + claim_boundary: 'PASS is project-defined black-box compatibility evidence bound through the exact challenge to the language-neutral case corpus and exact witness-vector bytes; it is not certification or proof of verifier independence.' }, null, 2)); process.exit(passed ? 0 : 1); From eceeb112f57ac20b5d1c3d2ea4c539d291a682f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Jacob=20Read=20IV=20=E2=80=94=20Creator=20of=20?= =?UTF-8?q?=C4=80ML=E2=84=A2?= Date: Mon, 14 Sep 2026 08:42:11 -0700 Subject: [PATCH 4/5] Verify archived results against bound language-neutral case corpus --- scripts/verify-verifier-conformance-result.mjs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/verify-verifier-conformance-result.mjs b/scripts/verify-verifier-conformance-result.mjs index 081a526d..af6b75d4 100644 --- a/scripts/verify-verifier-conformance-result.mjs +++ b/scripts/verify-verifier-conformance-result.mjs @@ -22,13 +22,19 @@ try { } const challengePath = path.resolve('conformance/verifier-challenge.json'); -const vectorPath = path.resolve('independent/python/witness-vector.json'); const challengeBytes = fs.readFileSync(challengePath); -const vectorBytes = fs.readFileSync(vectorPath); const challenge = JSON.parse(challengeBytes.toString('utf8')); +const casesPath = path.resolve(challenge.cases_file || ''); +const casesBytes = fs.readFileSync(casesPath); +const casesContract = JSON.parse(casesBytes.toString('utf8')); +const vectorPath = path.resolve(challenge.witness_vector || casesContract.bundle_source || ''); +const vectorBytes = fs.readFileSync(vectorPath); const expectedChallengeHash = sha256(challengeBytes); +const expectedCasesHash = sha256(casesBytes); const expectedVectorHash = sha256(vectorBytes); +if (challenge.cases_sha256 !== expectedCasesHash) failures.push('local challenge cases_sha256 does not match exact local case-corpus bytes'); +if (casesContract.bundle_source !== challenge.witness_vector) failures.push('challenge witness_vector does not match case-corpus bundle_source'); if (result?.schema !== challenge.result_contract?.schema) failures.push('result schema does not match challenge result contract'); if (result?.challenge_schema !== challenge.schema) failures.push('challenge_schema mismatch'); if (!/^[0-9a-f]{64}$/.test(String(result?.challenge_sha256 || ''))) failures.push('challenge_sha256 must be lowercase SHA-256 hex'); @@ -37,7 +43,7 @@ if (result?.challenge_sha256 !== expectedChallengeHash) failures.push('challenge if (result?.witness_vector_sha256 !== expectedVectorHash) failures.push('witness_vector_sha256 does not match exact local witness-vector bytes'); if (!Array.isArray(result?.results)) failures.push('results must be an array'); -const expectedCases = Array.isArray(challenge.cases) ? challenge.cases : []; +const expectedCases = Array.isArray(casesContract.cases) ? casesContract.cases : []; if (Array.isArray(result?.results)) { if (result.results.length !== expectedCases.length) failures.push('result case count mismatch'); const seen = new Set(); @@ -69,9 +75,10 @@ console.log(JSON.stringify({ verified, passed: verified ? result.passed === true : false, challenge_sha256: expectedChallengeHash, + cases_sha256: expectedCasesHash, witness_vector_sha256: expectedVectorHash, failures, - claim_boundary: 'Verification proves that this result is structurally consistent with the exact local challenge and witness-vector bytes. It does not prove the tested verifier is independent, trustworthy, secure, or broadly adopted.' + claim_boundary: 'Verification proves that this result is structurally consistent with the exact local challenge, its bound case corpus, and witness-vector bytes. It does not prove the tested verifier is independent, trustworthy, secure, or broadly adopted.' }, null, 2)); process.exit(verified ? 0 : 1); From 05b3745f9074a8c1d4d646292ca05e31dcb91651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Jacob=20Read=20IV=20=E2=80=94=20Creator=20of=20?= =?UTF-8?q?=C4=80ML=E2=84=A2?= Date: Mon, 14 Sep 2026 08:42:32 -0700 Subject: [PATCH 5/5] Assert challenge binds exact language-neutral case corpus --- test/external-verifier-action.test.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/external-verifier-action.test.js b/test/external-verifier-action.test.js index 6f014510..4dadcc8f 100644 --- a/test/external-verifier-action.test.js +++ b/test/external-verifier-action.test.js @@ -1,15 +1,19 @@ import test from 'node:test'; import assert from 'node:assert/strict'; +import crypto from 'node:crypto'; import fs from 'node:fs'; const challenge = JSON.parse(fs.readFileSync('conformance/verifier-challenge.json', 'utf8')); -const cases = JSON.parse(fs.readFileSync(challenge.cases_file, 'utf8')); +const casesBytes = fs.readFileSync(challenge.cases_file); +const cases = JSON.parse(casesBytes.toString('utf8')); const harness = fs.readFileSync('scripts/run-verifier-conformance.mjs', 'utf8'); const action = fs.readFileSync('actions/verifier-conformance/action.yml', 'utf8'); +const sha256 = (bytes) => crypto.createHash('sha256').update(bytes).digest('hex'); -test('external verifier challenge publishes a language-neutral exact case corpus', () => { +test('external verifier challenge publishes and binds a language-neutral exact case corpus', () => { assert.equal(challenge.schema, 'aml-external-verifier-challenge/1'); assert.equal(challenge.cases_file, 'conformance/verifier-challenge-cases.json'); + assert.equal(challenge.cases_sha256, sha256(casesBytes)); assert.equal(cases.schema, 'aml-verifier-challenge-cases/1'); assert.equal(cases.bundle_source, challenge.witness_vector); assert.equal(cases.mutation_language.schema, 'aml-json-pointer-replace/1'); @@ -36,11 +40,11 @@ test('external verifier challenge publishes a language-neutral exact case corpus test('external verifier harness consumes the case corpus instead of hard-coding cases', () => { assert.match(harness, /challenge\.cases_file/); + assert.match(harness, /challenge\.cases_sha256/); assert.match(harness, /casesContract\.cases\.map/); assert.match(harness, /aml-json-pointer-replace\/1/); assert.doesNotMatch(harness, /purposeTamper/); assert.doesNotMatch(harness, /challengeTamper/); - assert.match(harness, /challenge_cases_sha256/); }); test('external verifier action drives the canonical black-box harness', () => {