diff --git a/conformance/verifier-challenge-cases.json b/conformance/verifier-challenge-cases.json new file mode 100644 index 0000000..bfe8e4e --- /dev/null +++ b/conformance/verifier-challenge-cases.json @@ -0,0 +1,49 @@ +{ + "schema": "aml-verifier-challenge-cases/1", + "bundle_source": "independent/python/witness-vector.json", + "mutation_language": { + "schema": "aml-json-pointer-replace/1", + "operation": "replace", + "path_semantics": "RFC 6901 JSON Pointer", + "supported_operations": ["replace"] + }, + "cases": [ + { + "id": "golden-valid", + "now": "2030-01-01T00:05:00Z", + "expected_valid": true, + "mutations": [] + }, + { + "id": "tampered-purpose", + "now": "2030-01-01T00:05:00Z", + "expected_valid": false, + "mutations": [ + { + "op": "replace", + "path": "/evidence/receipt/decisions/0/purpose", + "value": "tampered-by-conformance-harness" + } + ] + }, + { + "id": "tampered-challenge", + "now": "2030-01-01T00:05:00Z", + "expected_valid": false, + "mutations": [ + { + "op": "replace", + "path": "/challenge/nonce", + "value": "tampered-challenge-nonce-000000000000000000000" + } + ] + }, + { + "id": "expired-challenge", + "now": "2030-01-01T00:11:00Z", + "expected_valid": false, + "mutations": [] + } + ], + "evidence_boundary": "This file defines project challenge inputs and expected validity. Matching it is black-box interoperability evidence, not certification or proof of implementation independence." +} diff --git a/conformance/verifier-challenge.json b/conformance/verifier-challenge.json index af5dea1..89b1dab 100644 --- a/conformance/verifier-challenge.json +++ b/conformance/verifier-challenge.json @@ -4,6 +4,7 @@ "canonical_repository": "https://github.com/aruintelligence/aml-core", "harness": "scripts/run-verifier-conformance.mjs", "witness_vector": "independent/python/witness-vector.json", + "cases_file": "conformance/verifier-challenge-cases.json", "command_contract": { "invocation": " --now ", "stdout": "single JSON object", @@ -11,32 +12,6 @@ "success_exit_code_for_valid_bundle": 0, "nonzero_exit_code_for_invalid_bundle": true }, - "cases": [ - { - "id": "golden-valid", - "expected_valid": true, - "now": "2030-01-01T00:05:00Z", - "mutation": null - }, - { - "id": "tampered-purpose", - "expected_valid": false, - "now": "2030-01-01T00:05:00Z", - "mutation": "evidence.receipt.decisions[0].purpose is changed" - }, - { - "id": "tampered-challenge", - "expected_valid": false, - "now": "2030-01-01T00:05:00Z", - "mutation": "challenge.nonce is changed" - }, - { - "id": "expired-challenge", - "expected_valid": false, - "now": "2030-01-01T00:11:00Z", - "mutation": null - } - ], "independence": { "required_for_external_witness_credit": true, "must_be_maintained_outside_canonical_repository": true, diff --git a/scripts/build-external-verifier-kit.mjs b/scripts/build-external-verifier-kit.mjs index 2fe4c06..b848d75 100644 --- a/scripts/build-external-verifier-kit.mjs +++ b/scripts/build-external-verifier-kit.mjs @@ -7,6 +7,7 @@ import { execFileSync } from 'node:child_process'; export const KIT_FILES = [ 'conformance/verifier-challenge.json', + 'conformance/verifier-challenge-cases.json', 'conformance/witness-record.example.json', 'independent/python/witness-vector.json', 'protocol/sorted-json-v1.md', @@ -69,7 +70,7 @@ export function buildExternalVerifierKit(outputDir = 'dist/external-verifier-kit } entries.sort((a, b) => codeUnitCompare(a.path, b.path)); - const readme = `# ĀML External Verifier Kit\n\nThis artifact is intentionally **reference-code-free**. It contains public protocol text, JSON Schemas, canonicalization/test vectors, the black-box verifier challenge, one JSON witness fixture, and witness-submission material. It does not contain the JavaScript, Python, Go, or other reference verifier implementations from aml-core.\n\nImplement the published contract in your own runtime, then run the External Verifier Challenge from your own repository. PASS, FAIL, and MIXED results are all useful.\n\nCommand contract:\n\n\`\`\`text\n --now \n\`\`\`\n\nA valid bundle must emit JSON with \`valid: true\` and exit 0. Invalid challenge cases must be rejected with a nonzero exit.\n\nThis kit reduces accidental dependence on reference implementation code. Possessing or using the kit does not itself prove an implementation is independent.\n`; + const readme = `# ĀML External Verifier Kit\n\nThis artifact is intentionally **reference-code-free**. It contains public protocol text, JSON Schemas, canonicalization/test vectors, the black-box verifier challenge, its language-neutral case corpus, one JSON witness fixture, and witness-submission material. It does not contain the JavaScript, Python, Go, or other reference verifier implementations from aml-core.\n\nImplement the published contract in your own runtime, then run the External Verifier Challenge from your own repository. PASS, FAIL, and MIXED results are all useful.\n\nThe challenge cases are data-defined in \`conformance/verifier-challenge-cases.json\` using RFC 6901 JSON Pointer replacement operations.\n\nCommand contract:\n\n\`\`\`text\n --now \n\`\`\`\n\nA valid bundle must emit JSON with \`valid: true\` and exit 0. Invalid challenge cases must be rejected with a nonzero exit.\n\nThis kit reduces accidental dependence on reference implementation code. Possessing or using the kit does not itself prove an implementation is independent.\n`; const readmeBytes = Buffer.from(readme, 'utf8'); fs.writeFileSync(path.join(outputDir, 'README.md'), readmeBytes); entries.push({ path: 'README.md', bytes: readmeBytes.length, sha256: sha256(readmeBytes) }); diff --git a/scripts/run-verifier-conformance.mjs b/scripts/run-verifier-conformance.mjs index 48e3f8b..2ff3fe8 100644 --- a/scripts/run-verifier-conformance.mjs +++ b/scripts/run-verifier-conformance.mjs @@ -15,7 +15,9 @@ const command = process.argv[split + 1]; const baseArgs = process.argv.slice(split + 2); const scriptDir = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.resolve(scriptDir, '..'); -const vectorPath = path.join(repoRoot, 'independent/python/witness-vector.json'); +const challenge = JSON.parse(fs.readFileSync(path.join(repoRoot, 'conformance/verifier-challenge.json'), 'utf8')); +const casesContract = JSON.parse(fs.readFileSync(path.join(repoRoot, challenge.cases_file), 'utf8')); +const vectorPath = path.join(repoRoot, casesContract.bundle_source); const source = JSON.parse(fs.readFileSync(vectorPath, 'utf8')); const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'aml-verifier-conformance-')); @@ -41,39 +43,40 @@ function write(name, value) { return target; } -const purposeTamper = structuredClone(source); -purposeTamper.evidence.receipt.decisions[0].purpose = 'tampered-by-conformance-harness'; - -const challengeTamper = structuredClone(source); -challengeTamper.challenge.nonce = 'tampered-challenge-nonce-000000000000000000000'; +function decodePointerToken(token) { + return token.replace(/~1/g, '/').replace(/~0/g, '~'); +} -const cases = [ - { - id: 'golden-valid', - expected: true, - run: () => invoke(vectorPath, '2030-01-01T00:05:00Z') - }, - { - id: 'tampered-purpose', - expected: false, - run: () => invoke(write('tampered-purpose', purposeTamper), '2030-01-01T00:05:00Z') - }, - { - id: 'tampered-challenge', - expected: false, - run: () => invoke(write('tampered-challenge', challengeTamper), '2030-01-01T00:05:00Z') - }, - { - id: 'expired-challenge', - expected: false, - run: () => invoke(vectorPath, '2030-01-01T00:11:00Z') +function applyReplace(root, mutation) { + if (!mutation || mutation.op !== 'replace' || typeof mutation.path !== 'string' || !mutation.path.startsWith('/')) { + throw new Error('Unsupported verifier challenge mutation'); } -]; + const tokens = mutation.path.slice(1).split('/').map(decodePointerToken); + let parent = root; + for (const token of tokens.slice(0, -1)) { + if (parent === null || typeof parent !== 'object' || !(token in parent)) throw new Error(`Mutation path does not exist: ${mutation.path}`); + parent = parent[token]; + } + const leaf = tokens.at(-1); + if (parent === null || typeof parent !== 'object' || !(leaf in parent)) throw new Error(`Mutation path does not exist: ${mutation.path}`); + parent[leaf] = structuredClone(mutation.value); +} + +function materializeCase(testCase) { + const bundle = structuredClone(source); + for (const mutation of testCase.mutations || []) applyReplace(bundle, mutation); + if (!(testCase.mutations || []).length) return vectorPath; + return write(testCase.id, bundle); +} + +if (casesContract.schema !== 'aml-verifier-challenge-cases/1') throw new Error('Unsupported verifier challenge case schema'); +if (casesContract.mutation_language?.schema !== 'aml-json-pointer-replace/1') throw new Error('Unsupported verifier challenge mutation language'); +if (!Array.isArray(casesContract.cases) || !casesContract.cases.length) throw new Error('Verifier challenge requires cases'); -const results = cases.map(test => { - const observed = test.run(); - const passed = observed.valid === test.expected && (test.expected ? observed.exit_code === 0 : observed.exit_code !== 0); - return { id: test.id, expected_valid: test.expected, passed, observed }; +const results = casesContract.cases.map(testCase => { + const observed = invoke(materializeCase(testCase), testCase.now); + const passed = observed.valid === testCase.expected_valid && (testCase.expected_valid ? observed.exit_code === 0 : observed.exit_code !== 0); + return { id: testCase.id, expected_valid: testCase.expected_valid, passed, observed }; }); const passed = results.every(r => r.passed); @@ -81,6 +84,7 @@ console.log(JSON.stringify({ schema: 'aml-verifier-conformance-result/1', prototype: true, harness_root: repoRoot, + challenge_cases: challenge.cases_file, command: [command, ...baseArgs], passed, results, diff --git a/test/external-verifier-action.test.js b/test/external-verifier-action.test.js index 8dba784..5aab1de 100644 --- a/test/external-verifier-action.test.js +++ b/test/external-verifier-action.test.js @@ -3,13 +3,18 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; const challenge = JSON.parse(fs.readFileSync('conformance/verifier-challenge.json', 'utf8')); +const cases = JSON.parse(fs.readFileSync('conformance/verifier-challenge-cases.json', 'utf8')); const harness = fs.readFileSync('scripts/run-verifier-conformance.mjs', 'utf8'); const action = fs.readFileSync('actions/verifier-conformance/action.yml', 'utf8'); -test('external verifier challenge publishes the exact harness cases', () => { +test('external verifier challenge publishes a language-neutral exact case corpus', () => { assert.equal(challenge.schema, 'aml-external-verifier-challenge/1'); + assert.equal(challenge.cases_file, 'conformance/verifier-challenge-cases.json'); + assert.equal(cases.schema, 'aml-verifier-challenge-cases/1'); + assert.equal(cases.bundle_source, challenge.witness_vector); + assert.equal(cases.mutation_language.schema, 'aml-json-pointer-replace/1'); assert.deepEqual( - challenge.cases.map((entry) => [entry.id, entry.expected_valid]), + cases.cases.map((entry) => [entry.id, entry.expected_valid]), [ ['golden-valid', true], ['tampered-purpose', false], @@ -17,9 +22,24 @@ test('external verifier challenge publishes the exact harness cases', () => { ['expired-challenge', false] ] ); - for (const entry of challenge.cases) { - assert.match(harness, new RegExp(`id: ['\"]${entry.id}['\"]`)); - } + assert.deepEqual(cases.cases[1].mutations, [{ + op: 'replace', + path: '/evidence/receipt/decisions/0/purpose', + value: 'tampered-by-conformance-harness' + }]); + assert.deepEqual(cases.cases[2].mutations, [{ + op: 'replace', + path: '/challenge/nonce', + value: 'tampered-challenge-nonce-000000000000000000000' + }]); +}); + +test('external verifier harness consumes the case corpus instead of hard-coding cases', () => { + assert.match(harness, /challenge\.cases_file/); + assert.match(harness, /casesContract\.cases\.map/); + assert.match(harness, /aml-json-pointer-replace\/1/); + assert.doesNotMatch(harness, /purposeTamper/); + assert.doesNotMatch(harness, /challengeTamper/); }); test('external verifier action drives the canonical black-box harness', () => {