From 48bd5a897fde019ca8910fa968fac2b2776d23a1 Mon Sep 17 00:00:00 2001 From: MongLong0214 Date: Tue, 11 Aug 2026 17:24:02 +0900 Subject: [PATCH] Enumerate CDEB candidates without selecting them The benchmark cannot fill a short corpus by lowering the bar or manufacturing records. It needs an auditable answer about the records currently available instead: every record is retained in deterministic YAML, cutoff and provenance facts are recorded as rejections, and task-qualification calls that history cannot settle stay explicitly undecided. The enumerator calls the compiled shipping query engine rather than parsing history itself. It can therefore report the same records the product sees while separately applying the frozen snapshot reachability test, the product-repo exclusion, reconstructed provenance, and an explicit fork-author authorization boundary. The schema now permits rejected and undecided rows rather than silently encoding them as passing booleans. Limit: wrong-path viability, deterministic oracle feasibility, code disclosure, bounded implementation, and unproven ordinary or benchmark authorship cannot be decided from history and remain undecided for human review Blast: module Undo: easy Certainty: firm Verified: npm ci; package and bench typechecks; seven focused candidate-registry tests; two builds with no dist diff; CDEB and benchmark verifiers; candidate schema validation; local history census Record-Id: r-cdeb10reg --- bench/cdeb/freeze/candidate-registry.ts | 512 +++++++++++++++++++++++ bench/cdeb/schemas/candidate.schema.json | 19 +- package.json | 3 +- test/cdeb-candidate-registry.test.ts | 168 ++++++++ 4 files changed, 693 insertions(+), 9 deletions(-) create mode 100644 bench/cdeb/freeze/candidate-registry.ts create mode 100644 test/cdeb-candidate-registry.test.ts diff --git a/bench/cdeb/freeze/candidate-registry.ts b/bench/cdeb/freeze/candidate-registry.ts new file mode 100644 index 00000000..5287adf7 --- /dev/null +++ b/bench/cdeb/freeze/candidate-registry.ts @@ -0,0 +1,512 @@ +/** + * CDEB-10 candidate enumeration (PRD §3.1–§3.2). + * + * This is deliberately an enumerator, not a corpus selector. It reads records + * through `runQuery`, the same query path the product ships, applies only + * mechanical facts, and leaves the task-qualification calls as `undecided`. + * A later freeze may consume this registry, but must not promote an undecided + * entry merely because the quota is short. + */ + +import { createHash } from "node:crypto"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { dump } from "js-yaml"; + +// This is the compiled shipping surface. The registry does not parse git +// trailers itself: it asks exactly the query engine the product delivers. +import { execGit } from "../../../dist/core/git.js"; +import { + RULED_OUT_KEY, + runQuery, + valuesOf, + type GradedRecord, +} from "../../../dist/core/query.js"; +import { splitRuledOut } from "../../../dist/core/trailers.js"; + +export type EligibilityValue = true | false | "undecided"; + +export interface CandidateEligibility { + readonly explicit_rejection_reason: EligibilityValue; + readonly wrong_path_functionally_viable: EligibilityValue; + readonly deterministic_oracle_possible: EligibilityValue; + readonly current_code_does_not_reveal_reason: EligibilityValue; + readonly bounded_implementation: EligibilityValue; +} + +/** One §3.2 registry row. The key order is also the canonical YAML order. */ +export interface CandidateRegistryEntry { + readonly schema_version: 1; + readonly benchmark: "cdeb-v1"; + readonly candidate_id: string; + readonly repository_id: string; + /** A real Record-Id, or a transparent source token when the input had none. */ + readonly record_ids: readonly string[]; + readonly decision_source_refs: readonly string[]; + readonly record_commit_or_note_ref: string; + readonly record_created_at: string; + readonly natural_record: EligibilityValue; + readonly benchmark_authored: EligibilityValue; + readonly eligibility: CandidateEligibility; + readonly review_status: "accepted" | "rejected"; + readonly rejection_reason: string | null; +} + +export type RejectionCode = + | "after_snapshot_cutoff" + | "missing_explicit_rejection_reason" + | "invalid_record_identity" + | "synthetic_or_backfilled_record" + | "benchmark_authored_record" + | "commitlore_repository" + | "notes_cutoff_undecidable" + | "source_authorization_unverified" + | "human_review_required"; + +export interface CandidateRegistryCensus { + readonly records_examined: number; + readonly candidates_reported: number; + readonly eligible: number; + readonly rejected: number; + readonly blocked_on_human_review: number; + readonly rejection_reasons: Readonly>; + readonly undecided_fields: Readonly>; +} + +export interface CandidateRegistryResult { + readonly candidates: readonly CandidateRegistryEntry[]; + readonly census: CandidateRegistryCensus; +} + +export interface CandidateRegistryOptions { + /** Repository whose history `runQuery` reads. Defaults to the current directory. */ + readonly cwd?: string; + /** Stable CDEB repository name, not a filesystem path. */ + readonly repositoryId: string; + /** Frozen §6.1 snapshot commit or ref. Every commit record must reach it. */ + readonly snapshotRef: string; + /** + * An explicit assertion for callers that know this is the product repository. + * Omitted means the registry detects the repository id and package name. + */ + readonly commitLoreRepository?: boolean; + /** Record ids a human has already established were created for CDEB. */ + readonly benchmarkAuthoredRecordIds?: readonly string[]; + /** + * The authorization boundary for an upstream fork. When set, a source record + * must have one author in this allowlist. An empty allowlist is intentionally + * not treated as authorization. + */ + readonly authorizedDecisionAuthors?: readonly string[]; + readonly requireAuthorizedDecisionAuthor?: boolean; +} + +type UndecidedField = + | "natural_record" + | "benchmark_authored" + | keyof CandidateEligibility; + +const REJECTION_TEXT: Readonly> = { + after_snapshot_cutoff: "record declaration is after the frozen snapshot cutoff", + missing_explicit_rejection_reason: "record lacks an explicit Ruled-out alternative and rejection reason", + invalid_record_identity: "record has no valid Record-Id", + synthetic_or_backfilled_record: "record provenance is reconstructed, so it is synthetic or backfilled", + benchmark_authored_record: "record is identified as benchmark-authored", + commitlore_repository: "CommitLore repository decisions are excluded from the primary corpus", + notes_cutoff_undecidable: + "notes-only record creation cannot be placed at the commit snapshot without a frozen notes reference", + source_authorization_unverified: + "decision-source author is not covered by the supplied source authorization", + human_review_required: "human review is required for undecided eligibility fields", +}; + +const EMPTY_REJECTION_COUNTS = (): Record => ({ + after_snapshot_cutoff: 0, + missing_explicit_rejection_reason: 0, + invalid_record_identity: 0, + synthetic_or_backfilled_record: 0, + benchmark_authored_record: 0, + commitlore_repository: 0, + notes_cutoff_undecidable: 0, + source_authorization_unverified: 0, + human_review_required: 0, +}); + +const sha256 = (value: string): string => + createHash("sha256").update(value, "utf8").digest("hex"); + +const sourceToken = (record: GradedRecord): string => { + const trailerDigest = sha256( + record.trailers.map((trailer) => `${trailer.key}\u0000${trailer.value}`).join("\u0001"), + ).slice(0, 12); + return `unidentified:${record.source}:${record.sha}:${trailerDigest}`; +}; + +const validRecordId = (value: string | undefined): value is string => + value !== undefined && /^r-[a-z0-9]{6,}$/.test(value); + +const candidateIdFor = (record: GradedRecord): string => + validRecordId(record.recordId) + ? record.recordId + : `record-${record.sha.slice(0, 12)}-${sha256(sourceToken(record)).slice(0, 12)}`; + +const resolveSnapshot = (cwd: string, snapshotRef: string): string => { + const result = execGit( + ["rev-parse", "--verify", "--end-of-options", `${snapshotRef}^{commit}`], + { cwd }, + ); + if (result.code !== 0 || result.stdout.trim() === "") { + throw new Error(`candidate registry: snapshot ref ${JSON.stringify(snapshotRef)} is not a commit`); + } + return result.stdout.trim(); +}; + +const isAncestor = (cwd: string, ancestor: string, descendant: string): boolean => { + const result = execGit(["merge-base", "--is-ancestor", ancestor, descendant], { cwd }); + if (result.code === 0) return true; + if (result.code === 1) return false; + throw new Error( + `candidate registry: could not compare ${ancestor} with snapshot ${descendant}: ${result.stderr.trim()}`, + ); +}; + +const authorOf = (cwd: string, ref: string): string | null => { + const result = execGit(["show", "-s", "--format=%an <%ae>", "--end-of-options", ref], { cwd }); + return result.code === 0 && result.stdout.trim() !== "" ? result.stdout.trim() : null; +}; + +const committedAt = (cwd: string, ref: string): string | null => { + const result = execGit(["show", "-s", "--format=%cI", "--end-of-options", ref], { cwd }); + return result.code === 0 && result.stdout.trim() !== "" ? result.stdout.trim() : null; +}; + +const isCommitLoreRepository = (cwd: string, repositoryId: string, explicit: boolean | undefined): boolean => { + if (explicit !== undefined) return explicit; + if (repositoryId === "commitlore") return true; + const packagePath = resolve(cwd, "package.json"); + if (!existsSync(packagePath)) return false; + try { + const parsed = JSON.parse(readFileSync(packagePath, "utf8")) as { name?: unknown }; + return parsed.name === "commitlore"; + } catch { + return false; + } +}; + +const hasExplicitRejectionReason = (record: GradedRecord): boolean => + valuesOf(record, RULED_OUT_KEY).some((value) => { + const ruledOut = splitRuledOut(value); + return ( + !ruledOut.malformed && + !ruledOut.unterminatedCodeSpan && + ruledOut.alternative !== "" && + ruledOut.reason !== "" + ); + }); + +const recordAuthorsAreAuthorized = ( + cwd: string, + record: GradedRecord, + allowed: ReadonlySet, +): boolean => + record.shas.some((sha) => { + const author = authorOf(cwd, sha); + return author !== null && allowed.has(author); + }); + +const hasUndecidedField = (entry: Pick): boolean => + entry.natural_record === "undecided" || + entry.benchmark_authored === "undecided" || + Object.values(entry.eligibility).some((value) => value === "undecided"); + +const compareCandidate = (left: CandidateRegistryEntry, right: CandidateRegistryEntry): number => { + if (left.candidate_id !== right.candidate_id) { + return left.candidate_id < right.candidate_id ? -1 : 1; + } + const leftSource = left.record_commit_or_note_ref; + const rightSource = right.record_commit_or_note_ref; + return leftSource < rightSource ? -1 : leftSource > rightSource ? 1 : 0; +}; + +interface AssessedCandidate { + readonly entry: CandidateRegistryEntry; + readonly rejectionCodes: readonly RejectionCode[]; +} + +const assessRecord = ( + record: GradedRecord, + cwd: string, + options: Required> & CandidateRegistryOptions, + snapshot: string, + commitLoreRepository: boolean, + benchmarkAuthored: ReadonlySet, + authorizedAuthors: ReadonlySet, +): AssessedCandidate => { + const recordId = record.recordId; + const recordIdIsValid = validRecordId(recordId); + const id = recordIdIsValid ? recordId : sourceToken(record); + const benchmarkAuthoredRecord = recordIdIsValid && benchmarkAuthored.has(recordId); + const reconstructed = record.provenance?.kind === "reconstructed"; + const references = record.shas.length > 0 ? record.shas : [record.sha]; + // A re-declaration after the cutoff does not erase an earlier record with + // the same identity. `runQuery` already blocks divergent identities; here + // we ask the §3.1 question literally: did this record exist by the snapshot? + const referencesAtSnapshot = references.filter((ref) => isAncestor(cwd, ref, snapshot)); + const afterSnapshot = referencesAtSnapshot.length === 0; + const notesOnly = record.sources.length === 1 && record.sources[0] === "notes"; + const recordRef = notesOnly ? record.sha : (referencesAtSnapshot[0] ?? references[0] ?? record.sha); + const requiresAuthorizedAuthor = options.requireAuthorizedDecisionAuthor === true; + const authorizationVerified = + !requiresAuthorizedAuthor || + (authorizedAuthors.size > 0 && recordAuthorsAreAuthorized(cwd, record, authorizedAuthors)); + + const eligibility: CandidateEligibility = { + explicit_rejection_reason: hasExplicitRejectionReason(record), + wrong_path_functionally_viable: "undecided", + deterministic_oracle_possible: "undecided", + current_code_does_not_reveal_reason: "undecided", + bounded_implementation: "undecided", + }; + const natural_record: EligibilityValue = + reconstructed || benchmarkAuthoredRecord ? false : "undecided"; + const benchmark_authored: EligibilityValue = benchmarkAuthoredRecord ? true : "undecided"; + const rejectionCodes: RejectionCode[] = []; + if (afterSnapshot) rejectionCodes.push("after_snapshot_cutoff"); + if (!eligibility.explicit_rejection_reason) rejectionCodes.push("missing_explicit_rejection_reason"); + if (!recordIdIsValid) rejectionCodes.push("invalid_record_identity"); + if (reconstructed) rejectionCodes.push("synthetic_or_backfilled_record"); + if (benchmarkAuthoredRecord) rejectionCodes.push("benchmark_authored_record"); + if (commitLoreRepository) rejectionCodes.push("commitlore_repository"); + if (notesOnly) rejectionCodes.push("notes_cutoff_undecidable"); + if (!authorizationVerified) rejectionCodes.push("source_authorization_unverified"); + + const provisional: Omit = { + schema_version: 1, + benchmark: "cdeb-v1", + candidate_id: candidateIdFor(record), + repository_id: options.repositoryId, + record_ids: [id], + decision_source_refs: references, + record_commit_or_note_ref: notesOnly ? `refs/notes/commitlore:${record.sha}` : recordRef, + record_created_at: recordRef === record.sha ? record.committedAt : (committedAt(cwd, recordRef) ?? record.committedAt), + natural_record, + benchmark_authored, + eligibility, + }; + if (hasUndecidedField(provisional)) rejectionCodes.push("human_review_required"); + + const review_status = rejectionCodes.length === 0 ? "accepted" : "rejected"; + return { + entry: { + ...provisional, + review_status, + rejection_reason: + review_status === "accepted" + ? null + : rejectionCodes.map((code) => REJECTION_TEXT[code]).join("; "), + }, + rejectionCodes, + }; +}; + +/** + * Enumerates every record visible through the product query path. No model, + * ON/OFF run, path heuristic, or task selection is involved. + */ +export const enumerateCandidateRegistry = ( + options: CandidateRegistryOptions, +): CandidateRegistryResult => { + const cwd = options.cwd ?? process.cwd(); + const snapshot = resolveSnapshot(cwd, options.snapshotRef); + // `allHistory` preserves superseded candidates too; §3.4 deliberately has a + // lifecycle category, so the query's normal active-only display would be a + // selection decision hidden in the enumerator. + const queried = runQuery({ + cwd, + allHistory: true, + // A fixed far-future instant keeps future-dated fixture commits visible and + // makes enumeration independent of the wall clock. + at: new Date("9999-12-31T23:59:59.999Z"), + }); + if (queried.history !== "ready") { + throw new Error( + `candidate registry: repository history is ${queried.history}; enumeration cannot treat that as an empty corpus`, + ); + } + + const commitLoreRepository = isCommitLoreRepository(cwd, options.repositoryId, options.commitLoreRepository); + const benchmarkAuthored = new Set(options.benchmarkAuthoredRecordIds ?? []); + const authorizedAuthors = new Set(options.authorizedDecisionAuthors ?? []); + const assessed = queried.records.map((record) => + assessRecord(record, cwd, options, snapshot, commitLoreRepository, benchmarkAuthored, authorizedAuthors), + ); + const candidates = assessed.map(({ entry }) => entry).sort(compareCandidate); + const rejectionReasons = EMPTY_REJECTION_COUNTS(); + const undecidedFields: Record = { + natural_record: 0, + benchmark_authored: 0, + explicit_rejection_reason: 0, + wrong_path_functionally_viable: 0, + deterministic_oracle_possible: 0, + current_code_does_not_reveal_reason: 0, + bounded_implementation: 0, + }; + for (const assessedCandidate of assessed) { + for (const code of assessedCandidate.rejectionCodes) rejectionReasons[code] += 1; + const entry = assessedCandidate.entry; + if (entry.natural_record === "undecided") undecidedFields.natural_record += 1; + if (entry.benchmark_authored === "undecided") undecidedFields.benchmark_authored += 1; + for (const field of Object.keys(entry.eligibility) as (keyof CandidateEligibility)[]) { + if (entry.eligibility[field] === "undecided") undecidedFields[field] += 1; + } + } + + return { + candidates, + census: { + records_examined: queried.records.length, + candidates_reported: candidates.length, + eligible: candidates.filter((candidate) => candidate.review_status === "accepted").length, + rejected: candidates.filter((candidate) => candidate.review_status === "rejected").length, + blocked_on_human_review: candidates.filter(hasUndecidedField).length, + rejection_reasons: rejectionReasons, + undecided_fields: undecidedFields, + }, + }; +}; + +/** Canonical YAML for a registry: a document list of exactly the §3.2 rows. */ +export const serializeCandidateRegistry = (registry: CandidateRegistryResult): string => + dump(registry.candidates, { + noRefs: true, + lineWidth: -1, + sortKeys: false, + }); + +/** Human-readable audit census, deterministic in both field and reason order. */ +export const formatCandidateCensus = (census: CandidateRegistryCensus): string => { + const lines = [ + `records examined: ${String(census.records_examined)}`, + `candidates reported: ${String(census.candidates_reported)}`, + `eligible: ${String(census.eligible)}`, + `rejected: ${String(census.rejected)}`, + `blocked on human review: ${String(census.blocked_on_human_review)}`, + "disqualified:", + ]; + for (const code of Object.keys(EMPTY_REJECTION_COUNTS()) as RejectionCode[]) { + lines.push(` ${code}: ${String(census.rejection_reasons[code])}`); + } + lines.push("undecided:"); + for (const [field, count] of Object.entries(census.undecided_fields)) { + lines.push(` ${field}: ${String(count)}`); + } + return `${lines.join("\n")}\n`; +}; + +interface CliOptions { + cwd?: string; + output?: string; + repositoryId?: string; + snapshotRef?: string; + commitLoreRepository?: boolean; + benchmarkAuthoredRecordIds: string[]; + authorizedDecisionAuthors: string[]; + requireAuthorizedDecisionAuthor?: boolean; +} + +const usage = (): string => + [ + "usage: node --experimental-strip-types bench/cdeb/freeze/candidate-registry.ts \\", + " --repository-id --snapshot-ref [--cwd ] [--output ] \\", + " [--commitlore-repository] [--benchmark-authored-record-id ] \\", + " [--require-authorized-decision-author --authorized-decision-author >]", + ].join("\n"); + +const requiredValue = (argv: readonly string[], index: number, flag: string): string => { + const value = argv[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error(`${flag} requires a value`); + return value; +}; + +const parseCli = (argv: readonly string[]): CliOptions => { + const options: CliOptions = { + benchmarkAuthoredRecordIds: [], + authorizedDecisionAuthors: [], + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case "--cwd": + options.cwd = requiredValue(argv, index, arg); + index += 1; + break; + case "--output": + options.output = requiredValue(argv, index, arg); + index += 1; + break; + case "--repository-id": + options.repositoryId = requiredValue(argv, index, arg); + index += 1; + break; + case "--snapshot-ref": + options.snapshotRef = requiredValue(argv, index, arg); + index += 1; + break; + case "--commitlore-repository": + options.commitLoreRepository = true; + break; + case "--benchmark-authored-record-id": + options.benchmarkAuthoredRecordIds.push(requiredValue(argv, index, arg)); + index += 1; + break; + case "--require-authorized-decision-author": + options.requireAuthorizedDecisionAuthor = true; + break; + case "--authorized-decision-author": + options.authorizedDecisionAuthors.push(requiredValue(argv, index, arg)); + index += 1; + break; + case "--help": + case "-h": + throw new Error(usage()); + default: + throw new Error(`unknown argument ${JSON.stringify(arg)}\n${usage()}`); + } + } + if (options.repositoryId === undefined || options.snapshotRef === undefined) { + throw new Error(`--repository-id and --snapshot-ref are required\n${usage()}`); + } + return options; +}; + +const isMain = (): boolean => + process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url); + +if (isMain()) { + try { + const options = parseCli(process.argv.slice(2)); + const registryOptions: CandidateRegistryOptions = { + repositoryId: options.repositoryId!, + snapshotRef: options.snapshotRef!, + benchmarkAuthoredRecordIds: options.benchmarkAuthoredRecordIds, + authorizedDecisionAuthors: options.authorizedDecisionAuthors, + ...(options.cwd === undefined ? {} : { cwd: options.cwd }), + ...(options.commitLoreRepository === undefined + ? {} + : { commitLoreRepository: options.commitLoreRepository }), + ...(options.requireAuthorizedDecisionAuthor === undefined + ? {} + : { requireAuthorizedDecisionAuthor: options.requireAuthorizedDecisionAuthor }), + }; + const registry = enumerateCandidateRegistry(registryOptions); + const yaml = serializeCandidateRegistry(registry); + if (options.output === undefined) process.stdout.write(yaml); + else writeFileSync(options.output, yaml, "utf8"); + process.stderr.write(formatCandidateCensus(registry.census)); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 2; + } +} diff --git a/bench/cdeb/schemas/candidate.schema.json b/bench/cdeb/schemas/candidate.schema.json index 8d181370..fbf31f2a 100644 --- a/bench/cdeb/schemas/candidate.schema.json +++ b/bench/cdeb/schemas/candidate.schema.json @@ -15,25 +15,28 @@ "benchmark": { "const": "cdeb-v1" }, "candidate_id": { "type": "string", "pattern": "^[a-z0-9-]+$" }, "repository_id": { "type": "string", "minLength": 1 }, - "record_ids": { "type": "array", "minItems": 1, "items": { "type": "string", "pattern": "^r-[a-z0-9]{6,}$" } }, + "record_ids": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }, "decision_source_refs": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }, "record_commit_or_note_ref": { "type": "string", "minLength": 1 }, "record_created_at": { "type": "string", "format": "date-time" }, - "natural_record": { "const": true }, - "benchmark_authored": { "const": false }, + "natural_record": { "$ref": "#/$defs/eligibilityValue" }, + "benchmark_authored": { "$ref": "#/$defs/eligibilityValue" }, "eligibility": { "type": "object", "additionalProperties": false, "required": ["explicit_rejection_reason", "wrong_path_functionally_viable", "deterministic_oracle_possible", "current_code_does_not_reveal_reason", "bounded_implementation"], "properties": { - "explicit_rejection_reason": { "type": "boolean" }, - "wrong_path_functionally_viable": { "type": "boolean" }, - "deterministic_oracle_possible": { "type": "boolean" }, - "current_code_does_not_reveal_reason": { "type": "boolean" }, - "bounded_implementation": { "type": "boolean" } + "explicit_rejection_reason": { "$ref": "#/$defs/eligibilityValue" }, + "wrong_path_functionally_viable": { "$ref": "#/$defs/eligibilityValue" }, + "deterministic_oracle_possible": { "$ref": "#/$defs/eligibilityValue" }, + "current_code_does_not_reveal_reason": { "$ref": "#/$defs/eligibilityValue" }, + "bounded_implementation": { "$ref": "#/$defs/eligibilityValue" } } }, "review_status": { "enum": ["accepted", "rejected"] }, "rejection_reason": { "type": ["string", "null"] } + }, + "$defs": { + "eligibilityValue": { "enum": [true, false, "undecided"] } } } diff --git a/package.json b/package.json index 0233fae2..f38d5f1c 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ "bundle": "esbuild src/cli.ts --bundle --platform=node --format=esm --target=node22 --outfile=dist/commitlore.mjs", "bench:m5": "node --experimental-strip-types bench/m5-analysis.ts", "bench:cdeb:verify": "node bench/cdeb/verify.mjs", - "bench:cdeb:analyze": "node --experimental-strip-types bench/cdeb/analyze.ts" + "bench:cdeb:analyze": "node --experimental-strip-types bench/cdeb/analyze.ts", + "bench:cdeb:registry": "node --experimental-strip-types bench/cdeb/freeze/candidate-registry.ts" }, "keywords": [ "git", diff --git a/test/cdeb-candidate-registry.test.ts b/test/cdeb-candidate-registry.test.ts new file mode 100644 index 00000000..1b1801a3 --- /dev/null +++ b/test/cdeb-candidate-registry.test.ts @@ -0,0 +1,168 @@ +/** CDEB-10 candidate enumeration: only mechanical facts may reject a record. */ + +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterAll, describe, expect, it } from "vitest"; + +import { + enumerateCandidateRegistry, + serializeCandidateRegistry, +} from "../bench/cdeb/freeze/candidate-registry.ts"; +import { gitOrThrow } from "../bench/git.ts"; +import { createTestRepo } from "./git-fixtures.js"; + +const scratch: string[] = []; + +afterAll(() => { + for (const path of scratch) rmSync(path, { recursive: true, force: true }); +}); + +const repo = (label: string): string => { + const path = createTestRepo({ path: mkdtempSync(join(tmpdir(), `cdeb-registry-${label}-`)) }); + scratch.push(path); + return path; +}; + +const commit = (cwd: string, serial: number, message: string): string => { + writeFileSync(join(cwd, "decision.ts"), `export const revision = ${String(serial)};\n`); + gitOrThrow(cwd, ["add", "decision.ts"]); + gitOrThrow(cwd, ["commit", "--quiet", "-m", message]); + return gitOrThrow(cwd, ["rev-parse", "HEAD"]).trim(); +}; + +const ruledOutRecord = (id: string, ruledOut: string): string => + [ + "decision: retain the portable path", + "", + `Ruled-out: ${ruledOut}`, + `Record-Id: ${id}`, + "Provenance: authored", + ].join("\n"); + +describe("CDEB-10 candidate registry", () => { + it("keeps a post-cutoff record and names the cutoff rejection", () => { + const cwd = repo("post-cutoff"); + commit(cwd, 1, "base"); + const snapshot = gitOrThrow(cwd, ["rev-parse", "HEAD"]).trim(); + commit(cwd, 2, ruledOutRecord("r-postcut1", "global cache | it leaks state across tenants")); + + const registry = enumerateCandidateRegistry({ cwd, repositoryId: "repo-a", snapshotRef: snapshot }); + const candidate = registry.candidates.find((entry) => entry.record_ids.includes("r-postcut1")); + + expect(candidate?.review_status).toBe("rejected"); + expect(candidate?.schema_version).toBe(1); + expect(candidate?.benchmark).toBe("cdeb-v1"); + expect(candidate?.rejection_reason).toContain("after the frozen snapshot cutoff"); + expect(registry.census.rejection_reasons.after_snapshot_cutoff).toBe(1); + }); + + it("rejects a record without an explicit rejection reason and retains it", () => { + const cwd = repo("missing-reason"); + const snapshot = commit(cwd, 1, ruledOutRecord("r-noreason1", "global cache")); + + const registry = enumerateCandidateRegistry({ cwd, repositoryId: "repo-a", snapshotRef: snapshot }); + const candidate = registry.candidates.find((entry) => entry.record_ids.includes("r-noreason1")); + + expect(candidate).toMatchObject({ + eligibility: { explicit_rejection_reason: false }, + review_status: "rejected", + }); + expect(candidate?.rejection_reason).toContain("lacks an explicit Ruled-out alternative and rejection reason"); + expect(registry.census.rejection_reasons.missing_explicit_rejection_reason).toBe(1); + expect(registry.census.candidates_reported).toBe(1); + }); + + it("produces byte-identical YAML for unchanged history", () => { + const cwd = repo("deterministic"); + const snapshot = commit( + cwd, + 1, + ruledOutRecord("r-stable01", "global cache | it leaks state across tenants"), + ); + const options = { cwd, repositoryId: "repo-a", snapshotRef: snapshot }; + + expect(serializeCandidateRegistry(enumerateCandidateRegistry(options))).toBe( + serializeCandidateRegistry(enumerateCandidateRegistry(options)), + ); + }); + + it("never counts human-only fields as eligible", () => { + const cwd = repo("undecided"); + const snapshot = commit( + cwd, + 1, + ruledOutRecord("r-undecid", "global cache | it leaks state across tenants"), + ); + + const registry = enumerateCandidateRegistry({ cwd, repositoryId: "repo-a", snapshotRef: snapshot }); + expect(registry.candidates[0]?.eligibility.wrong_path_functionally_viable).toBe("undecided"); + expect(registry.candidates[0]?.review_status).toBe("rejected"); + expect(registry.census.eligible).toBe(0); + expect(registry.census.blocked_on_human_review).toBe(1); + }); + + it("rejects reconstructed records as synthetic or backfilled", () => { + const cwd = repo("reconstructed"); + const snapshot = commit( + cwd, + 1, + [ + "decision: preserve the original rationale", + "", + "Ruled-out: global cache | it leaks state across tenants", + "Record-Id: r-backfill1", + "Provenance: reconstructed", + ].join("\n"), + ); + + const registry = enumerateCandidateRegistry({ cwd, repositoryId: "repo-a", snapshotRef: snapshot }); + const candidate = registry.candidates[0]; + + expect(candidate?.natural_record).toBe(false); + expect(candidate?.rejection_reason).toContain("synthetic or backfilled"); + expect(registry.census.rejection_reasons.synthetic_or_backfilled_record).toBe(1); + }); + + it("refuses a fork source whose decision author is not authorized", () => { + const cwd = repo("fork-author"); + const snapshot = commit( + cwd, + 1, + ruledOutRecord("r-forkauth", "global cache | it leaks state across tenants"), + ); + + const registry = enumerateCandidateRegistry({ + cwd, + repositoryId: "forked-repo", + snapshotRef: snapshot, + requireAuthorizedDecisionAuthor: true, + authorizedDecisionAuthors: ["Owner "], + }); + + expect(registry.candidates[0]?.review_status).toBe("rejected"); + expect(registry.candidates[0]?.rejection_reason).toContain("not covered by the supplied source authorization"); + expect(registry.census.rejection_reasons.source_authorization_unverified).toBe(1); + }); + + it("excludes CommitLore repository decisions regardless of record content", () => { + const cwd = repo("commitlore"); + const snapshot = commit( + cwd, + 1, + ruledOutRecord("r-product01", "global cache | it leaks state across tenants"), + ); + + const registry = enumerateCandidateRegistry({ + cwd, + repositoryId: "commitlore", + snapshotRef: snapshot, + }); + const candidate = registry.candidates[0]; + + expect(candidate?.review_status).toBe("rejected"); + expect(candidate?.rejection_reason).toContain("CommitLore repository decisions are excluded"); + expect(registry.census.rejection_reasons.commitlore_repository).toBe(1); + }); +});