From 48bf1deab14d25357ccbfc9dd14d5be6ebc6b05b Mon Sep 17 00:00:00 2001 From: Copilot Date: Fri, 18 Sep 2026 20:24:38 +0200 Subject: [PATCH 01/99] Add regression investigation contracts and endpoint admission Define versioned evidence and investigation-state contracts, a pinned execution policy, and a fail-closed bisection admission gate with dependency-free tests. No workflow or reproduction execution is enabled. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/policies/regression-bisect.json | 39 ++ eng/regression-bisect/README.md | 162 +++++++ eng/regression-bisect/fixtures/contracts.cjs | 115 +++++ eng/regression-bisect/model.cjs | 429 +++++++++++++++++++ eng/regression-bisect/model.test.cjs | 388 +++++++++++++++++ 5 files changed, 1133 insertions(+) create mode 100644 .github/policies/regression-bisect.json create mode 100644 eng/regression-bisect/README.md create mode 100644 eng/regression-bisect/fixtures/contracts.cjs create mode 100644 eng/regression-bisect/model.cjs create mode 100644 eng/regression-bisect/model.test.cjs diff --git a/.github/policies/regression-bisect.json b/.github/policies/regression-bisect.json new file mode 100644 index 00000000000..7ad53c8d6c2 --- /dev/null +++ b/.github/policies/regression-bisect.json @@ -0,0 +1,39 @@ +{ + "schemaVersion": 1, + "repository": "dotnet/fsharp", + "selection": { + "state": "open", + "label": "Regression", + "createdAtOrAfter": "2026-09-01T00:00:00Z" + }, + "authorization": { + "mode": "reviewed-manifests-only", + "policySource": "pinned-trusted-default-branch", + "approvalDirectory": ".github/regression-bisect/approved", + "assets": "reviewed-repository-paths-and-sha256" + }, + "accounting": { + "scope": "issue-allowance", + "reserveBeforeExecution": true, + "retriesCharged": true, + "cancellationRefunds": false, + "inputEditsResetAllowance": false, + "labelChangesResetAllowance": false, + "dispatchResetsAllowance": false, + "newAllowanceRequires": "maintainer-authorization" + }, + "limits": { + "sourceBuilds": 24, + "workerUnits": 40, + "reservedMinutes": 720, + "attemptsPerItem": 2, + "assertionRepetitions": 2, + "buildTimeoutMinutes": 90, + "reproductionTimeoutSeconds": 300, + "workerTimeoutMinutes": 110, + "discoveryIssues": 3, + "discoveryPages": 2, + "discoveryPageSize": 100 + }, + "evidenceBytesPerStream": 16384 +} diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md new file mode 100644 index 00000000000..dfa60c3e339 --- /dev/null +++ b/eng/regression-bisect/README.md @@ -0,0 +1,162 @@ +# Regression investigation contracts + +This directory defines version 1 of the deterministic input, evidence, state, +and admission contracts. It does not activate a workflow, execute reproductions, +persist state, or spawn subprocesses. Tests use synthetic records under +`fixtures`; they are not reproduction approvals. + +Run the dependency-free suite from the repository root: + +```powershell +node --test eng\regression-bisect\model.test.cjs +``` + +## Trust boundary + +The exact `Regression` label selects an **open issue, not a pull request**, created +at or after `2026-09-01T00:00:00Z`. Updates and label toggles do not change that +cutoff or authorize execution. + +The host must load this module and `.github/policies/regression-bisect.json` from +a pinned, trusted default-branch revision, not from the source revision under +test. `validatePolicy` accepts only that policy, including its finite limits; an +issue, dispatch input, or agent cannot override it. + +The host must construct `budget.authorization` from the reviewed approval at +`.github/regression-bisect/approved/.json` at `budget.policyRevision`. +Its `manifestDigest` covers the entire manifest; `inputDigest` must equal the +manifest's `approvedInputDigest`. Computing or supplying a digest is **not** +approval. The approval path, repository, issue number, and trusted revision must +also match. + +Likewise, `budget.authenticatedObservationDigests` comes exclusively from the +workflow's authenticated worker records. It is not an agent-supplied allowlist. +`verified`, provenance strings, and hashes in arbitrary JSON do not authenticate +anything. These pure validators check structure and consistency; the host is +responsible for authenticating the records before supplying this trusted +context. Source checkout and untrusted reproduction execution must not be able +to replace that context. + +## JSON contracts and exports + +All records use `schemaVersion: 1`. Validators return `true` or throw a +path-qualified error. Unknown or missing fields, unsupported versions, and +malformed identities are rejected. `classifyAttempts` instead maps incomplete +attempt evidence to an explicit `untestable` outcome; `admitBisection` returns +`{ decision, reason }`, with `decision` exactly `admit`, `stop`, or `blocked`. + +`fixtures/contracts.cjs` provides executable examples of every record shape. + +| Export | Contract | +| --- | --- | +| `isEligibleIssue(issue)` | Boolean selection using the issues API's `state`, `created_at`, `labels`, and absence of `pull_request`. Dates must have an explicit timezone and valid calendar components. | +| `validatePolicy(policy)` | Exact agreement with the trusted policy, including accounting rules and finite positive integer limits. | +| `validateManifest(manifest)` | Frozen repository/issue, discussion fingerprint, reproduction kind, reviewed source hashes, exact positive/failure assertions, matrix, timeout, pinned producer/packages, search hints, and approved-input digest. | +| `fingerprintInput(json)` | SHA-256 of canonical JSON: sorted object keys, preserved array order and string bytes. Rejects non-JSON values. It never removes comments based on markers, authors, or claimed authentication. | +| `classifyAttempts(attempts, manifest)` | Checks both required repetitions against the frozen assertions and all setup/identity checks. Returns a versioned outcome with an explicit reason. | +| `validateOutcome(outcome)` | Only `pass`, `product-failure`, or `untestable`, with a reason appropriate to that outcome. | +| `validateObservation(observation, manifest, options?)` | Validates target, all reproduction fingerprints, matrix evidence, local artifacts, provenance, and recomputed outcomes. | +| `fingerprintEpoch(manifest, budget, pinnedMainSha)` | Binds the manifest, pinned main, resolved stable release, and policy/workflow revisions. | +| `validateState(state)` | Validates the shared persisted-state shape, epoch, authenticated evidence references, cumulative budget, and any charged pending reservation. | +| `admitBisection({ manifest, stable, main, good, ancestry, budget })` | Fail-closed gate for a verified source interval; no execution side effects. | + +### Frozen reproduction + +Source entries are `{ path, sha256 }`, hashing the **raw source bytes**. Serialized +repository paths use `/` regardless of worker OS. Absolute paths, traversal, +encoded paths, Windows device names/streams, and URLs are forbidden. The asset +loader must also verify tracked text blobs and hashes and reject symlinks or +resolved paths outside its isolated checkout before execution. + +Each matrix cell has an ID, OS (`linux`, `windows`, `macos`), architecture (`x64`, +`arm64`), project or `null`, TFM, configuration, language version, and command. +Only `["fsi", ""]` and +`["run-project", ""]` are allowed, matching the manifest +kind. These are operation descriptors, not shell command lines. The engine +expands them using trusted executable paths and the cell's frozen settings, +never by shell interpolation. HTTPS search-hint URLs are discovery hints only, +not executable assets or approval sources. + +Assertions are exact `{ exitCode, stdout, stderr }` triples. Both streams are +bounded by the policy and compared without trimming or diagnostic normalization. +A nonzero exit, unrelated exception, truncated output, or missing evidence is +not proof of the frozen product failure. Required restore/build/setup success is +separate from the reproduction assertion: an expected compiler rejection can +satisfy the failure assertion only after the compiler itself was built and +verified successfully. + +`approvedInputDigest` hashes the manifest with only that field omitted. It does +not include the moving main SHA. The discussion fingerprint must cover the +complete issue and all reporter/maintainer comments, corrections, edits, and +edit timestamps. The collector must supply the complete snapshot; hashing +cannot detect omitted input. This module excludes **no** discussion records. +If the collector excludes workflow publications, it must authenticate record +IDs and content against its own publication provenance; a lookalike marker or +bot display name is insufficient. + +### Evidence and reuse + +A source observation targets `{ sourceSha }`; a released observation targets +`{ sdk }`. The stable SDK identity binds a non-preview version, verified latest +release metadata at an official release-index URL, the metadata digest, and +per-OS/architecture archive hashes. An installed SDK version alone is not a +stable release resolution. Historical source builds may use a pinned prerelease +build SDK without treating it as a stable release. + +Every matrix cell records its command, OS/architecture and runner-image digest, +verified SDK/runtime versions and hashes, producer identity, compiler/tasks/ +targets/Core artifact identities, both attempts, and derived outcome. Local +artifacts must name the tested source SHA and observation's build-instance ID. +Released artifacts cannot satisfy a source identity. An unavailable identity +can be `null` only in untestable evidence with the corresponding checks failed. + +The observation binds the complete manifest, approved input, discussion, matrix, +producer, and package fingerprints. Admission also compares runner environments +across all endpoints and SDK/runtime identities across the two source endpoints. +Cached source observations are reusable only with these same inputs and trusted +provenance. Different source revisions naturally have different artifact hashes. + +Final confirmation calls must supply +`{ finalConfirmation: { runId, buildInstanceId } }` from the new confirmation +reservation. Cached evidence, a different run, or a different build instance is +rejected even when all reproduction fingerprints match. Omit options for +ordinary endpoint validation. + +### State, budgets, and admission + +State contains the issue/approval fingerprints, manifest, pinned main, epoch, +phase, endpoint evidence digests, tested source revisions, skips, current +good/bad interval and Git log, and publication digest/comment ID. Its `budget` +contains policy/workflow identities, the resolved stable release, authorization, +allowance ID, authenticated observation digests, cumulative consumed counters, +per-work-item attempt counts, next work-item ID, and nullable `pendingWork`. + +The host must durably and atomically charge a reservation **before execution**. +Counters include pending and completed work, failed attempts, and cancelled +work; clearing a pending reservation is not a refund. Retries increment the +attempt count and cost another reservation. Cancellation, redispatch, new runs, +input edits, main advances, and label toggles must retain the same issue +allowance and counters. Only a deliberate maintainer-authorized new +investigation can create a new allowance. Shape validation is not storage, +locking, transition authorization, or recovery. + +Admission accepts an already charged reservation for the same allowance/item/ +attempt, or checks space for the next source build, worker unit, and full +110-minute worker reservation. This conservative reservation includes the +90-minute build timeout and two reproductions capped at five minutes each. +An already charged last reservation is usable even at the cumulative limit; +an exhausted allowance cannot start another item. + +The only admitted endpoint combination is stable `product-failure`, pinned-main +`product-failure`, and older source `pass`, consistently across every repetition +and matrix cell. `ancestry` must bind `goodSha`, `badSha`, `pinnedMainSha`, and +verified `isAncestor: true`, with distinct good/bad SHAs. The host must obtain +this relationship from the pinned source graph, never issue claims or timestamps. +Stable pass stops investigation; main pass stops as release-specific/already +fixed; untestable evidence blocks it. + +Only the orchestration engine may eventually spawn `git bisect`. It must call +this gate immediately before the first operation and again after any +evidence-invalidating change, using the current authenticated context and +charged reservation. This contract suite does not claim to prove an engine's +subprocess behavior; there is no engine or subprocess execution here. diff --git a/eng/regression-bisect/fixtures/contracts.cjs b/eng/regression-bisect/fixtures/contracts.cjs new file mode 100644 index 00000000000..9999e141488 --- /dev/null +++ b/eng/regression-bisect/fixtures/contracts.cjs @@ -0,0 +1,115 @@ +const { createHash } = require("node:crypto"); +const policy = () => structuredClone(require("../../../.github/policies/regression-bisect.json")); +const digest = text => createHash("sha256").update(text).digest("hex"); +const sha = text => createHash("sha1").update(text).digest("hex"); +const model = () => require("../model.cjs"); +const seal = m => { + const { approvedInputDigest, ...input } = m; + m.approvedInputDigest = model().fingerprintInput(input); + return m; +}; + +function manifest() { + return seal({ + schemaVersion: 1, repository: "dotnet/fsharp", issueNumber: 123, + discussionFingerprint: digest("complete issue and edited comments"), kind: "fsi", + sources: [{ path: "fixtures/repro.fsx", sha256: digest("printfn \"ok\"") }], + assertions: { + pass: { exitCode: 0, stdout: "ok\n", stderr: "" }, + failure: { exitCode: 1, stdout: "", stderr: "expected regression\n" }, + }, + matrix: [{ + id: "linux-x64", os: "linux", architecture: "x64", project: null, + tfm: "net10.0", configuration: "Release", languageVersion: "10.0", + command: ["fsi", "fixtures/repro.fsx"], + }], + timeoutSeconds: 300, producer: { name: "repro", version: "1.0.0", sha256: digest("producer") }, + packages: [{ id: "Example", version: "1.0.0", sha256: digest("package") }], + searchHints: { sdkVersions: ["9.0.100"], sourceShas: [sha("good")], urls: ["https://github.com/dotnet/fsharp"] }, + }); +} + +function attempt(outcome) { + return { + schemaVersion: 1, + checks: { restore: true, build: true, setup: true, identity: true, sdk: true, runtime: true }, + execution: { + status: "completed", complete: true, + ...structuredClone(manifest().assertions[outcome === "pass" ? "pass" : "failure"]), + ...(outcome === "untestable" ? { stderr: "unexpected exception" } : {}), + }, + }; +} + +const stableSdk = (matrix = [{ os: "linux", architecture: "x64" }]) => ({ + version: "10.0.100", origin: "official-release-index", latest: true, verified: true, + releaseIndexUrl: "https://builds.dotnet.microsoft.com/dotnet/release-metadata/releases-index.json", + releaseIndexDigest: digest("release index"), + archives: [...new Map(matrix.map(c => [`${c.os}-${c.architecture}`, { + os: c.os, architecture: c.architecture, sha256: digest(`SDK archive ${c.os}-${c.architecture}`), + }])).values()], +}); + +function observation(kind, outcome, m = manifest(), sourceSha = sha("main")) { + const fingerprint = model().fingerprintInput; + const cells = m.matrix.map(cell => ({ + id: cell.id, command: cell.command, + environment: { os: cell.os, architecture: cell.architecture, imageDigest: digest("runner image") }, + sdk: { version: "10.0.100", sha256: digest(`SDK archive ${cell.os}-${cell.architecture}`), verified: true }, + runtime: { version: "10.0.0", sha256: digest("runtime"), verified: true }, + artifacts: Object.fromEntries(["compiler", "tasks", "targets", "core"].map(name => [name, { + sha256: digest(`${kind}-${sourceSha}-${name}`), origin: kind === "source" ? "local-build" : "released-sdk", + sourceSha: kind === "source" ? sourceSha : null, buildInstanceId: `build-${sourceSha}`, + }])), + producer: structuredClone(m.producer), + attempts: [attempt(outcome), attempt(outcome)], + outcome: model().classifyAttempts([attempt(outcome), attempt(outcome)], m), + })); + return { + schemaVersion: 1, kind, target: kind === "source" ? { sourceSha } : { sdk: stableSdk(m.matrix) }, + manifestDigest: fingerprint(m), inputDigest: m.approvedInputDigest, discussionFingerprint: m.discussionFingerprint, + matrixFingerprint: fingerprint(m.matrix), producerFingerprint: fingerprint(m.producer), packageFingerprint: fingerprint(m.packages), + buildInstanceId: `build-${sourceSha}`, cached: false, cells, + outcome: cells[0].outcome, + provenance: { workerId: "worker-1", runId: "12345", workflowRevision: sha("workflow"), policyRevision: sha("policy") }, + }; +} + +function admission(stable = "product-failure", main = "product-failure", good = "pass") { + const m = manifest(); + const result = { + manifest: m, stable: observation("released", stable, m), main: observation("source", main, m), + good: observation("source", good, m, sha("good")), + ancestry: { goodSha: sha("good"), badSha: sha("main"), pinnedMainSha: sha("main"), isAncestor: true }, + budget: { + schemaVersion: 1, policy: policy(), policyRevision: sha("policy"), workflowRevision: sha("workflow"), + allowanceId: "maintainer-approved-investigation-1", + authorization: { + repository: m.repository, issueNumber: m.issueNumber, + approvalPath: ".github/regression-bisect/approved/123.json", approvalRevision: sha("policy"), + manifestDigest: model().fingerprintInput(m), inputDigest: m.approvedInputDigest, + }, + stableSdk: stableSdk(), authenticatedObservationDigests: [], + consumed: { sourceBuilds: 2, workerUnits: 3, reservedMinutes: 330 }, + attempts: [], nextWorkItemId: "first-bisection-build", pendingWork: null, + }, + }; + result.budget.authenticatedObservationDigests = [result.stable, result.main, result.good].map(model().fingerprintInput); + return result; +} + +function state() { + const a = admission(); + return { + schemaVersion: 1, repository: a.manifest.repository, issueNumber: a.manifest.issueNumber, + issueFingerprint: a.manifest.discussionFingerprint, approvalFingerprint: model().fingerprintInput(a.manifest), + manifest: a.manifest, budget: a.budget, pinnedMainSha: a.ancestry.pinnedMainSha, + epoch: model().fingerprintEpoch(a.manifest, a.budget, a.ancestry.pinnedMainSha), phase: "admission", + endpointEvidence: { stable: model().fingerprintInput(a.stable), main: model().fingerprintInput(a.main), good: model().fingerprintInput(a.good) }, + testedRevisions: [{ sourceSha: sha("good"), observationDigest: model().fingerprintInput(a.good) }], + skips: [], interval: { goodSha: sha("good"), badSha: sha("main"), gitLog: [] }, + publication: { digest: null, commentId: null }, + }; +} + +module.exports = { policy, manifest, attempt, observation, admission, state, seal, sha, digest }; diff --git a/eng/regression-bisect/model.cjs b/eng/regression-bisect/model.cjs new file mode 100644 index 00000000000..6d0296b31fb --- /dev/null +++ b/eng/regression-bisect/model.cjs @@ -0,0 +1,429 @@ +const { createHash } = require("node:crypto"); +const trustedPolicy = require("../../.github/policies/regression-bisect.json"); + +class ContractError extends Error {} + +function check(condition, path, message) { + if (!condition) throw new ContractError(`${path}: ${message}`); +} + +const text = (v, p) => check(typeof v === "string" && v.length > 0 && v.length <= 1024, p, "expected bounded nonempty string"); +const integer = (v, p) => check(Number.isSafeInteger(v) && v >= 0, p, "expected nonnegative safe integer"); +const positive = (v, p) => { integer(v, p); check(v > 0, p, "expected positive integer"); }; +const bool = (v, p) => check(typeof v === "boolean", p, "expected boolean"); +const literal = expected => (v, p) => check(v === expected, p, `expected ${expected}`); +const oneOf = values => (v, p) => check(values.includes(v), p, `expected one of ${values.join(", ")}`); +const matching = regex => (v, p) => check(typeof v === "string" && regex.test(v), p, "invalid format"); +const digest = matching(/^[0-9a-f]{64}$/); +const sourceSha = matching(/^[0-9a-f]{40}$/); +const version = matching(/^\d+\.\d+\.\d+(?:\.\d+)?$/); +const pinnedVersion = matching(/^\d+\.\d+\.\d+(?:\.\d+)?(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$/); +const nullable = validate => (v, p) => { if (v !== null) validate(v, p); }; + +function object(value, fields, path) { + check(value !== null && typeof value === "object" && Object.getPrototypeOf(value) === Object.prototype, path, "expected JSON object"); + check(Object.keys(value).length === Object.keys(fields).length && Object.keys(value).every(k => Object.hasOwn(fields, k)), + path, "missing or unknown fields"); + for (const [key, validate] of Object.entries(fields)) validate(value[key], `${path}.${key}`); +} + +const shape = fields => (v, p) => object(v, fields, p); +const list = (validate, min = 0, max = 256) => (v, p) => { + check(Array.isArray(v) && v.length >= min && v.length <= max, p, `expected array with ${min}..${max} entries`); + for (let i = 0; i < v.length; i++) validate(v[i], `${p}[${i}]`); +}; +const unique = (values, p) => check(new Set(values).size === values.length, p, "duplicate entries"); + +function fingerprintInput(value) { + const ancestors = new Set(); + function canonical(v) { + if (v === null || typeof v === "string" || typeof v === "boolean") return JSON.stringify(v); + if (typeof v === "number") { + check(Number.isFinite(v), "JSON", "nonfinite number"); + return JSON.stringify(v); + } + check(typeof v === "object" && v !== null && !ancestors.has(v), "JSON", "unsupported or cyclic value"); + ancestors.add(v); + let result; + if (Array.isArray(v)) { + check(Object.keys(v).length === v.length, "JSON", "sparse array or extra fields"); + result = `[${Array.from(v, canonical).join(",")}]`; + } else { + check(Object.getPrototypeOf(v) === Object.prototype, "JSON", "expected plain object"); + check(Reflect.ownKeys(v).length === Object.keys(v).length, "JSON", "non-JSON properties"); + result = `{${Object.keys(v).sort().map(k => `${JSON.stringify(k)}:${canonical(v[k])}`).join(",")}}`; + } + ancestors.delete(v); + return result; + } + return createHash("sha256").update(canonical(value)).digest("hex"); +} + +const equal = (a, b) => fingerprintInput(a) === fingerprintInput(b); + +function validTimestamp(value) { + if (typeof value !== "string") return false; + const match = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d{1,3})?(Z|[+-]\d{2}:\d{2})$/.exec(value); + if (!match) return false; + const [, y, mo, d, h, mi, s, zone] = match; + const days = new Date(Date.UTC(Number(y), Number(mo), 0)).getUTCDate(); + return Number(mo) >= 1 && Number(mo) <= 12 && Number(d) >= 1 && Number(d) <= days && + Number(h) < 24 && Number(mi) < 60 && Number(s) < 60 && + (zone === "Z" || (Number(zone.slice(1, 3)) <= 23 && Number(zone.slice(4)) < 60)) && + Number.isFinite(Date.parse(value)); +} + +function isEligibleIssue(issue) { + return !!issue && typeof issue === "object" && !Object.hasOwn(issue, "pull_request") && + issue.state === trustedPolicy.selection.state && validTimestamp(issue.created_at) && + Date.parse(issue.created_at) >= Date.parse(trustedPolicy.selection.createdAtOrAfter) && + Array.isArray(issue.labels) && issue.labels.some(label => + (typeof label === "string" ? label : label?.name) === trustedPolicy.selection.label); +} + +function validatePolicy(policy) { + check(policy !== null && typeof policy === "object", "policy", "expected policy object"); + check(policy.limits !== null && typeof policy.limits === "object", "policy.limits", "missing limits"); + for (const name of Object.keys(trustedPolicy.limits)) positive(policy.limits[name], `policy.limits.${name}`); + check(equal(policy, trustedPolicy), "policy", "must match the policy loaded from the pinned trusted revision; overrides are forbidden"); + return true; +} + +function repositoryPath(value, path) { + text(value, path); + check(value.split("/").every(part => + /^[A-Za-z0-9_.-]+$/.test(part) && part !== "." && part !== ".." && !part.endsWith(".") && + !/^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(part)), + path, "expected safe repository-relative path"); +} + +function httpsUrl(value, path) { + text(value, path); + let url; + try { url = new URL(value); } catch (error) { + if (!(error instanceof TypeError)) throw error; + throw new ContractError(`${path}: invalid URL`); + } + check(url.protocol === "https:" && !url.username && !url.password && !url.hash, path, "expected credential-free HTTPS URL"); +} + +function evidenceText(value, path) { + check(typeof value === "string" && Buffer.byteLength(value, "utf8") <= trustedPolicy.evidenceBytesPerStream, + path, "missing or oversized assertion evidence"); +} + +const exitCode = (v, p) => check(Number.isSafeInteger(v) && v >= -2147483648 && v <= 2147483647, p, "invalid exit code"); +const assertion = shape({ exitCode, stdout: evidenceText, stderr: evidenceText }); +const identity = shape({ name: text, version: pinnedVersion, sha256: digest }); +const os = oneOf(["linux", "windows", "macos"]); +const architecture = oneOf(["x64", "arm64"]); +const matrixCell = shape({ + id: text, os, architecture, project: nullable(repositoryPath), + tfm: matching(/^net(?:standard|coreapp)?\d+\.\d+(?:-[a-z0-9.]+)?$/), + configuration: oneOf(["Debug", "Release"]), + languageVersion: matching(/^(?:\d+\.\d+|preview)$/), + command: list(text, 2, 2), +}); + +function validateManifest(manifest) { + object(manifest, { + schemaVersion: literal(1), repository: literal(trustedPolicy.repository), issueNumber: positive, + discussionFingerprint: digest, kind: oneOf(["fsi", "project"]), + sources: list(shape({ path: repositoryPath, sha256: digest }), 1), + assertions: shape({ pass: assertion, failure: assertion }), + matrix: list(matrixCell, 1, 64), timeoutSeconds: positive, + producer: identity, packages: list(shape({ id: text, version: pinnedVersion, sha256: digest })), + searchHints: shape({ sdkVersions: list(version), sourceShas: list(sourceSha), urls: list(httpsUrl) }), + approvedInputDigest: digest, + }, "manifest"); + check(manifest.timeoutSeconds <= trustedPolicy.limits.reproductionTimeoutSeconds, "manifest.timeoutSeconds", "exceeds policy"); + unique(manifest.sources.map(s => s.path.toLowerCase()), "manifest.sources"); + unique(manifest.packages.map(p => p.id.toLowerCase()), "manifest.packages"); + unique(manifest.matrix.map(c => c.id), "manifest.matrix"); + check(!equal(manifest.assertions.pass, manifest.assertions.failure), "manifest.assertions", "pass and failure must differ"); + const paths = new Set(manifest.sources.map(s => s.path)); + for (const cell of manifest.matrix) { + const operation = manifest.kind === "fsi" ? "fsi" : "run-project"; + check(cell.command[0] === operation && paths.has(cell.command[1]), "manifest.matrix.command", "expected allowed operation and reviewed source"); + check(manifest.kind === "fsi" + ? cell.project === null && cell.command[1].endsWith(".fsx") + : cell.project === cell.command[1] && cell.project.endsWith(".fsproj"), + "manifest.matrix.project", "command does not match reproduction kind"); + } + const { approvedInputDigest, ...input } = manifest; + check(approvedInputDigest === fingerprintInput(input), "manifest.approvedInputDigest", "digest mismatch"); + return true; +} + +const untestableReasons = [ + "restore-failure", "build-failure", "setup-failure", "wrong-identity", "unavailable-sdk", "unavailable-runtime", + "timeout", "incomplete-evidence", "flaky", "unrelated-exception", "assertion-mismatch", +]; +function validateOutcome(outcome) { + object(outcome, { + schemaVersion: literal(1), outcome: oneOf(["pass", "product-failure", "untestable"]), reason: text, + }, "outcome"); + check(outcome.outcome === "pass" ? outcome.reason === "positive-assertion" + : outcome.outcome === "product-failure" ? outcome.reason === "failure-assertion" + : untestableReasons.includes(outcome.reason), "outcome.reason", "reason does not match outcome"); + return true; +} + +const outcome = (value, reason) => ({ schemaVersion: 1, outcome: value, reason }); +const untestable = reason => outcome("untestable", reason); +const checks = { + restore: "restore-failure", build: "build-failure", setup: "setup-failure", identity: "wrong-identity", + sdk: "unavailable-sdk", runtime: "unavailable-runtime", +}; +const attemptSchema = shape({ + schemaVersion: literal(1), + checks: shape(Object.fromEntries(Object.keys(checks).map(k => [k, bool]))), + execution: shape({ + status: oneOf(["completed", "timeout", "exception"]), complete: bool, + exitCode: nullable(exitCode), stdout: evidenceText, stderr: evidenceText, + }), +}); + +function classifyAttempts(attempts, manifest) { + validateManifest(manifest); + try { + list(attemptSchema, trustedPolicy.limits.assertionRepetitions, trustedPolicy.limits.assertionRepetitions)(attempts, "attempts"); + } catch (error) { + if (!(error instanceof ContractError)) throw error; + return untestable("incomplete-evidence"); + } + const results = attempts.map(attempt => { + for (const [name, reason] of Object.entries(checks)) { + if (!attempt.checks[name]) return untestable(reason); + } + const execution = attempt.execution; + if (execution.status === "timeout") return untestable("timeout"); + if (execution.status === "exception") return untestable("unrelated-exception"); + if (!execution.complete || execution.exitCode === null) return untestable("incomplete-evidence"); + const actual = { exitCode: execution.exitCode, stdout: execution.stdout, stderr: execution.stderr }; + if (equal(actual, manifest.assertions.pass)) return outcome("pass", "positive-assertion"); + if (equal(actual, manifest.assertions.failure)) return outcome("product-failure", "failure-assertion"); + return untestable("assertion-mismatch"); + }); + const failedSetup = results.find(r => r.outcome === "untestable"); + if (failedSetup) return failedSetup; + return results.every(r => equal(r, results[0])) ? results[0] : untestable("flaky"); +} + +function stableSdk(value, path) { + object(value, { + version, origin: literal("official-release-index"), latest: literal(true), verified: literal(true), + releaseIndexUrl: httpsUrl, releaseIndexDigest: digest, + archives: list(shape({ os, architecture, sha256: digest }), 1, 6), + }, path); + unique(value.archives.map(a => `${a.os}-${a.architecture}`), `${path}.archives`); + const url = new URL(value.releaseIndexUrl); + check(["builds.dotnet.microsoft.com", "dotnetcli.blob.core.windows.net"].includes(url.hostname) && + url.pathname === "/dotnet/release-metadata/releases-index.json" && !url.search && !url.port, + path, "expected official stable release index"); +} + +const productIdentity = shape({ version: pinnedVersion, sha256: digest, verified: literal(true) }); +const artifact = shape({ + sha256: digest, origin: oneOf(["local-build", "released-sdk"]), + sourceSha: nullable(sourceSha), buildInstanceId: text, +}); +const observationCell = shape({ + id: text, command: list(text, 2, 2), + environment: shape({ os, architecture, imageDigest: digest }), + sdk: nullable(productIdentity), runtime: nullable(productIdentity), + artifacts: shape({ compiler: nullable(artifact), tasks: nullable(artifact), targets: nullable(artifact), core: nullable(artifact) }), + producer: identity, attempts: list(attemptSchema, 1, 2), outcome: validateOutcome, +}); + +function validateObservation(observation, manifest, options = { finalConfirmation: null }) { + validateManifest(manifest); + object(observation, { + schemaVersion: literal(1), kind: oneOf(["source", "released"]), + target: (v, p) => object(v, observation.kind === "source" ? { sourceSha } : { sdk: stableSdk }, p), + manifestDigest: digest, inputDigest: digest, discussionFingerprint: digest, + matrixFingerprint: digest, producerFingerprint: digest, packageFingerprint: digest, + buildInstanceId: text, cached: bool, cells: list(observationCell, 1, 64), outcome: validateOutcome, + provenance: shape({ workerId: text, runId: text, workflowRevision: sourceSha, policyRevision: sourceSha }), + }, "observation"); + object(options, { + finalConfirmation: nullable(shape({ runId: text, buildInstanceId: text })), + }, "observation.fresh"); + const final = options.finalConfirmation; + check(final === null || (!observation.cached && observation.provenance.runId === final.runId && + observation.buildInstanceId === final.buildInstanceId), "observation", "final confirmation requires fresh execution"); + for (const [name, expected] of Object.entries({ + manifestDigest: fingerprintInput(manifest), inputDigest: manifest.approvedInputDigest, + discussionFingerprint: manifest.discussionFingerprint, matrixFingerprint: fingerprintInput(manifest.matrix), + producerFingerprint: fingerprintInput(manifest.producer), packageFingerprint: fingerprintInput(manifest.packages), + })) check(observation[name] === expected, `observation.${name}`, "input mismatch"); + unique(observation.cells.map(c => c.id), "observation.cells"); + check(observation.cells.length === manifest.matrix.length, "observation.cells", "incomplete matrix"); + for (const cell of observation.cells) { + const expected = manifest.matrix.find(c => c.id === cell.id); + check(expected && equal(cell.command, expected.command) && cell.environment.os === expected.os && + cell.environment.architecture === expected.architecture, "observation.cells", "matrix or command mismatch"); + check(equal(cell.producer, manifest.producer), "observation.cells.producer", "producer mismatch"); + for (const [name, missing] of Object.entries({ + sdk: cell.sdk === null, runtime: cell.runtime === null, + identity: Object.values(cell.artifacts).some(a => a === null), + })) { + check(!missing || (cell.outcome.outcome === "untestable" && cell.attempts.every(a => !a.checks[name])), + "observation.cells.identity", "absent identities require untestable checks"); + } + for (const value of Object.values(cell.artifacts)) { + if (value === null) continue; + check(value.origin === (observation.kind === "source" ? "local-build" : "released-sdk") && + value.sourceSha === (observation.kind === "source" ? observation.target.sourceSha : null) && + value.buildInstanceId === observation.buildInstanceId, "observation.cells.artifacts", "wrong source or build identity"); + } + if (observation.kind === "released" && cell.sdk !== null) { + const archive = observation.target.sdk.archives.find(a => + a.os === cell.environment.os && a.architecture === cell.environment.architecture); + check(cell.sdk.version === observation.target.sdk.version && cell.sdk.sha256 === archive?.sha256, + "observation.cells.sdk", "released SDK identity mismatch"); + } + check(equal(cell.outcome, classifyAttempts(cell.attempts, manifest)), "observation.cells.outcome", "does not match repeated evidence"); + } + const results = observation.cells.map(c => c.outcome); + const aggregate = results.find(r => r.outcome === "untestable") ?? + (results.every(r => equal(r, results[0])) ? results[0] : untestable("flaky")); + check(equal(observation.outcome, aggregate), "observation.outcome", "does not match matrix evidence"); + return true; +} + +const counters = shape({ sourceBuilds: integer, workerUnits: integer, reservedMinutes: integer }); +const reservation = shape({ + id: text, allowanceId: text, workItemId: text, attempt: positive, + reserved: counters, status: oneOf(["reserved", "running"]), +}); +function validateBudget(budget, manifest) { + object(budget, { + schemaVersion: literal(1), policy: validatePolicy, policyRevision: sourceSha, workflowRevision: sourceSha, + allowanceId: text, + authorization: shape({ + repository: literal(trustedPolicy.repository), issueNumber: positive, + approvalPath: repositoryPath, approvalRevision: sourceSha, manifestDigest: digest, inputDigest: digest, + }), + stableSdk, authenticatedObservationDigests: list(digest), + consumed: counters, attempts: list(shape({ workItemId: text, count: positive })), + nextWorkItemId: text, pendingWork: nullable(reservation), + }, "budget"); + const auth = budget.authorization; + check(auth.repository === manifest.repository && auth.issueNumber === manifest.issueNumber && + auth.approvalPath === `${trustedPolicy.authorization.approvalDirectory}/${manifest.issueNumber}.json` && + auth.approvalRevision === budget.policyRevision && + auth.manifestDigest === fingerprintInput(manifest) && auth.inputDigest === manifest.approvedInputDigest, + "budget.authorization", "missing matching pinned approval"); + for (const name of Object.keys(budget.consumed)) { + check(budget.consumed[name] <= budget.policy.limits[name], `budget.consumed.${name}`, "exceeds allowance"); + } + unique(budget.attempts.map(a => a.workItemId), "budget.attempts"); + check(budget.attempts.every(a => a.count <= budget.policy.limits.attemptsPerItem), "budget.attempts", "exceeds attempt limit"); + check(budget.attempts.reduce((sum, a) => sum + a.count, 0) <= budget.consumed.workerUnits, + "budget.attempts", "attempts must be charged"); + if (budget.pendingWork !== null) { + const pending = budget.pendingWork; + check(pending.allowanceId === budget.allowanceId && pending.workItemId === budget.nextWorkItemId && + pending.attempt === budget.attempts.find(a => a.workItemId === pending.workItemId)?.count && + pending.reserved.workerUnits === 1 && pending.reserved.sourceBuilds <= 1 && + pending.reserved.reservedMinutes === budget.policy.limits.workerTimeoutMinutes, + "budget.pendingWork", "reservation must be charged to this allowance and attempt"); + for (const name of Object.keys(pending.reserved)) { + check(pending.reserved[name] <= budget.consumed[name], "budget.pendingWork", "unaccounted reservation"); + } + } +} + +function fingerprintEpoch(manifest, budget, pinnedMainSha) { + validateManifest(manifest); + sourceSha(pinnedMainSha, "epoch.pinnedMainSha"); + stableSdk(budget.stableSdk, "epoch.stableSdk"); + return fingerprintInput({ + schemaVersion: 1, manifestDigest: fingerprintInput(manifest), + policyRevision: budget.policyRevision, workflowRevision: budget.workflowRevision, + pinnedMainSha, stableSdk: budget.stableSdk, + }); +} + +function admitBisection(input) { + try { + object(input, { + manifest: validateManifest, stable: () => {}, main: () => {}, good: () => {}, + ancestry: shape({ goodSha: sourceSha, badSha: sourceSha, pinnedMainSha: sourceSha, isAncestor: bool }), + budget: () => {}, + }, "admission"); + const { manifest, stable, main, good, ancestry, budget } = input; + validateBudget(budget, manifest); + for (const observation of [stable, main, good]) { + validateObservation(observation, manifest); + check(budget.authenticatedObservationDigests.includes(fingerprintInput(observation)), + "admission", "observation not authenticated by the trusted collector"); + check(observation.provenance.policyRevision === budget.policyRevision && + observation.provenance.workflowRevision === budget.workflowRevision, "admission", "provenance mismatch"); + } + check(stable.kind === "released" && main.kind === "source" && good.kind === "source", + "admission", "missing source interval or released stable endpoint"); + check(equal(stable.target.sdk, budget.stableSdk), "admission", "not the pinned latest stable SDK"); + check(main.target.sourceSha === ancestry.pinnedMainSha && main.target.sourceSha === ancestry.badSha && + good.target.sourceSha === ancestry.goodSha && ancestry.goodSha !== ancestry.badSha && ancestry.isAncestor, + "admission", "good must be a strict ancestor of pinned main"); + for (const cell of main.cells) { + const older = good.cells.find(c => c.id === cell.id); + const released = stable.cells.find(c => c.id === cell.id); + check(equal(cell.environment, older.environment) && equal(cell.environment, released.environment) && + equal(cell.sdk, older.sdk) && equal(cell.runtime, older.runtime), "admission", "incomparable environment or source toolchain"); + } + if ([stable, main, good].some(o => o.outcome.outcome === "untestable")) { + return { decision: "blocked", reason: "untestable endpoint evidence" }; + } + if (stable.outcome.outcome !== "product-failure") return { decision: "stop", reason: "not reproduced on latest stable" }; + if (main.outcome.outcome !== "product-failure") return { decision: "stop", reason: "release-specific or already fixed on main" }; + if (good.outcome.outcome !== "pass") return { decision: "stop", reason: "missing passing source endpoint" }; + const limits = budget.policy.limits; + check(budget.pendingWork !== null ? budget.pendingWork.reserved.sourceBuilds === 1 : + budget.consumed.sourceBuilds < limits.sourceBuilds && budget.consumed.workerUnits < limits.workerUnits && + budget.consumed.reservedMinutes + limits.workerTimeoutMinutes <= limits.reservedMinutes && + (budget.attempts.find(a => a.workItemId === budget.nextWorkItemId)?.count ?? 0) < limits.attemptsPerItem, + "admission", "insufficient remaining budget"); + return { decision: "admit", reason: "verified stable failure, pinned-main failure, and comparable ancestor pass" }; + } catch (error) { + if (!(error instanceof ContractError)) throw error; + return { decision: "blocked", reason: error.message }; + } +} + +function validateState(state) { + try { + object(state, { + schemaVersion: literal(1), repository: literal(trustedPolicy.repository), issueNumber: positive, + issueFingerprint: digest, approvalFingerprint: digest, manifest: validateManifest, + budget: () => {}, pinnedMainSha: sourceSha, epoch: digest, + phase: oneOf(["discovery", "stable", "main", "good-search", "admission", "bisect", "confirmation", "complete", "blocked", "stopped"]), + endpointEvidence: shape({ stable: nullable(digest), main: nullable(digest), good: nullable(digest) }), + testedRevisions: list(shape({ sourceSha, observationDigest: digest })), + skips: list(shape({ sourceSha, reason: oneOf(untestableReasons), observationDigest: digest })), + interval: nullable(shape({ goodSha: sourceSha, badSha: sourceSha, gitLog: list(text, 0, 1000) })), + publication: shape({ digest: nullable(digest), commentId: nullable(positive) }), + }, "state"); + validateBudget(state.budget, state.manifest); + check(state.repository === state.manifest.repository && state.issueNumber === state.manifest.issueNumber && + state.issueFingerprint === state.manifest.discussionFingerprint && + state.approvalFingerprint === fingerprintInput(state.manifest) && + state.epoch === fingerprintEpoch(state.manifest, state.budget, state.pinnedMainSha), + "state", "identity or epoch mismatch"); + const budget = state.budget; + const evidence = [...Object.values(state.endpointEvidence).filter(x => x !== null), + ...state.testedRevisions.map(r => r.observationDigest), ...state.skips.map(r => r.observationDigest)]; + check(evidence.every(d => budget.authenticatedObservationDigests.includes(d)), "state", "unauthenticated evidence"); + return true; + } catch (error) { + if (!(error instanceof ContractError)) throw error; + throw new ContractError(`state: ${error.message}`); + } +} + +module.exports = { + isEligibleIssue, validatePolicy, validateManifest, fingerprintInput, classifyAttempts, + validateObservation, admitBisection, validateOutcome, validateState, fingerprintEpoch, +}; diff --git a/eng/regression-bisect/model.test.cjs b/eng/regression-bisect/model.test.cjs new file mode 100644 index 00000000000..f87b3661e4a --- /dev/null +++ b/eng/regression-bisect/model.test.cjs @@ -0,0 +1,388 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); +const { + isEligibleIssue, validatePolicy, validateManifest, fingerprintInput, fingerprintEpoch, + classifyAttempts, validateOutcome, validateObservation, validateState, admitBisection, +} = require("./model.cjs"); +const { policy, manifest, attempt, observation, admission, state, seal, sha, digest } = require("./fixtures/contracts.cjs"); + +const rejects = (fn, pattern = /./) => assert.throws(fn, pattern); + +for (const [name, patch, expected] of [ + ["before cutoff", { created_at: "2026-08-31T23:59:59.999Z" }, false], + ["at cutoff", {}, true], + ["after cutoff", { created_at: "2026-09-01T00:00:00.001Z" }, true], + ["equivalent offset", { created_at: "2026-09-01T02:00:00+02:00" }, true], + ["offset before cutoff", { created_at: "2026-09-01T00:00:00+01:00" }, false], + ["invalid date", { created_at: "not a date" }, false], + ["normalized invalid date", { created_at: "2026-09-31T00:00:00Z" }, false], + ["no timezone", { created_at: "2026-09-01T00:00:00" }, false], + ["closed", { state: "closed" }, false], + ["pull request", { pull_request: {} }, false], + ["null pull request marker", { pull_request: null }, false], + ["wrong case", { labels: [{ name: "regression" }] }, false], + ["missing labels", { labels: undefined }, false], + ["string label", { labels: ["Regression"] }, true], + ["updated is not created", { created_at: "2025-09-01T00:00:00Z", updated_at: "2026-09-02T00:00:00Z" }, false], +]) { + test(`eligibility: ${name}`, () => { + assert.equal(isEligibleIssue({ + state: "open", created_at: "2026-09-01T00:00:00Z", labels: [{ name: "Regression" }], ...patch, + }), expected); + }); +} + +test("policy has finite operational defaults, separate from issue authorization", () => { + assert.equal(validatePolicy(policy()), true); + assert.deepEqual(policy().limits, { + sourceBuilds: 24, workerUnits: 40, reservedMinutes: 720, attemptsPerItem: 2, + assertionRepetitions: 2, buildTimeoutMinutes: 90, reproductionTimeoutSeconds: 300, + workerTimeoutMinutes: 110, discoveryIssues: 3, discoveryPages: 2, discoveryPageSize: 100, + }); +}); + +for (const key of Object.keys(policy().limits)) { + for (const value of [-1, 0, 1.5, null, "unlimited", Infinity, policy().limits[key] + 1]) { + test(`policy rejects ${key}=${value}`, () => { + const p = policy(); + p.limits[key] = value; + rejects(() => validatePolicy(p), /policy/); + }); + } +} + +for (const [name, edit] of [ + ["schema", m => m.schemaVersion = 2], + ["unknown field", m => m.instructions = "run issue commands"], + ["missing approval", m => delete m.approvedInputDigest], + ["wrong approval digest", m => m.approvedInputDigest = digest("wrong")], + ["traversal", m => m.sources[0].path = "../payload.fsx"], + ["encoded traversal", m => m.sources[0].path = "%2e%2e/payload.fsx"], + ["absolute path", m => m.sources[0].path = "C:\\payload.fsx"], + ["UNC path", m => m.sources[0].path = "\\\\host\\payload.fsx"], + ["URL source", m => m.sources[0].path = "https://example.org/code.fsx"], + ["alternate stream", m => m.sources[0].path = "fixture.fsx:code"], + ["shell string", m => m.matrix[0].command = "dotnet fsi repro.fsx"], + ["interpolated argument", m => m.matrix[0].command.push("$(curl evil)")], + ["unreviewed command source", m => m.matrix[0].command[1] = "other.fsx"], + ["unsafe hint scheme", m => m.searchHints.urls = ["file:///tmp/code"]], + ["malformed hint URL", m => m.searchHints.urls = ["not-a-url"]], + ["URL credentials", m => m.searchHints.urls = ["https://user:secret@example.org/"]], + ["unbounded timeout", m => m.timeoutSeconds = 301], + ["package wildcard", m => m.packages[0].version = "1.*"], + ["identical assertions", m => m.assertions.failure = structuredClone(m.assertions.pass)], + ["empty matrix", m => m.matrix = []], + ["duplicate matrix cell", m => m.matrix.push(structuredClone(m.matrix[0]))], + ["duplicate source", m => m.sources.push(structuredClone(m.sources[0]))], +]) { + test(`manifest rejects ${name}`, () => { + const m = manifest(); + edit(m); + if (!name.includes("approval")) seal(m); + rejects(() => validateManifest(m), /manifest/); + }); +} + +test("both reviewed fsi and project manifests validate", () => { + assert.equal(validateManifest(manifest()), true); + const m = manifest(); + m.kind = "project"; + m.sources.push({ path: "fixtures/repro.fsproj", sha256: digest("project") }); + m.matrix[0].project = "fixtures/repro.fsproj"; + m.matrix[0].command = ["run-project", "fixtures/repro.fsproj"]; + seal(m); + assert.equal(validateManifest(m), true); +}); + +for (const [name, results, edit, expected, reason] of [ + ["passes", ["pass", "pass"], () => {}, "pass", "positive-assertion"], + ["failures", ["product-failure", "product-failure"], () => {}, "product-failure", "failure-assertion"], + ["flake", ["pass", "product-failure"], () => {}, "untestable", "flaky"], + ["restore", ["pass", "pass"], a => a.checks.restore = false, "untestable", "restore-failure"], + ["build", ["pass", "pass"], a => a.checks.build = false, "untestable", "build-failure"], + ["setup", ["pass", "pass"], a => a.checks.setup = false, "untestable", "setup-failure"], + ["identity", ["pass", "pass"], a => a.checks.identity = false, "untestable", "wrong-identity"], + ["SDK", ["pass", "pass"], a => a.checks.sdk = false, "untestable", "unavailable-sdk"], + ["runtime", ["pass", "pass"], a => a.checks.runtime = false, "untestable", "unavailable-runtime"], + ["timeout", ["pass", "pass"], a => a.execution.status = "timeout", "untestable", "timeout"], + ["exception", ["pass", "pass"], a => a.execution.status = "exception", "untestable", "unrelated-exception"], + ["nonzero is not assertion", ["product-failure", "product-failure"], a => a.execution.stderr = "unrelated", "untestable", "assertion-mismatch"], + ["incomplete output", ["pass", "pass"], a => a.execution.complete = false, "untestable", "incomplete-evidence"], + ["missing evidence", ["pass", "pass"], a => delete a.execution, "untestable", "incomplete-evidence"], + ["one repetition", ["pass"], () => {}, "untestable", "incomplete-evidence"], +]) { + test(`classification: ${name}`, () => { + const attempts = results.map(r => attempt(r)); + edit(attempts[0]); + assert.deepEqual(classifyAttempts(attempts, manifest()), { schemaVersion: 1, outcome: expected, reason }); + }); +} + +test("outcome versions and reason combinations are validated", () => { + assert.equal(validateOutcome({ schemaVersion: 1, outcome: "untestable", reason: "flaky" }), true); + for (const value of [ + { schemaVersion: 2, outcome: "pass", reason: "positive-assertion" }, + { schemaVersion: 1, outcome: "green", reason: "positive-assertion" }, + { schemaVersion: 1, outcome: "pass", reason: "restore-failure" }, + { schemaVersion: 1, outcome: "product-failure", reason: "exit-code" }, + ]) rejects(() => validateOutcome(value), /outcome/); +}); + +test("neither repeated passes nor expected failures survive failed required checks", () => { + for (const result of ["pass", "product-failure"]) { + for (const check of Object.keys(attempt(result).checks)) { + const attempts = [attempt(result), attempt(result)]; + attempts[1].checks[check] = false; + assert.equal(classifyAttempts(attempts, manifest()).outcome, "untestable", `${result}: ${check}`); + } + } +}); + +const outcomes = ["pass", "product-failure", "untestable"]; +for (const stable of outcomes) { + for (const main of outcomes) { + for (const good of outcomes) { + test(`admission: ${stable}/${main}/${good}`, () => { + const args = admission(stable, main, good); + const result = admitBisection(args); + assert.equal(result.decision === "admit", stable === "product-failure" && main === "product-failure" && good === "pass"); + assert.equal(typeof result.reason, "string"); + if ([stable, main, good].includes("untestable")) assert.equal(result.decision, "blocked"); + }); + } + } +} + +for (const [name, edit] of [ + ["missing manifest", a => delete a.manifest], + ["missing good", a => delete a.good], + ["missing approval", a => delete a.budget.authorization], + ["self approved changed manifest", a => { a.manifest.assertions.pass.stdout = "changed"; seal(a.manifest); }], + ["released-product-only good", a => a.good = observation("released", "pass", a.manifest)], + ["nonancestor", a => a.ancestry.isAncestor = false], + ["unknown ancestry", a => delete a.ancestry.isAncestor], + ["same source", a => a.good.target.sourceSha = a.main.target.sourceSha], + ["wrong main SHA", a => a.ancestry.pinnedMainSha = sha("other")], + ["wrong matrix", a => a.good.matrixFingerprint = digest("other")], + ["wrong producer", a => a.good.producerFingerprint = digest("other")], + ["wrong digest", a => a.main.manifestDigest = digest("other")], + ["wrong discussion", a => a.good.discussionFingerprint = digest("other")], + ["wrong environment", a => a.good.cells[0].environment.imageDigest = digest("other")], + ["wrong source SDK", a => a.good.cells[0].sdk.version = "9.0.100"], + ["missing local Core", a => delete a.main.cells[0].artifacts.core], + ["released compiler in source test", a => a.main.cells[0].artifacts.compiler.origin = "released-sdk"], + ["wrong build instance", a => a.main.cells[0].artifacts.core.buildInstanceId = "other"], + ["wrong artifact source", a => a.good.cells[0].artifacts.tasks.sourceSha = sha("other")], + ["missing cell", a => a.main.cells = []], + ["unverified evidence", a => a.budget.authenticatedObservationDigests = []], + ["unverified SDK", a => a.stable.target.sdk.verified = false], + ["installed SDK", a => a.stable.target.sdk.origin = "installed"], + ["preview SDK", a => a.stable.target.sdk.version = "11.0.100-preview.1"], + ["not latest stable", a => a.stable.target.sdk.latest = false], + ["wrong resolved stable", a => a.budget.stableSdk.version = "9.0.100"], + ["unofficial release index", a => a.stable.target.sdk.releaseIndexUrl = "https://example.org/releases-index.json"], + ["wrong workflow", a => a.good.provenance.workflowRevision = sha("other")], + ["forged outcome", a => a.good.outcome.outcome = "pass"], + ["source budget", a => a.budget.consumed.sourceBuilds = policy().limits.sourceBuilds], + ["worker budget", a => a.budget.consumed.workerUnits = policy().limits.workerUnits], + ["time budget", a => a.budget.consumed.reservedMinutes = policy().limits.reservedMinutes - 109], + ["attempts exhausted", a => a.budget.attempts = [{ workItemId: a.budget.nextWorkItemId, count: 2 }]], + ["expanded budget", a => a.budget.policy.limits.workerUnits++], + ["unknown admission field", a => a.override = true], +]) { + test(`admission denies ${name}`, () => { + const args = admission(); + if (name === "forged outcome") args.good = observation("source", "untestable", args.manifest, sha("good")); + edit(args); + if (name !== "unverified evidence") { + args.budget.authenticatedObservationDigests = [args.stable, args.main, args.good] + .filter(Boolean).map(fingerprintInput); + } + assert.notEqual(admitBisection(args).decision, "admit"); + }); +} + +test("invalid observation evidence is rejected even if the collector authenticated its digest", () => { + for (const edit of [ + o => o.cells[0].attempts[0].checks.identity = false, + o => o.cells[0].command = ["fsi", "unreviewed.fsx"], + o => o.cells[0].attempts[0].execution.stdout = "x".repeat(16385), + o => o.cells.push(structuredClone(o.cells[0])), + o => o.provenance.runId = "", + o => o.schemaVersion = 2, + ]) { + const o = observation("source", "pass", manifest(), sha("good")); + edit(o); + rejects(() => validateObservation(o, manifest()), /observation/); + } +}); + +test("fresh final confirmations cannot use cached endpoint evidence", () => { + const o = observation("source", "pass", manifest(), sha("good")); + const finalConfirmation = { runId: o.provenance.runId, buildInstanceId: o.buildInstanceId }; + assert.equal(validateObservation(o, manifest(), { finalConfirmation }), true); + rejects(() => validateObservation(o, manifest(), { + finalConfirmation: { ...finalConfirmation, runId: "new-run" }, + }), /fresh/); + rejects(() => validateObservation(o, manifest(), { + finalConfirmation: { ...finalConfirmation, buildInstanceId: "new-build" }, + }), /fresh/); + rejects(() => validateObservation(o, manifest(), { finalConfirmation: true }), /fresh/); + o.cached = true; + assert.equal(validateObservation(o, manifest()), true); + rejects(() => validateObservation(o, manifest(), { finalConfirmation }), /fresh/); +}); + +test("multi-platform stable release identities bind each platform's archive", () => { + const m = manifest(); + m.matrix.push({ ...m.matrix[0], id: "windows-arm64", os: "windows", architecture: "arm64" }); + seal(m); + const o = observation("released", "product-failure", m); + assert.equal(validateObservation(o, m), true); + o.cells[1].sdk.sha256 = o.cells[0].sdk.sha256; + rejects(() => validateObservation(o, m), /released SDK identity/); +}); + +test("source builds may use pinned prerelease toolchains without treating them as released stable", () => { + const a = admission(); + for (const o of [a.main, a.good]) { + o.cells[0].sdk.version = "11.0.100-rc.1.26420.103"; + o.cells[0].runtime.version = "11.0.0-rc.1.26420.103"; + } + a.budget.authenticatedObservationDigests = [a.stable, a.main, a.good].map(fingerprintInput); + assert.equal(admitBisection(a).decision, "admit"); +}); + +test("assertions preserve signed process exit codes", () => { + const m = manifest(); + m.assertions.failure.exitCode = -1; + seal(m); + const attempts = [attempt("product-failure"), attempt("product-failure")]; + for (const a of attempts) a.execution.exitCode = -1; + assert.equal(classifyAttempts(attempts, m).outcome, "product-failure"); +}); + +test("every matrix cell must reproduce consistently, not just the first cell", () => { + const a = admission(); + a.manifest.matrix.push({ ...a.manifest.matrix[0], id: "debug", configuration: "Debug" }); + seal(a.manifest); + a.stable = observation("released", "product-failure", a.manifest); + a.main = observation("source", "product-failure", a.manifest); + a.good = observation("source", "pass", a.manifest, sha("good")); + a.budget.authorization.manifestDigest = fingerprintInput(a.manifest); + a.budget.authorization.inputDigest = a.manifest.approvedInputDigest; + a.budget.authenticatedObservationDigests = [a.stable, a.main, a.good].map(fingerprintInput); + assert.equal(admitBisection(a).decision, "admit"); + a.good.cells[1].attempts[1] = attempt("product-failure"); + a.budget.authenticatedObservationDigests = [a.stable, a.main, a.good].map(fingerprintInput); + assert.equal(admitBisection(a).decision, "blocked"); +}); + +test("budget boundary includes the whole worker reservation and retry cost", () => { + const a = admission(); + a.budget.consumed = { sourceBuilds: 23, workerUnits: 39, reservedMinutes: 610 }; + a.budget.attempts = [{ workItemId: a.budget.nextWorkItemId, count: 1 }]; + assert.equal(admitBisection(a).decision, "admit"); + a.budget.consumed.reservedMinutes++; + assert.match(admitBisection(a).reason, /budget/); +}); + +test("a charged reservation remains usable at the allowance boundary, but cannot cover another item", () => { + const a = admission(); + a.budget.consumed = { sourceBuilds: 24, workerUnits: 40, reservedMinutes: 720 }; + a.budget.attempts = [{ workItemId: a.budget.nextWorkItemId, count: 2 }]; + a.budget.pendingWork = { + id: "last-reservation", allowanceId: a.budget.allowanceId, workItemId: a.budget.nextWorkItemId, + attempt: 2, reserved: { sourceBuilds: 1, workerUnits: 1, reservedMinutes: 110 }, status: "reserved", + }; + assert.equal(admitBisection(a).decision, "admit"); + a.budget.pendingWork.workItemId = "unreserved-item"; + assert.equal(admitBisection(a).decision, "blocked"); +}); + +test("unavailable runtime can be recorded as untestable without fabricating its identity", () => { + const o = observation("source", "pass", manifest()); + o.cells[0].runtime = null; + for (const a of o.cells[0].attempts) a.checks.runtime = false; + o.cells[0].outcome = o.outcome = classifyAttempts(o.cells[0].attempts, manifest()); + assert.equal(o.outcome.reason, "unavailable-runtime"); + assert.equal(validateObservation(o, manifest()), true); + o.cells[0].attempts[0].checks.runtime = true; + rejects(() => validateObservation(o, manifest()), /identity/); +}); + +test("canonical hashes preserve array order, exact source bytes, and diagnostic whitespace", () => { + assert.equal(fingerprintInput({ b: 2, a: { d: 4, c: 3 } }), fingerprintInput({ a: { c: 3, d: 4 }, b: 2 })); + for (const [a, b] of [ + [["a", "b"], ["b", "a"]], ["source\r\n", "source\n"], ["FS0001: a b", "FS0001: a b"], + [{ body: "old" }, { body: "corrected" }], + [{ comments: [{ id: 1, body: "old", updatedAt: "a" }] }, { comments: [{ id: 1, body: "edited", updatedAt: "b" }] }], + ]) assert.notEqual(fingerprintInput(a), fingerprintInput(b)); + for (const value of [undefined, NaN, Infinity, { a: undefined }, new Date(), () => {}]) { + rejects(() => fingerprintInput(value), /JSON/); + } + const cycle = {}; + cycle.self = cycle; + rejects(() => fingerprintInput(cycle), /JSON/); + rejects(() => fingerprintInput(Array(1)), /JSON/); +}); + +test("pending reservations and cancelled attempts remain charged in state", () => { + const s = state(); + s.budget.attempts = [{ workItemId: s.budget.nextWorkItemId, count: 1 }]; + s.budget.pendingWork = { + id: "reservation-1", allowanceId: s.budget.allowanceId, workItemId: s.budget.nextWorkItemId, + attempt: 1, reserved: { sourceBuilds: 1, workerUnits: 1, reservedMinutes: 110 }, status: "reserved", + }; + assert.equal(validateState(s), true); + s.budget.pendingWork.status = "running"; + assert.equal(validateState(s), true); + const consumed = structuredClone(s.budget.consumed); + s.budget.pendingWork = null; + assert.equal(validateState(s), true); + assert.deepEqual(s.budget.consumed, consumed); + s.budget.consumed.workerUnits = 0; + rejects(() => validateState(s), /charged/); +}); + +test("workflow-looking text never erases discussion records", () => { + const input = { issue: { body: "repro" }, comments: [{ id: 1, body: "", author: "github-actions[bot]" }] }; + assert.notEqual(fingerprintInput(input), fingerprintInput({ ...input, comments: [] })); + input.comments[0].authenticated = true; + assert.notEqual(fingerprintInput(input), fingerprintInput({ ...input, comments: [] })); +}); + +test("approval and input are independent of main; epoch binds main and stable", () => { + const args = admission(); + const first = fingerprintEpoch(args.manifest, args.budget, args.ancestry.pinnedMainSha); + const approvalDigest = args.manifest.approvedInputDigest; + assert.notEqual(first, fingerprintEpoch(args.manifest, args.budget, sha("new-main"))); + args.budget.stableSdk.version = "10.0.101"; + assert.notEqual(first, fingerprintEpoch(args.manifest, args.budget, args.ancestry.pinnedMainSha)); + assert.equal(args.manifest.approvedInputDigest, approvalDigest); + for (const edit of [m => m.sources[0].sha256 = digest("edit"), m => m.assertions.failure.stderr += " "]) { + const m = manifest(); + edit(m); + rejects(() => validateManifest(m), /digest/); + } +}); + +test("state contract carries durable allowance, reservations, evidence, interval and publication", () => { + assert.equal(validateState(state()), true); + for (const edit of [ + s => delete s.budget.pendingWork, + s => s.schemaVersion = 2, + s => s.epoch = digest("wrong"), + s => s.budget.consumed.workerUnits = -1, + s => s.budget.pendingWork = { id: "unaccounted" }, + s => s.budget.allowanceId = "", + s => s.phase = "anything", + s => s.endpointEvidence.main = "not a digest", + s => s.publication.commentId = -1, + s => s.budget.authorization.approvalPath = ".github/regression-bisect/approved/other.json", + ]) { + const s = state(); + edit(s); + rejects(() => validateState(s), /state/); + } +}); From db3f8921bd37d2cb8860364e29a88a08522c99f2 Mon Sep 17 00:00:00 2001 From: Copilot Date: Fri, 18 Sep 2026 20:29:41 +0200 Subject: [PATCH 02/99] Use the trusted repetition limit in observation validation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/model.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/regression-bisect/model.cjs b/eng/regression-bisect/model.cjs index 6d0296b31fb..b724acf63f4 100644 --- a/eng/regression-bisect/model.cjs +++ b/eng/regression-bisect/model.cjs @@ -233,7 +233,7 @@ const observationCell = shape({ environment: shape({ os, architecture, imageDigest: digest }), sdk: nullable(productIdentity), runtime: nullable(productIdentity), artifacts: shape({ compiler: nullable(artifact), tasks: nullable(artifact), targets: nullable(artifact), core: nullable(artifact) }), - producer: identity, attempts: list(attemptSchema, 1, 2), outcome: validateOutcome, + producer: identity, attempts: list(attemptSchema, 1, trustedPolicy.limits.assertionRepetitions), outcome: validateOutcome, }); function validateObservation(observation, manifest, options = { finalConfirmation: null }) { From 64e8231d7517f0f850c37669e7ce498378df2b1f Mon Sep 17 00:00:00 2001 From: Copilot Date: Fri, 18 Sep 2026 22:04:17 +0200 Subject: [PATCH 03/99] Validate trusted context before hashing investigation epochs Reuse the budget contract to reject missing approval and unpinned policy or workflow identities. Cover invalid contexts and counter-independent epochs with table-driven tests. Validation: node --test eng\regression-bisect\model.test.cjs (RED: 5 failing epoch cases; GREEN: 215 passing tests). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 6 ++++-- eng/regression-bisect/model.cjs | 2 +- eng/regression-bisect/model.test.cjs | 30 ++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index dfa60c3e339..789223ae2fb 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -56,7 +56,7 @@ attempt evidence to an explicit `untestable` outcome; `admitBisection` returns | `classifyAttempts(attempts, manifest)` | Checks both required repetitions against the frozen assertions and all setup/identity checks. Returns a versioned outcome with an explicit reason. | | `validateOutcome(outcome)` | Only `pass`, `product-failure`, or `untestable`, with a reason appropriate to that outcome. | | `validateObservation(observation, manifest, options?)` | Validates target, all reproduction fingerprints, matrix evidence, local artifacts, provenance, and recomputed outcomes. | -| `fingerprintEpoch(manifest, budget, pinnedMainSha)` | Binds the manifest, pinned main, resolved stable release, and policy/workflow revisions. | +| `fingerprintEpoch(manifest, budget, pinnedMainSha)` | Validates the manifest and trusted budget context, then binds the manifest, pinned main, resolved stable release, and policy/workflow revisions. | | `validateState(state)` | Validates the shared persisted-state shape, epoch, authenticated evidence references, cumulative budget, and any charged pending reservation. | | `admitBisection({ manifest, stable, main, good, ancestry, budget })` | Fail-closed gate for a verified source interval; no execution side effects. | @@ -86,7 +86,9 @@ satisfy the failure assertion only after the compiler itself was built and verified successfully. `approvedInputDigest` hashes the manifest with only that field omitted. It does -not include the moving main SHA. The discussion fingerprint must cover the +not include the moving main SHA. Epochs require the same validated approval and +pinned policy/workflow identities as admission; charging more work does not +change the epoch. The discussion fingerprint must cover the complete issue and all reporter/maintainer comments, corrections, edits, and edit timestamps. The collector must supply the complete snapshot; hashing cannot detect omitted input. This module excludes **no** discussion records. diff --git a/eng/regression-bisect/model.cjs b/eng/regression-bisect/model.cjs index b724acf63f4..239ea99868c 100644 --- a/eng/regression-bisect/model.cjs +++ b/eng/regression-bisect/model.cjs @@ -337,8 +337,8 @@ function validateBudget(budget, manifest) { function fingerprintEpoch(manifest, budget, pinnedMainSha) { validateManifest(manifest); + validateBudget(budget, manifest); sourceSha(pinnedMainSha, "epoch.pinnedMainSha"); - stableSdk(budget.stableSdk, "epoch.stableSdk"); return fingerprintInput({ schemaVersion: 1, manifestDigest: fingerprintInput(manifest), policyRevision: budget.policyRevision, workflowRevision: budget.workflowRevision, diff --git a/eng/regression-bisect/model.test.cjs b/eng/regression-bisect/model.test.cjs index f87b3661e4a..9b91fa79490 100644 --- a/eng/regression-bisect/model.test.cjs +++ b/eng/regression-bisect/model.test.cjs @@ -367,6 +367,36 @@ test("approval and input are independent of main; epoch binds main and stable", } }); +for (const [name, edit] of [ + ["missing policy revision", b => delete b.policyRevision], + ["unpinned policy revision", b => b.policyRevision = "main"], + ["null workflow revision", b => b.workflowRevision = null], + ["unpinned workflow revision", b => b.workflowRevision = "main"], + ["missing approval", b => delete b.authorization], +]) { + test(`epoch rejects ${name}`, () => { + const a = admission(); + edit(a.budget); + rejects(() => fingerprintEpoch(a.manifest, a.budget, a.ancestry.pinnedMainSha), /budget/); + }); +} + +test("epoch identity excludes charged work and binds both trusted revisions", () => { + const a = admission(); + const epoch = () => fingerprintEpoch(a.manifest, a.budget, a.ancestry.pinnedMainSha); + const original = epoch(); + a.budget.consumed = { sourceBuilds: 3, workerUnits: 4, reservedMinutes: 440 }; + assert.equal(epoch(), original); + for (const field of ["policyRevision", "workflowRevision"]) { + const revision = a.budget[field]; + a.budget[field] = sha("new-revision"); + if (field === "policyRevision") a.budget.authorization.approvalRevision = a.budget[field]; + assert.notEqual(epoch(), original); + a.budget[field] = revision; + a.budget.authorization.approvalRevision = a.budget.policyRevision; + } +}); + test("state contract carries durable allowance, reservations, evidence, interval and publication", () => { assert.equal(validateState(state()), true); for (const edit of [ From 4d409fdf68452d1584495a35e7a1f5cb0aef2c7d Mon Sep 17 00:00:00 2001 From: Copilot Date: Fri, 18 Sep 2026 22:28:58 +0200 Subject: [PATCH 04/99] Keep regression policy independent of the shared JSON cache Read the trusted checkout policy into a private snapshot so caller mutations cannot expand limits or alter authorization and issue selection. Cover mutations both before and after model initialization. RED: three new policy mutation cases failed with Missing expected exception. GREEN: node --test eng\regression-bisect\model.test.cjs passed all 218 tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 3 +++ eng/regression-bisect/model.cjs | 3 ++- eng/regression-bisect/model.test.cjs | 27 +++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index 789223ae2fb..255ed06b9fd 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -21,6 +21,9 @@ The host must load this module and `.github/policies/regression-bisect.json` fro a pinned, trusted default-branch revision, not from the source revision under test. `validatePolicy` accepts only that policy, including its finite limits; an issue, dispatch input, or agent cannot override it. +The module reads its own policy snapshot from that checkout, bypassing Node's +shared JSON module cache so caller mutations cannot change validation limits, +authorization, or selection, even before this module is loaded. The host must construct `budget.authorization` from the reviewed approval at `.github/regression-bisect/approved/.json` at `budget.policyRevision`. diff --git a/eng/regression-bisect/model.cjs b/eng/regression-bisect/model.cjs index 239ea99868c..dd2d1bdfdde 100644 --- a/eng/regression-bisect/model.cjs +++ b/eng/regression-bisect/model.cjs @@ -1,5 +1,6 @@ const { createHash } = require("node:crypto"); -const trustedPolicy = require("../../.github/policies/regression-bisect.json"); +const { readFileSync } = require("node:fs"); +const trustedPolicy = JSON.parse(readFileSync(require.resolve("../../.github/policies/regression-bisect.json"), "utf8")); class ContractError extends Error {} diff --git a/eng/regression-bisect/model.test.cjs b/eng/regression-bisect/model.test.cjs index 9b91fa79490..53e6726f292 100644 --- a/eng/regression-bisect/model.test.cjs +++ b/eng/regression-bisect/model.test.cjs @@ -51,6 +51,33 @@ for (const key of Object.keys(policy().limits)) { } } +for (const [section, field, value] of [ + ["limits", "sourceBuilds", 25], + ["authorization", "mode", "issue-instructions"], + ["selection", "createdAtOrAfter", "2025-09-01T00:00:00Z"], +]) { + test(`policy rejects shared JSON mutation: ${section}.${field}`, () => { + const shared = require("../../.github/policies/regression-bisect.json"); + const original = shared[section][field]; + const modelPath = require.resolve("./model.cjs"); + const cachedModel = require.cache[modelPath]; + try { + shared[section][field] = value; + delete require.cache[modelPath]; + for (const model of [cachedModel.exports, require("./model.cjs")]) { + rejects(() => model.validatePolicy(shared), /policy/); + assert.equal(model.isEligibleIssue({ + state: "open", created_at: "2025-09-01T00:00:00Z", labels: ["Regression"], + }), false); + } + } finally { + require.cache[modelPath] = cachedModel; + shared[section][field] = original; + } + assert.equal(validatePolicy(shared), true); + }); +} + for (const [name, edit] of [ ["schema", m => m.schemaVersion = 2], ["unknown field", m => m.instructions = "run issue commands"], From 404ba867a07f4c9461d61d790c9323a6dd032a19 Mon Sep 17 00:00:00 2001 From: Copilot Date: Fri, 18 Sep 2026 23:38:22 +0200 Subject: [PATCH 05/99] Accept Windows process statuses and strengthen admission coverage Preserve unsigned Windows DWORD exit statuses as well as existing signed statuses without normalizing exact frozen assertions. Cover valid boundaries, malformed statuses, repeated classification, and source observation validation. Make the same-source fixture internally consistent and exercise mixed matrix outcomes at all three endpoints, including rejection of forged aggregates. Production admission logic and shared fixtures remain unchanged. RED: node --test --test-reporter=tap eng\regression-bisect\model.test.cjs failed four Windows status cases (246 passed) before the validator fix. GREEN: node --test eng\regression-bisect\model.test.cjs passed all 250 tests. All three CommonJS syntax checks and committed/working diff checks passed. Disposable mutations passed all 218 original tests. Corrected tests detect removed strict ancestry (1 failure) and first-cell-only aggregation (12 failures). No compiler builds, workflow activation, live issue execution, or push. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 3 + eng/regression-bisect/model.cjs | 2 +- eng/regression-bisect/model.test.cjs | 100 +++++++++++++++++++++------ 3 files changed, 81 insertions(+), 24 deletions(-) diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index 255ed06b9fd..a42b3c15253 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -82,6 +82,9 @@ not executable assets or approval sources. Assertions are exact `{ exitCode, stdout, stderr }` triples. Both streams are bounded by the policy and compared without trimming or diagnostic normalization. +Exit codes preserve signed 32-bit statuses and unsigned Windows DWORD statuses +(`-2147483648` through `4294967295`); signed and unsigned representations are not +normalized into equivalent assertions. A nonzero exit, unrelated exception, truncated output, or missing evidence is not proof of the frozen product failure. Required restore/build/setup success is separate from the reproduction assertion: an expected compiler rejection can diff --git a/eng/regression-bisect/model.cjs b/eng/regression-bisect/model.cjs index dd2d1bdfdde..f94c4614b25 100644 --- a/eng/regression-bisect/model.cjs +++ b/eng/regression-bisect/model.cjs @@ -113,7 +113,7 @@ function evidenceText(value, path) { path, "missing or oversized assertion evidence"); } -const exitCode = (v, p) => check(Number.isSafeInteger(v) && v >= -2147483648 && v <= 2147483647, p, "invalid exit code"); +const exitCode = (v, p) => check(Number.isSafeInteger(v) && v >= -2147483648 && v <= 4294967295, p, "invalid exit code"); const assertion = shape({ exitCode, stdout: evidenceText, stderr: evidenceText }); const identity = shape({ name: text, version: pinnedVersion, sha256: digest }); const os = oneOf(["linux", "windows", "macos"]); diff --git a/eng/regression-bisect/model.test.cjs b/eng/regression-bisect/model.test.cjs index 53e6726f292..8ea28974a54 100644 --- a/eng/regression-bisect/model.test.cjs +++ b/eng/regression-bisect/model.test.cjs @@ -188,7 +188,10 @@ for (const [name, edit] of [ ["released-product-only good", a => a.good = observation("released", "pass", a.manifest)], ["nonancestor", a => a.ancestry.isAncestor = false], ["unknown ancestry", a => delete a.ancestry.isAncestor], - ["same source", a => a.good.target.sourceSha = a.main.target.sourceSha], + ["same source", a => { + a.good = observation("source", "pass", a.manifest, a.main.target.sourceSha); + a.ancestry.goodSha = a.main.target.sourceSha; + }], ["wrong main SHA", a => a.ancestry.pinnedMainSha = sha("other")], ["wrong matrix", a => a.good.matrixFingerprint = digest("other")], ["wrong producer", a => a.good.producerFingerprint = digest("other")], @@ -280,31 +283,82 @@ test("source builds may use pinned prerelease toolchains without treating them a assert.equal(admitBisection(a).decision, "admit"); }); -test("assertions preserve signed process exit codes", () => { - const m = manifest(); - m.assertions.failure.exitCode = -1; - seal(m); - const attempts = [attempt("product-failure"), attempt("product-failure")]; - for (const a of attempts) a.execution.exitCode = -1; - assert.equal(classifyAttempts(attempts, m).outcome, "product-failure"); -}); +for (const exitCode of [-2147483648, -1073741819, -1, 0, 2147483647, 2147483648, 3221225477, 4294967295]) { + test(`assertions preserve process exit code ${exitCode}`, () => { + const m = manifest(); + m.matrix[0].id = "windows-x64"; + m.matrix[0].os = "windows"; + m.assertions.failure.exitCode = exitCode; + seal(m); + assert.equal(validateManifest(m), true); + const o = observation("source", "product-failure", m); + for (const a of o.cells[0].attempts) a.execution.exitCode = exitCode; + o.cells[0].outcome = o.outcome = classifyAttempts(o.cells[0].attempts, m); + assert.deepEqual(o.outcome, { schemaVersion: 1, outcome: "product-failure", reason: "failure-assertion" }); + assert.equal(validateObservation(o, m), true); + }); +} -test("every matrix cell must reproduce consistently, not just the first cell", () => { - const a = admission(); - a.manifest.matrix.push({ ...a.manifest.matrix[0], id: "debug", configuration: "Debug" }); - seal(a.manifest); - a.stable = observation("released", "product-failure", a.manifest); - a.main = observation("source", "product-failure", a.manifest); - a.good = observation("source", "pass", a.manifest, sha("good")); - a.budget.authorization.manifestDigest = fingerprintInput(a.manifest); - a.budget.authorization.inputDigest = a.manifest.approvedInputDigest; - a.budget.authenticatedObservationDigests = [a.stable, a.main, a.good].map(fingerprintInput); - assert.equal(admitBisection(a).decision, "admit"); - a.good.cells[1].attempts[1] = attempt("product-failure"); - a.budget.authenticatedObservationDigests = [a.stable, a.main, a.good].map(fingerprintInput); - assert.equal(admitBisection(a).decision, "blocked"); +for (const exitCode of [-2147483649, 4294967296, 1.5, "1", null, NaN, Infinity]) { + test(`invalid process exit code ${exitCode} (${typeof exitCode}) is rejected`, () => { + const m = manifest(); + const attempts = [attempt("product-failure"), attempt("product-failure")]; + attempts[0].execution.exitCode = exitCode; + assert.deepEqual(classifyAttempts(attempts, m), { + schemaVersion: 1, outcome: "untestable", reason: "incomplete-evidence", + }); + m.assertions.failure.exitCode = exitCode; + rejects(() => validateManifest(m), /manifest.assertions.failure.exitCode: invalid exit code/); + }); +} + +test("signed and unsigned process statuses do not satisfy each other's exact assertion", () => { + for (const [expected, actual] of [[3221225477, -1073741819], [-1073741819, 3221225477]]) { + const m = manifest(); + m.assertions.failure.exitCode = expected; + seal(m); + const attempts = [attempt("product-failure"), attempt("product-failure")]; + for (const a of attempts) a.execution.exitCode = actual; + assert.deepEqual(classifyAttempts(attempts, m), { + schemaVersion: 1, outcome: "untestable", reason: "assertion-mismatch", + }); + } }); +for (const endpoint of ["stable", "main", "good"]) { + for (const [first, second, reason] of [ + ["pass", "product-failure", "flaky"], ["product-failure", "pass", "flaky"], + ["pass", "untestable", "assertion-mismatch"], ["untestable", "pass", "assertion-mismatch"], + ["product-failure", "untestable", "assertion-mismatch"], ["untestable", "product-failure", "assertion-mismatch"], + ]) { + test(`matrix aggregation: ${endpoint} ${first}/${second}`, () => { + const a = admission(); + a.manifest.matrix.push({ ...a.manifest.matrix[0], id: "debug", configuration: "Debug" }); + seal(a.manifest); + a.stable = observation("released", "product-failure", a.manifest); + a.main = observation("source", "product-failure", a.manifest); + a.good = observation("source", "pass", a.manifest, sha("good")); + a.budget.authorization.manifestDigest = fingerprintInput(a.manifest); + a.budget.authorization.inputDigest = a.manifest.approvedInputDigest; + a.budget.authenticatedObservationDigests = [a.stable, a.main, a.good].map(fingerprintInput); + assert.equal(admitBisection(a).decision, "admit"); + const o = a[endpoint]; + const claimed = o.outcome; + for (const [i, result] of [first, second].entries()) { + const cell = o.cells[i]; + cell.attempts = [attempt(result), attempt(result)]; + cell.outcome = classifyAttempts(cell.attempts, a.manifest); + } + o.outcome = { schemaVersion: 1, outcome: "untestable", reason }; + assert.equal(validateObservation(o, a.manifest), true); + a.budget.authenticatedObservationDigests = [a.stable, a.main, a.good].map(fingerprintInput); + assert.deepEqual(admitBisection(a), { decision: "blocked", reason: "untestable endpoint evidence" }); + o.outcome = claimed; + rejects(() => validateObservation(o, a.manifest), /observation.outcome: does not match matrix evidence/); + }); + } +} + test("budget boundary includes the whole worker reservation and retry cost", () => { const a = admission(); a.budget.consumed = { sourceBuilds: 23, workerUnits: 39, reservedMinutes: 610 }; From 1e3e9fb161c0f3a2b84fc0c684d87c866d232198 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 00:06:09 +0200 Subject: [PATCH 06/99] Add bounded regression intake and authenticated checkpoints Read complete issue discussions and linked investigations; resume bounded discovery without cursor starvation. Persist authenticated issue-comment checkpoints with charged reservations, replay-safe completions, cancellation recovery and selective evidence invalidation. Validation: 368 Node tests passed, including serialized checkpoint replay and intake/state integration; scoped syntax and diff checks passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .gitattributes | 1 + eng/regression-bisect/README.md | 202 ++++++- eng/regression-bisect/fixtures/checkpoint.txt | 5 + eng/regression-bisect/intake.cjs | 364 ++++++++++++ eng/regression-bisect/intake.test.cjs | 505 +++++++++++++++++ eng/regression-bisect/state.cjs | 503 ++++++++++++++++ eng/regression-bisect/state.test.cjs | 535 ++++++++++++++++++ 7 files changed, 2111 insertions(+), 4 deletions(-) create mode 100644 eng/regression-bisect/fixtures/checkpoint.txt create mode 100644 eng/regression-bisect/intake.cjs create mode 100644 eng/regression-bisect/intake.test.cjs create mode 100644 eng/regression-bisect/state.cjs create mode 100644 eng/regression-bisect/state.test.cjs diff --git a/.gitattributes b/.gitattributes index 4b84dc7b095..4aee28b4472 100644 --- a/.gitattributes +++ b/.gitattributes @@ -30,6 +30,7 @@ mono/config.make text eol=lf targets.make text eol=lf *.sh text eol=lf +eng/regression-bisect/fixtures/checkpoint.txt text eol=lf *.bsl linguist-vendored=true *.xlf linguist-generated=true diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index a42b3c15253..f3a1576452f 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -1,14 +1,16 @@ # Regression investigation contracts This directory defines version 1 of the deterministic input, evidence, state, -and admission contracts. It does not activate a workflow, execute reproductions, -persist state, or spawn subprocesses. Tests use synthetic records under -`fixtures`; they are not reproduction approvals. +and admission contracts, complete issue intake, and authenticated issue-comment +checkpoints. It does not activate a workflow, execute reproductions, or spawn +subprocesses. GitHub access is injected by the trusted controller; tests use +synthetic records and an in-memory transport, never live issue mutations. +Records under `fixtures` are not reproduction approvals. Run the dependency-free suite from the repository root: ```powershell -node --test eng\regression-bisect\model.test.cjs +node --test eng\regression-bisect\model.test.cjs eng\regression-bisect\intake.test.cjs eng\regression-bisect\state.test.cjs ``` ## Trust boundary @@ -168,3 +170,195 @@ this gate immediately before the first operation and again after any evidence-invalidating change, using the current authenticated context and charged reservation. This contract suite does not claim to prove an engine's subprocess behavior; there is no engine or subprocess execution here. + +## Complete intake and bounded discovery + +`intake.cjs` exports `readIssueSnapshot(options)`, +`discoverEligibleIssues(options)`, and `findActiveInvestigations(options)`. +They use an injected Octokit-style `transport.request(route, params)` returning +`{ data, headers }`. Repository/issue identities and all callbacks come from +the trusted controller, not issue content. The helpers only read GitHub. + +`readIssueSnapshot({ transport, repository, issueNumber, ... })` returns +`{ status: "complete", snapshot }`, or an explicit `ineligible`/`blocked` +result without a partial snapshot. Opened-with-label, late human/bot labeling, +scheduled recovery, and manual requests use the same shared exact eligibility +predicate; no trigger authorizes a reproduction. + +The snapshot reads metadata, **all** comments, and timeline/cross-references. +It preserves stable comment identities, current text, edits, corrections, +labels, and relevant issue state. Its `input`, `canonicalInput`, `inputDigest`, +and `discussionFingerprint` describe the complete frozen input; issue +`updated_at` alone is not an input identity. State's `issueFingerprint` uses +`snapshot.discussionFingerprint`. Snapshot `inputDigest` is a collection +fingerprint, distinct from the manifest's `approvedInputDigest`. + +Optional `reviewedManifest` must match that discussion fingerprint. +`readSource({ path, sha256 })` must return the raw reviewed bytes from the pinned +trusted checkout; intake verifies their hashes and freezes the reproduction, +exact pass/failure assertions, and dependency identities. A snapshot without a +reviewed reproduction is suitable for a proposal, not execution. The state +helpers separately require today's matching default-branch approval. + +Optional `authenticateRecord(comment)` must delegate to authenticated state/ +publication provenance. Only exact `true` excludes a record. No callback means +**nothing is excluded**. Markers and copied JSON remain untrusted input. +The integration test uses the actual signed checkpoint fixture to verify that +workflow records are excluded while reporter copies remain hashed. +Production dispatch must supply both authentication and trusted-checkpoint +callbacks. Once a record authenticates, omitting `readTrustedCheckpoints` blocks +the freeze. Omitting both callbacks is only untrusted-input collection, not +duplicate-safe dispatch. + +`findActiveInvestigations` examines linked issues/PRs, their current status and +activity, and trusted checkpoints. Suppression requires an open/unmerged target +active within 14 days and a timeline cross-reference or verified backlink; an +arbitrary "already investigating" comment is not a permanent veto. +The controller must inspect `snapshot.investigations.active` before dispatch. +The result records links, status, and the basis for suppression; unreadable +required targets block intake rather than silently disappearing. + +To supply trusted checkpoints, adapt `loadCheckpoint()` into +`readTrustedCheckpoints` records +`{ status: "loaded", state, pending, commentId, owner: { runId, attempt } }`, +using `result.checkpoint.state` and `result.checkpoint.pending`. Required read/ +evidence failures must propagate as blocked, not as an absent checkpoint. +Use `result.commentId` and `result.owner`, which are recovered from authenticated +provenance, rather than substituting the new controller's run identity. +Returning `[]` asserts that a complete trusted-checkpoint lookup found none. +Pending ownership takes precedence; the specific owning Actions run attempt is +inspected before suppressing or reclaiming work. + +Default safety bounds are 20 comment pages, 20 timeline pages, 100 linked +targets, 4 MiB of canonical input, and 1 MiB of aggregate source bytes. +Overrides can only lower these bounds. Exceeding a bound, incomplete pagination, +or an API failure blocks the freeze; first-page truncation is never accepted. + +### Discovery cursor lifetime + +`discoverEligibleIssues({ transport, repository, cursor, scheduleOrdinal })` +uses created-ordered REST issue listing, not Search's 1,000-result-limited +index. It returns `{ status, issues, nextCursor }`, with at most two pages of +100 records and three candidates. API label filtering only narrows the query; +every candidate still passes `isEligibleIssue`. + +Persist the returned `{ version, page, offset }` cursor after consuming its +candidates. The offset retains overflow candidates on a page rather than +skipping them when three have been selected. Traversal resets and revisits old +eligible creation dates so later bot labeling, including issues created exactly +at the cutoff, is eventually discovered. + +Supply a monotonic schedule ordinal independently of the cursor cache (for +example, the scheduled UTC slot number from a fixed epoch). Lost or corrupt +cursors use expanding triangular page rounds and rotating three-item offsets; +repeated cache loss must not always return page one's first three issues. +The cursor is only a discovery optimization. Its loss never initializes a new +allowance, removes a checkpoint, or changes consumed budgets. Event/manual +paths and discovery candidates still enter the same serialized issue +transaction before any dispatch. + +## Authenticated comment checkpoints + +`state.cjs` wraps the shared model state rather than introducing a second model +or changing admission. It keeps the full observations and their manifests, +ordered reservation/completion/cancellation records, cumulative charges, +per-item attempts, pending owner/request/deadline, endpoint epoch history, +intervals, skips, and publication digest/comment ID. Identity tables intern +repeated objects and long assertion streams without dropping observations. +The entire signed UTF-8 comment must be **less than 48 KiB**. A write that cannot +fit blocks further work; it never discards evidence or refunds a reservation. +`fixtures/checkpoint.txt` is a concrete signed checkpoint using a **synthetic +test key**, replayed through reservation, commit, cancellation, and resume. +Its checkout line endings are pinned to LF so Windows Git settings cannot +change the authenticated bytes. + +The trusted controller supplies `context`: + +* `repository`, `issueNumber`, and the exact authoritative `commentId`; + `actor: { id, login, type, appId }` are pinned numeric GitHub author/app + identities, not just the shared Actions bot display name. +* `workflowPath`, `workflowRef` (for example `refs/heads/main`), + `workflowRevision`, and `trustedWorkflowRevisions`. Old orchestration + revisions may be retained only after independent maintainer review. +* `keys` maps key IDs to secret `Buffer`s of at least 32 bytes; `keyId` chooses + the current signing key. Production keys belong only to the trusted + controller's secret store, never this repository, issue input, source builds, + reproduction processes, worker output, or caches. +* `owner: { runId, attempt }`, with a decimal-string run ID; UTC `now`; + the freshly read `issue` and complete `issueFingerprint`; today's reviewed + `manifest`, `authorization`, `policy`, `policyRevision`, `pinnedMainSha`, and + `stableSdk`. These have the shared model shapes above. Missing or mismatched + current approval blocks execution even if the old checkpoint is authentic. + +The transport implements these asynchronous methods: + +| Method | Required result | +| --- | --- | +| `getComment(commentId)` | Full REST issue-comment object: `id`, `body`, `issue_url`, `user`, and `performed_via_github_app`. Missing comments throw, never return empty state. | +| `updateComment(commentId, body)` | The full updated comment. The helper verifies the returned body and provenance before granting execution. | +| `getRun(runId, attempt)` | The **specific run attempt**, with REST `id`, `run_attempt`, `path`, `head_sha`, `head_branch`, `repository.full_name`, `head_repository.full_name`, `status`, and `conclusion`. Do not silently substitute the latest rerun. | +| `readEvidence(url)` | Verified downloaded bytes as a `Buffer`, or `null` for confirmed missing/expired content. Authentication, rate-limit, and network errors throw. The controller restricts downloads to its evidence stores. | + +The versioned envelope has a human-readable authenticated status, a compact +payload, and HMAC-SHA256 over its canonical contents. Its signed binding includes +the exact comment, issue, repository, workflow path/ref/revision, and run attempt. +Reading also checks GitHub author/app identities and independently fetches the +bound run. Copying the marker, envelope, or bot name to another comment does not +establish authority. Untrusted workers must not receive the signing key or +construct the controller context. `commitObservation` accepts observations +collected and authenticated by that controller, not arbitrary reproduction +stdout presented as a worker result. + +The controller must retain or rediscover the **same authenticated comment ID**, +requiring a unique valid checkpoint for the issue; a missing or ambiguous record +blocks resumption. Never adopt the newest marker or bootstrap because a cache +missed. `initializeCheckpoint(transport, context, { allowanceId, deadline })` +is only for the initial maintainer-authorized allowance, under the issue lock. +`context.bootstrapDigest` must come from the trusted creation receipt for the +exact initial placeholder, not from hashing whichever comment is currently +present. Initialization refuses an existing envelope. There is deliberately no +automatic reset/new-allowance operation; ordinary issue edits, label toggles, +redispatches, and cancellations cannot replenish any counter. + +### Controller transaction + +Hold one issue-scoped controller lock across **read, reserve, worker, commit, +and report**. GitHub comment updates are not compare-and-swap. The generation +and owner checks below add stale-request protection; they do not replace that +lock. All state-transition calls reread the authoritative comment. + +| Export | Behavior | +| --- | --- | +| `loadCheckpoint(transport, context)` | Returns `{ checkpoint, commentId, owner, executionAllowed, publicationAllowed, blockers, warnings }`. Invalid authentication/schema throws; stale approvals/inputs or required evidence loss explicitly block reuse. | +| `reserveWork(transport, context, { generation, attemptId, request })` | Durably charges and saves the exact request before returning a reservation with `execute: true`. Duplicate requests return `execute: false`; they **must not launch another worker**. Conflicting attempt IDs, stale generations, active pending owners, exhausted counters, and deadlines fail closed. | +| `commitObservation(transport, context, { generation, attemptId, observation, evidence })` | Checks ownership, request/build/provenance and the shared observation contract, then commits all decisive evidence before clearing pending work. Exact duplicate completion is idempotent. | +| `invalidateStaleEvidence(transport, context, { generation })` | Invalidates changed input/approval/producer/orchestration dependencies without erasing history or charges. Main/stable changes start a new endpoint epoch while retaining exact-input historical source observations. Closed/unlabeled issues stop publication; existing result identities remain auditable. | +| `serializeWorkflowRecord({ kind, status, payload }, context)` | Signs a `checkpoint`, `proposal`, or `result` envelope. Low-level serialization is not approval or a storage transition. | +| `authenticateWorkflowRecord(transport, context, comment)` | Returns true only for a matching signed record and fetched run provenance. Forged records return false; required API failures throw. Bind the expected record ID before using this as intake's exclusion callback. | + +A work request is +`{ kind, target, endpoint, phase }`: `kind`/`target` use the observation shape; +`endpoint` is `stable`, `main`, `good`, or `null`; `phase` is `stable`, `main`, +`good-search`, `admission`, `bisect`, or `confirmation`. Work-item identity +binds kind, target, and approved input digest, so renaming a request or changing +its role cannot buy extra retries. Every reservation charges one worker unit, +one source build for source work, and the complete policy worker timeout +(110 minutes). The policy limits remain 24 builds, 40 worker units, +720 cumulative reserved minutes, and two attempts per item; the minutes limit +can bind before the other limits. + +The worker uses the returned reservation ID as its `buildInstanceId` and returns +the exact pinned request's observation. `evidence` contains +`{ required, sha256, url }` entries for external evidence/logs. Complete assertion +evidence and product/environment identities are retained in the comment itself. +Missing required external evidence blocks execution/publication; optional +expired logs produce an explicit warning. Neither artifacts nor Actions caches +are authoritative memory. + +On restart, an outstanding reservation is reclaimed only after fetching its +owning run attempt and establishing `status: completed` with a conclusion. +Queued, active, unknown, or unreadable owners remain blocking even after their +deadline. No cancellation finalizer is required. The old reservation stays +charged and a retry consumes another attempt/unit/time reservation; at most the +one uncommitted observation repeats. If a write succeeded but its acknowledgement +was lost, retrying that attempt does not grant another execution permission. diff --git a/eng/regression-bisect/fixtures/checkpoint.txt b/eng/regression-bisect/fixtures/checkpoint.txt new file mode 100644 index 00000000000..12a3864c8fa --- /dev/null +++ b/eng/regression-bisect/fixtures/checkpoint.txt @@ -0,0 +1,5 @@ +Regression investigation: discovery (generation 0). + +```json +{"schemaVersion":1,"kind":"checkpoint","status":"Regression investigation: discovery (generation 0).","binding":{"repository":"dotnet/fsharp","issueNumber":123,"commentId":456,"workflowPath":".github/workflows/regression-bisect.yml","workflowRef":"refs/heads/main","workflowRevision":"899531e0f885501b5b8520a9bd95bc83345eaa5b","runId":"12345","runAttempt":1,"keyId":"fixture"},"payload":{"root":["@",0],"table":[{"schemaVersion":1,"generation":0,"state":["@",1],"inputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a","reproductionDigest":"91815101645c42988cb8f780311ed49cd7c179071d76d17298a1c6810a965138","policyDigest":"deb550568d7b07a6d49bcbc92690ee602abc201b76329631f0dee9387b2ea09f","sequence":0,"pending":null,"records":["@",28],"observations":["@",32],"history":["@",28],"deadline":"2026-10-01T00:00:00Z","publicationStatus":"unpublished"},{"schemaVersion":1,"repository":"dotnet/fsharp","issueNumber":123,"issueFingerprint":"ba79e2bbb07eaf308944401cb8a62139d3eed051e25c88279ce781531fef37f5","approvalFingerprint":"16503092fdfbb9143595739dc023ca6e201f80203f63950dd884370b09878dcb","manifest":["@",2],"budget":["@",18],"pinnedMainSha":"b28b7af69320201d1cf206ebf28373980add1451","epoch":"09eea1ee5aea50efd7ac23639a5050c834e1824d056cbcc4b054532e8c9a02a3","phase":"discovery","endpointEvidence":["@",30],"testedRevisions":["@",28],"skips":["@",28],"interval":null,"publication":["@",31]},{"schemaVersion":1,"repository":"dotnet/fsharp","issueNumber":123,"discussionFingerprint":"ba79e2bbb07eaf308944401cb8a62139d3eed051e25c88279ce781531fef37f5","kind":"fsi","sources":["@",3],"assertions":["@",5],"matrix":["@",8],"timeoutSeconds":300,"producer":["@",11],"packages":["@",12],"searchHints":["@",14],"approvedInputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a"},[["@",4]],{"path":"fixtures/repro.fsx","sha256":"e1c5329a6b6f65fb0733f62c334e0bf4bb91cfa449f5eb213dc67f0d40b440af"},{"pass":["@",6],"failure":["@",7]},{"exitCode":0,"stdout":"ok\n","stderr":""},{"exitCode":1,"stdout":"","stderr":"expected regression\n"},[["@",9]],{"id":"linux-x64","os":"linux","architecture":"x64","project":null,"tfm":"net10.0","configuration":"Release","languageVersion":"10.0","command":["@",10]},["fsi","fixtures/repro.fsx"],{"name":"repro","version":"1.0.0","sha256":"d1990f97a1f4fe7b8e86da8c03c9dd3f7d536c14aa4a6f9eedab07f34cd961b0"},[["@",13]],{"id":"Example","version":"1.0.0","sha256":"bc4a71180870f7945155fbb02f4b0a2e3faa2a62d6d31b7039013055ed19869a"},{"sdkVersions":["@",15],"sourceShas":["@",16],"urls":["@",17]},["9.0.100"],["fc19318dd13128ce14344d066510a982269c241b"],["https://github.com/dotnet/fsharp"],{"schemaVersion":1,"policy":["@",19],"policyRevision":"9f00fad98badc1330e637a0a015d704113498420","workflowRevision":"899531e0f885501b5b8520a9bd95bc83345eaa5b","allowanceId":"maintainer-approved-investigation-1","authorization":["@",24],"stableSdk":["@",25],"authenticatedObservationDigests":["@",28],"consumed":["@",29],"attempts":["@",28],"nextWorkItemId":"none","pendingWork":null},{"schemaVersion":1,"repository":"dotnet/fsharp","selection":["@",20],"authorization":["@",21],"accounting":["@",22],"limits":["@",23],"evidenceBytesPerStream":16384},{"state":"open","label":"Regression","createdAtOrAfter":"2026-09-01T00:00:00Z"},{"mode":"reviewed-manifests-only","policySource":"pinned-trusted-default-branch","approvalDirectory":".github/regression-bisect/approved","assets":"reviewed-repository-paths-and-sha256"},{"scope":"issue-allowance","reserveBeforeExecution":true,"retriesCharged":true,"cancellationRefunds":false,"inputEditsResetAllowance":false,"labelChangesResetAllowance":false,"dispatchResetsAllowance":false,"newAllowanceRequires":"maintainer-authorization"},{"sourceBuilds":24,"workerUnits":40,"reservedMinutes":720,"attemptsPerItem":2,"assertionRepetitions":2,"buildTimeoutMinutes":90,"reproductionTimeoutSeconds":300,"workerTimeoutMinutes":110,"discoveryIssues":3,"discoveryPages":2,"discoveryPageSize":100},{"repository":"dotnet/fsharp","issueNumber":123,"approvalPath":".github/regression-bisect/approved/123.json","approvalRevision":"9f00fad98badc1330e637a0a015d704113498420","manifestDigest":"16503092fdfbb9143595739dc023ca6e201f80203f63950dd884370b09878dcb","inputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a"},{"version":"10.0.100","origin":"official-release-index","latest":true,"verified":true,"releaseIndexUrl":"https://builds.dotnet.microsoft.com/dotnet/release-metadata/releases-index.json","releaseIndexDigest":"2a21ef65c47867ae1b2056a246bf7c001b12ade326bd0e1fcdfb81af140e65b7","archives":["@",26]},[["@",27]],{"os":"linux","architecture":"x64","sha256":"9d9e20df50a502c04ee629c1282495a65a279635168510572f846de604da046e"},[],{"sourceBuilds":0,"workerUnits":0,"reservedMinutes":0},{"stable":null,"main":null,"good":null},{"digest":null,"commentId":null},{}]},"signature":"d815fab6dc3815c19a9f221efb0216087b01acf3deedfb3961b7318ae36a0ad4"} +``` diff --git a/eng/regression-bisect/intake.cjs b/eng/regression-bisect/intake.cjs new file mode 100644 index 00000000000..013eec6a316 --- /dev/null +++ b/eng/regression-bisect/intake.cjs @@ -0,0 +1,364 @@ +const { createHash } = require("node:crypto"); +const { isEligibleIssue, validateManifest, fingerprintInput } = require("./model.cjs"); + +const hardLimits = Object.freeze({ + commentPages: 20, timelinePages: 20, linkedTargets: 100, + inputBytes: 4 * 1024 * 1024, sourceBytes: 1024 * 1024, +}); +const requireValue = (condition, message) => { if (!condition) throw new Error(message); }; +const blocked = error => ({ status: "blocked", reason: error.message }); +const positive = value => Number.isSafeInteger(value) && value > 0; +const recordId = value => positive(value) || (typeof value === "string" && /^[1-9]\d*$/.test(value)); +const repoParts = repository => { + requireValue(typeof repository === "string" && /^[\w.-]+\/[\w.-]+$/.test(repository), "invalid repository"); + const [owner, repo] = repository.split("/"); + return { owner, repo }; +}; + +function limitsFor(options) { + const limits = { ...hardLimits }; + for (const [name, value] of Object.entries(options.limits ?? {})) { + requireValue(Object.hasOwn(hardLimits, name) && positive(value) && value <= hardLimits[name], `invalid ${name} limit`); + limits[name] = value; + } + return limits; +} + +async function request(options, route, params) { + const response = await options.transport.request(`GET /repos/{owner}/{repo}/${route}`, params); + requireValue(response && (!response.status || response.status < 400) && response.data !== undefined, `incomplete API response: ${route}`); + return response; +} + +async function page(options, route, params, number) { + const response = await request(options, route, { ...params, per_page: 100, page: number }); + requireValue(Array.isArray(response.data) && response.data.length <= 100, `invalid API page: ${route}`); + return { + rows: response.data, + next: response.headers?.link === undefined ? response.data.length === 100 + : /;\s*rel="?next"?/.test(response.headers.link), + }; +} + +async function discussionPages(options, kind, params, limits) { + const rows = []; + let size = 0; + for (let number = 1; number <= limits[`${kind === "comments" ? "comment" : "timeline"}Pages`]; number++) { + const result = await page(options, `issues/{issue_number}/${kind}`, params, number); + size += Buffer.byteLength(JSON.stringify(result.rows)); + requireValue(size <= limits.inputBytes, `${kind} input byte limit exceeded`); + rows.push(...result.rows); + if (!result.next) return rows; + } + throw new Error(`${kind} page limit exceeded; discussion incomplete`); +} + +function normalizeComment(record) { + requireValue(recordId(record.id), "comment missing stable ID"); + requireValue(typeof record.body === "string", "comment missing body"); + requireValue(record.user && recordId(record.user.id), "comment missing writer identity"); + return Object.fromEntries(["id", "node_id", "body", "created_at", "updated_at", "user", "performed_via_github_app", "author_association"] + .map(key => [key, record[key] ?? null])); +} + +function orderedRecords(records) { + const byId = new Map(); + for (const record of records) { + const id = `${record.event ?? "comment"}:${record.id ?? record.node_id ?? record.sha ?? fingerprintInput(record)}`; + const prior = byId.get(id); + requireValue(!prior || fingerprintInput(prior) === fingerprintInput(record), "conflicting duplicate record during pagination"); + byId.set(id, record); + } + return [...byId].sort(([a], [b]) => a.localeCompare(b, "en", { numeric: true })).map(([, value]) => value); +} + +function issueInput(issue) { + requireValue(positive(issue.number) && recordId(issue.id) && typeof issue.title === "string" && + (issue.body === null || typeof issue.body === "string"), "incomplete issue metadata"); + // updated_at and comment count also change for authenticated workflow progress. + return { + id: issue.id, number: issue.number, title: issue.title, body: issue.body, + state: issue.state, state_reason: issue.state_reason ?? null, created_at: issue.created_at, + closed_at: issue.closed_at ?? null, locked: issue.locked ?? false, user: issue.user ?? null, + labels: issue.labels.map(label => typeof label === "string" ? label : label.name).sort(), + }; +} + +/** + * All APIs take {transport: {request(route, params)}, repository, issueNumber}. + * request is Octokit's read-only request interface and returns {data, headers?}. + * Limits may only lower the finite defaults above; every failure returns blocked, + * never partial frozen input. No callback means no workflow record is trusted. + * + * authenticateRecord(comment) must asynchronously return exactly true only after + * authenticating the exact ID, body, writer/app and trusted workflow/run provenance. + * It is also called for timeline "commented" records; marker/author tests are not + * authentication. Any authenticated record requires readTrustedCheckpoints before + * freezing. Production controllers must supply both callbacks; omitting them is + * only suitable for untrusted read-only collection, never duplicate-safe dispatch. + * readTrustedCheckpoints is documented by findActiveInvestigations. + * + * reviewedManifest is supplied only by a trusted default-branch approval loader. + * readSource({path, sha256}) reads raw Buffer/Uint8Array bytes at that same pinned + * revision, enforcing tracked-blob/path/symlink policy. Intake checks every hash. + * First read without a manifest to obtain discussionFingerprint; a supplied + * manifest must match it. A complete snapshot by itself never authorizes work. + * canonicalInput is UTF-8 JSON; inputDigest hashes those bytes using model rules, + * not the manifest's distinct approvedInputDigest. Investigation status is kept + * outside input so authenticated checkpoint progress cannot invalidate itself. + */ +async function readIssueSnapshot(options) { + try { + const limits = limitsFor(options); + const params = { ...repoParts(options.repository), issue_number: options.issueNumber }; + requireValue(positive(options.issueNumber), "invalid issue number"); + const { data: issue } = await request(options, "issues/{issue_number}", params); + requireValue(issue?.number === options.issueNumber, "issue identity mismatch"); + if (!isEligibleIssue(issue)) return { status: "ineligible", reason: "issue does not meet exact eligibility" }; + const rawComments = await discussionPages(options, "comments", params, limits); + const rawTimeline = await discussionPages(options, "timeline", params, limits); + const comments = []; + const timeline = []; + const authenticate = async record => { + const authenticated = await options.authenticateRecord?.(structuredClone(record)) === true; + requireValue(!authenticated || typeof options.readTrustedCheckpoints === "function", + "trusted checkpoint reader required for authenticated records"); + return authenticated; + }; + for (const record of rawComments) { + const normalized = normalizeComment(record); + if (!(await authenticate(record))) comments.push(normalized); + } + for (const record of rawTimeline) { + requireValue(record && typeof record.event === "string", "incomplete timeline event"); + if (record.event === "commented") { + const normalized = normalizeComment(record); + if (!(await authenticate(record))) timeline.push({ ...normalized, event: record.event }); + } else { + timeline.push(record); + } + } + const discussion = { issue: issueInput(issue), comments: orderedRecords(comments), timeline: orderedRecords(timeline) }; + const discussionFingerprint = fingerprintInput(discussion); + let reproduction = null; + if (options.reviewedManifest !== undefined) { + const manifest = structuredClone(options.reviewedManifest); + validateManifest(manifest); + requireValue(manifest.repository === options.repository && manifest.issueNumber === options.issueNumber, "manifest issue mismatch"); + requireValue(manifest.discussionFingerprint === discussionFingerprint, "reviewed discussion fingerprint is stale"); + requireValue(typeof options.readSource === "function", "reviewed source reader required"); + const sources = []; + let size = 0; + for (const source of manifest.sources) { + const bytes = await options.readSource({ ...source }); + requireValue(bytes instanceof Uint8Array, `source ${source.path}: raw bytes required`); + size += bytes.byteLength; + requireValue(size <= limits.sourceBytes, "source byte limit exceeded"); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + requireValue(sha256 === source.sha256, `source ${source.path}: hash mismatch`); + sources.push({ ...source, bytesBase64: Buffer.from(bytes).toString("base64") }); + } + reproduction = { manifest, sources }; + } + const investigations = await findActiveInvestigations({ ...options, issue, comments: rawComments, timeline: rawTimeline }); + if (investigations.status !== "complete") return investigations; + const input = { schemaVersion: 1, repository: options.repository, discussion, reproduction }; + const inputDigest = fingerprintInput(input); + const canonicalInput = JSON.stringify(input, (_, value) => value && typeof value === "object" && !Array.isArray(value) + ? Object.fromEntries(Object.keys(value).sort().map(key => [key, value[key]])) : value); + requireValue(Buffer.byteLength(canonicalInput) <= limits.inputBytes, "canonical input byte limit exceeded"); + return { status: "complete", snapshot: { input, canonicalInput, inputDigest, discussionFingerprint, investigations } }; + } catch (error) { + return blocked(error); + } +} + +function issueLink(value) { + if (typeof value !== "string") return null; + const match = /^https:\/\/github\.com\/([\w.-]+\/[\w.-]+)\/(issues|pull)\/([1-9]\d*)(?:[/?#]|$)/.exec(value); + if (!match || !positive(Number(match[3]))) return null; + return { repository: match[1], kind: match[2] === "pull" ? "pull" : "issue", number: Number(match[3]) }; +} + +/** + * Supply complete current issue/comments/timeline; normally called by snapshot. + * GitHub cross-references establish a relationship; prose URLs require a backlink + * in the freshly read target. Open, unmerged targets updated within 14 days count. + * Arbitrary claims of ongoing work have no authority. Required linked reads block. + * + * readTrustedCheckpoints({repository, issueNumber, comments}) is an injected, + * authenticated reader returning an array of {status:"loaded", state, commentId, + * owner:{runId,attempt}, pending?:{owner}} or {status:"absent"} load results. + * A blocked result/throw blocks intake. An empty array means a complete lookup + * found no trusted checkpoints, not permission to ignore a known checkpoint. + * Owner is the authenticated checkpoint writer, not the new triggering controller. + * Pending ownership takes priority. Every live owning + * run is re-read; only nonterminal state with that exact active attempt suppresses + * duplication. Omission recognizes no trusted checkpoints (never trusts markers). + * Adapt state.loadCheckpoint's successful result by taking state and pending from + * result.checkpoint, and its authenticated result.commentId and result.owner. + * Never substitute the new controller's owner or trust an envelope's claims. + * Propagate read errors or unavailable required evidence as a throw or + * status:"blocked"; do not + * turn them into an empty array. This reader does not grant execution approval. + * now may be a timestamp string or epoch milliseconds for deterministic tests. + */ +async function findActiveInvestigations(options) { + try { + const limits = limitsFor(options); + const targets = new Map(); + const add = (link, basis) => { + if (!link || (link.repository === options.repository && link.number === options.issueNumber)) return; + const key = `${link.repository}#${link.number}`; + const existing = targets.get(key); + targets.set(key, { ...link, basis: existing?.basis === "timeline-cross-reference" ? existing.basis : basis }); + requireValue(targets.size <= limits.linkedTargets, "linked target limit exceeded"); + }; + for (const event of options.timeline) { + if (event.event === "cross-referenced" && event.source?.issue) { + const link = issueLink(event.source.issue.html_url); + requireValue(link, "required linked timeline target is unreadable"); + add(link, "timeline-cross-reference"); + } + } + for (const record of [options.issue, ...options.comments]) { + for (const match of (record.body ?? "").matchAll(/https:\/\/github\.com\/[\w.-]+\/[\w.-]+\/(?:issues|pull)\/\d+\b/g)) { + add(issueLink(match[0]), "reciprocal-reference"); + } + } + const now = options.now === undefined ? Date.now() : typeof options.now === "number" ? options.now : Date.parse(options.now); + requireValue(Number.isFinite(now), "invalid activity timestamp"); + const links = []; + const active = []; + for (const target of targets.values()) { + let value; + const params = repoParts(target.repository); + try { + if (target.kind === "pull") { + value = (await request(options, "pulls/{pull_number}", { ...params, pull_number: target.number })).data; + } else { + value = (await request(options, "issues/{issue_number}", { ...params, issue_number: target.number })).data; + if (Object.hasOwn(value, "pull_request")) { + target.kind = "pull"; + value = (await request(options, "pulls/{pull_number}", { ...params, pull_number: target.number })).data; + } + } + } catch (error) { + throw new Error(`required linked target ${target.repository}#${target.number}: ${error.message}`); + } + requireValue(value?.number === target.number && ["open", "closed"].includes(value.state), "incomplete linked target status"); + requireValue(target.kind !== "pull" || value.merged_at === null || typeof value.merged_at === "string", + "incomplete linked PR merged status"); + const updated = Date.parse(value.updated_at); + requireValue(Number.isFinite(updated) && updated <= now, "invalid linked target activity"); + const text = `${value.title ?? ""}\n${value.body ?? ""}`; + const reciprocal = [...text.matchAll(/https:\/\/github\.com\/[\w.-]+\/[\w.-]+\/issues\/\d+\b/g)] + .some(match => { + const link = issueLink(match[0]); + return link?.repository === options.repository && link.number === options.issueNumber; + }) || (target.repository === options.repository && new RegExp(`(?:^|\\s)#${options.issueNumber}(?!\\d)`).test(text)); + const current = { + ...target, state: value.state, merged: !!value.merged_at, updatedAt: value.updated_at, + active: value.state === "open" && !value.merged_at && now - updated <= 14 * 86400000 && + (target.basis === "timeline-cross-reference" || reciprocal), + }; + links.push(current); + if (current.active) active.push(current); + } + const checkpoints = options.readTrustedCheckpoints === undefined ? [] : await options.readTrustedCheckpoints({ + repository: options.repository, issueNumber: options.issueNumber, comments: structuredClone(options.comments), + }); + requireValue(Array.isArray(checkpoints) && checkpoints.length <= limits.linkedTargets, "invalid trusted checkpoint results"); + for (const loaded of checkpoints) { + if (loaded?.status === "absent") continue; + requireValue(loaded?.status === "loaded", `trusted checkpoint blocked: ${loaded?.reason ?? "load failed"}`); + requireValue(loaded.state?.repository === options.repository && loaded.state?.issueNumber === options.issueNumber, + "trusted checkpoint issue mismatch"); + const owner = loaded.pending?.owner ?? loaded.owner; + if (!owner) { + requireValue(!loaded.state.budget?.pendingWork, "trusted checkpoint pending owner missing"); + continue; + } + requireValue(recordId(owner.runId) && positive(owner.attempt), "trusted checkpoint run ownership missing"); + let run; + try { + run = (await request(options, "actions/runs/{run_id}/attempts/{attempt_number}", { + ...repoParts(options.repository), run_id: owner.runId, attempt_number: owner.attempt, + })).data; + } catch (error) { + throw new Error(`required checkpoint run: ${error.message}`); + } + requireValue(String(run.id) === String(owner.runId) && run.run_attempt === owner.attempt, "checkpoint run ownership mismatch"); + requireValue(["completed", "queued", "in_progress", "waiting", "pending", "requested"].includes(run.status), "unknown checkpoint run status"); + if (run.status !== "completed" && !["complete", "blocked", "stopped", "published"].includes(loaded.state.phase)) { + active.push({ repository: options.repository, number: options.issueNumber, commentId: loaded.commentId, + basis: "trusted-checkpoint-active-run", owner: { ...owner }, status: run.status }); + } + } + return { status: "complete", links, active }; + } catch (error) { + return blocked(error); + } +} + +function restartCursor(ordinal) { + requireValue(Number.isSafeInteger(ordinal) && ordinal >= 0, "scheduleOrdinal must be a nonnegative safe integer"); + // Triangular rounds revisit pages 1; 1,2; 1,2,3; ... . Each round rotates + // the three-item starting slot, so repeated cursor loss starves neither a + // later page nor the remaining eligible candidates on a densely eligible page. + const round = Math.floor((Math.sqrt(8 * ordinal + 1) - 1) / 2); + return { version: 1, page: ordinal - round * (round + 1) / 2 + 1, offset: (round % 34) * 3 }; +} + +/** + * Manual/event paths supply issueNumber and re-read it with the shared predicate. + * Scheduled paths omit it: at most two REST created-ascending 100-item pages and + * three identities per sweep. Persist nextCursor only after consuming candidates; + * its page/offset resumes before any overflow candidate, then wraps after the end. + * The cursor is discovery optimization, never issue state or deduplication. + * Missing/corrupt cursor restarts using scheduleOrdinal, a monotonic independent + * schedule slot (not a counter stored with the cursor). Ordinal defaults to zero + * for a one-off restart; scheduled callers MUST supply it to guarantee recovery. + * A shortened page restarts its offset; failures return no candidates and do not + * advance the supplied cursor. Candidate eligibility must be rechecked at intake. + */ +async function discoverEligibleIssues(options) { + try { + const params = repoParts(options.repository); + if (options.issueNumber !== undefined) { + requireValue(positive(options.issueNumber), "invalid issue number"); + const { data } = await request(options, "issues/{issue_number}", { ...params, issue_number: options.issueNumber }); + requireValue(data?.number === options.issueNumber, "issue identity mismatch"); + return { status: "complete", issues: isEligibleIssue(data) ? [{ repository: options.repository, number: data.number }] : [], nextCursor: null }; + } + const saved = options.cursor; + let cursor = saved?.version === 1 && positive(saved.page) && Number.isInteger(saved.offset) && saved.offset >= 0 && saved.offset < 100 + ? { ...saved } : restartCursor(options.scheduleOrdinal ?? 0); + const issues = []; + const seen = new Set(); + for (let reads = 0; reads < 2; reads++) { + const result = await page(options, "issues", { ...params, state: "open", labels: "Regression", sort: "created", direction: "asc" }, cursor.page); + const start = cursor.offset < result.rows.length ? cursor.offset : 0; + for (let index = start; index < result.rows.length; index++) { + const value = result.rows[index]; + requireValue(positive(value?.number), "issue listing missing identity"); + if (isEligibleIssue(value) && !seen.has(value.number)) { + seen.add(value.number); + issues.push({ repository: options.repository, number: value.number }); + } + if (issues.length === 3) { + cursor = index + 1 < result.rows.length ? { ...cursor, offset: index + 1 } + : { version: 1, page: result.next ? cursor.page + 1 : 1, offset: 0 }; + return { status: "complete", issues, nextCursor: cursor }; + } + } + cursor = { version: 1, page: result.next ? cursor.page + 1 : 1, offset: 0 }; + if (!result.next) break; + } + return { status: "complete", issues, nextCursor: cursor }; + } catch (error) { + return { ...blocked(error), issues: [], nextCursor: options.cursor ?? null }; + } +} + +module.exports = { readIssueSnapshot, discoverEligibleIssues, findActiveInvestigations }; diff --git a/eng/regression-bisect/intake.test.cjs b/eng/regression-bisect/intake.test.cjs new file mode 100644 index 00000000000..8970466a446 --- /dev/null +++ b/eng/regression-bisect/intake.test.cjs @@ -0,0 +1,505 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); +const { manifest, seal, digest, state, sha } = require("./fixtures/contracts.cjs"); +const { fingerprintInput, isEligibleIssue } = require("./model.cjs"); +const intake = () => require("./intake.cjs"); +const repository = "dotnet/fsharp"; +const now = "2026-09-18T12:00:00Z"; +const issue = (number = 123, patch = {}) => ({ + id: number + 1000, number, title: "Compiler regression", body: "The result should be ok.", + state: "open", created_at: "2026-09-01T00:00:00Z", updated_at: now, + labels: [{ name: "Regression" }], locked: false, user: { id: 1, login: "reporter" }, + html_url: `https://github.com/${repository}/issues/${number}`, ...patch, +}); +const comment = (id, body = `Comment ${id}`) => ({ + id, node_id: `IC_${id}`, body, created_at: now, updated_at: now, + user: { id: 1, login: "reporter", type: "User" }, +}); +const reference = (number, kind = "pull") => ({ + id: number + 2000, event: "cross-referenced", created_at: now, + source: { issue: { number, html_url: `https://github.com/${repository}/${kind}/${number}` } }, +}); + +function fixture(patch = {}) { + const data = { issue: issue(), comments: [], timeline: [], linked: {}, runs: {}, ...patch }; + const calls = []; + const transport = { + async request(route, args) { + calls.push({ route, args: structuredClone(args) }); + if (data.fail?.(route, args)) throw new Error("API unavailable"); + if (route.endsWith("/comments") || route.endsWith("/timeline")) { + const kind = route.endsWith("/comments") ? "comments" : "timeline"; + const pages = data[`${kind}Pages`]; + const rows = pages ? (pages[args.page - 1] ?? []) + : data[kind].slice((args.page - 1) * 100, args.page * 100); + const hasNext = pages ? args.page < pages.length : args.page * 100 < data[kind].length; + return { data: structuredClone(rows), headers: { link: hasNext ? '; rel="next"' : "" } }; + } + if (route.endsWith("/issues")) { + const rows = data.list ?? []; + return { + data: structuredClone(rows.slice((args.page - 1) * 100, args.page * 100)), + headers: { link: args.page * 100 < rows.length ? '; rel="next"' : "" }, + }; + } + if (route.endsWith("/attempts/{attempt_number}")) { + const run = data.runs[args.run_id]; + if (!run) throw new Error("Missing run"); + return { data: structuredClone(run) }; + } + const number = args.issue_number ?? args.pull_number; + const target = number === 123 && args.repo === "fsharp" ? data.issue : data.linked[number]; + if (!target) throw new Error(`Missing linked issue ${number}`); + return { data: structuredClone(target) }; + }, + }; + return { data, calls, options: { transport, repository, issueNumber: 123, now } }; +} + +async function snapshot(f, extra = {}) { + return intake().readIssueSnapshot({ ...f.options, ...extra }); +} + +function blocked(result, pattern = /./) { + assert.equal(result.status, "blocked"); + assert.match(result.reason, pattern); + assert.equal(result.snapshot, undefined); + assert.equal(result.inputDigest, undefined); +} + +test("reads later corrections and timeline pages before canonicalizing the full snapshot", async () => { + const f = fixture({ + commentsPages: [[comment(1)], [comment(2, "Correction: expected diagnostic, not a crash.")]], + timelinePages: [[{ id: 5, event: "labeled", label: { name: "Regression" } }], [{ id: 6, event: "renamed", rename: { from: "Old", to: "New" } }]], + }); + const result = await snapshot(f); + assert.equal(result.status, "complete"); + assert.equal(result.snapshot.input.discussion.comments[1].body, f.data.commentsPages[1][0].body); + assert.equal(result.snapshot.input.discussion.timeline.length, 2); + assert.equal(result.snapshot.inputDigest, fingerprintInput(result.snapshot.input)); + assert.equal(result.snapshot.inputDigest, digest(result.snapshot.canonicalInput)); + assert.deepEqual(JSON.parse(result.snapshot.canonicalInput), result.snapshot.input); + assert.equal(result.snapshot.discussionFingerprint, fingerprintInput(result.snapshot.input.discussion)); + assert.equal(f.calls.filter(c => c.args.page === 2).length, 2); +}); + +for (const [name, mutate] of [ + ["later correction", f => f.data.comments.push(comment(3, "Use preview language version"))], + ["body edit", f => f.data.comments[0].body = "Corrected reproduction"], + ["edit timestamp", f => f.data.comments[0].updated_at = "2026-09-19T00:00:00Z"], + ["comment deletion", f => f.data.comments.splice(0, 1)], + ["stable identity replacement", f => f.data.comments[0].id = 9], + ["title edit", f => f.data.issue.title = "Corrected title"], + ["body edit on issue", f => f.data.issue.body = "New reproduction"], + ["labels", f => f.data.issue.labels.push({ name: "Compiler" })], + ["timeline correction", f => f.data.timeline.push({ id: 44, event: "reopened" })], +]) { + test(`snapshot digest changes after ${name}`, async () => { + const f = fixture({ comments: [comment(1), comment(2)] }); + const before = await snapshot(f); + mutate(f); + const after = await snapshot(f); + assert.equal(after.status, "complete"); + assert.notEqual(after.snapshot.inputDigest, before.snapshot.inputDigest); + }); +} + +test("canonical ordering ignores API object key order and unrelated issue updated_at", async () => { + const f = fixture({ comments: [comment(1), comment(2)] }); + const before = await snapshot(f); + f.data.issue.updated_at = "2026-09-19T00:00:00Z"; + f.data.issue.comments = 999; + f.data.issue = Object.fromEntries(Object.entries(f.data.issue).reverse()); + f.data.comments.reverse(); + const after = await snapshot(f); + assert.equal(after.snapshot.inputDigest, before.snapshot.inputDigest); +}); + +test("markers and bot identities are input unless async authentication explicitly succeeds", async () => { + const bot = { id: 41898282, login: "github-actions[bot]", type: "Bot" }; + const genuine = { ...comment(8, " trusted run"), user: bot }; + const authenticatedBody = genuine.body; + const forged = { ...comment(9, " trusted run"), user: bot }; + const f = fixture({ comments: [comment(1), genuine, forged], timeline: [{ ...genuine, event: "commented" }, { ...forged, event: "commented" }] }); + const readTrustedCheckpoints = async () => []; + const plain = await snapshot(f); + assert.equal(plain.snapshot.input.discussion.comments.length, 3); + const authenticated = await snapshot(f, { readTrustedCheckpoints, authenticateRecord: async record => { + await Promise.resolve(); + return record.id === 8 && record.body === authenticatedBody && record.user.id === bot.id; + } }); + assert.deepEqual(authenticated.snapshot.input.discussion.comments.map(c => c.id), [1, 9]); + assert.deepEqual(authenticated.snapshot.input.discussion.timeline.map(c => c.id), [9]); + const truthy = await snapshot(f, { authenticateRecord: async () => ({ authenticated: true }) }); + assert.equal(truthy.snapshot.inputDigest, plain.snapshot.inputDigest); + f.data.comments[1].body = "Edited impersonation"; + const edited = await snapshot(f, { readTrustedCheckpoints, authenticateRecord: async c => c.id === 8 && c.body === authenticatedBody }); + assert.equal(edited.snapshot.input.discussion.comments.length, 3); +}); + +test("authenticated workflow progress does not self-invalidate discussion or input", async () => { + const f = fixture({ comments: [comment(1)] }); + const authenticateRecord = async c => c.id === 8 && c.user.id === 7 && c.body.startsWith("Authenticated "); + const readTrustedCheckpoints = async () => []; + const before = await snapshot(f, { authenticateRecord, readTrustedCheckpoints }); + const managed = { ...comment(8, "Authenticated progress"), user: { id: 7, login: "app" } }; + f.data.comments.push(managed); + f.data.timeline.push({ ...managed, event: "commented" }); + f.data.issue.updated_at = "2026-09-19T00:00:00Z"; + const after = await snapshot(f, { authenticateRecord, readTrustedCheckpoints }); + assert.equal(after.snapshot.inputDigest, before.snapshot.inputDigest); +}); + +test("authenticated records require an explicit trusted checkpoint reader before freezing", async () => { + const f = fixture({ comments: [comment(456, "A controller-authenticated record")] }); + blocked(await snapshot(f, { authenticateRecord: async () => true }), /checkpoint reader required/i); +}); + +test("real checkpoint authentication excludes only the pinned signed workflow record", async () => { + const { authenticateWorkflowRecord, loadCheckpoint, reserveWork } = require("./state.cjs"); + const body = require("node:fs").readFileSync(require.resolve("./fixtures/checkpoint.txt"), "utf8").trimEnd(); + const actor = { id: 77, login: "github-actions[bot]", type: "Bot", appId: 88 }; + const baseline = state(); + const context = { + repository, issueNumber: 123, commentId: 456, actor, + workflowPath: ".github/workflows/regression-bisect.yml", workflowRef: "refs/heads/main", + workflowRevision: sha("workflow"), trustedWorkflowRevisions: [sha("workflow")], + keys: { fixture: Buffer.from("synthetic-test-key-not-a-production-secret") }, keyId: "fixture", + owner: { runId: "12345", attempt: 1 }, now, issue: issue(), + policy: baseline.budget.policy, policyRevision: baseline.budget.policyRevision, + manifest: baseline.manifest, issueFingerprint: baseline.issueFingerprint, + authorization: baseline.budget.authorization, stableSdk: baseline.budget.stableSdk, pinnedMainSha: baseline.pinnedMainSha, + }; + let runReads = 0; + const stateTransport = { async getRun(runId, attempt) { + assert.equal(runId, "12345"); + assert.equal(attempt, 1); + runReads++; + return { + id: 12345, run_attempt: 1, path: context.workflowPath, head_sha: sha("workflow"), head_branch: "main", + status: "in_progress", + repository: { full_name: repository }, head_repository: { full_name: repository }, + }; + } }; + const managed = { + ...comment(456, body), issue_url: "https://api.github.com/repos/dotnet/fsharp/issues/123", + user: { id: actor.id, login: actor.login, type: actor.type }, performed_via_github_app: { id: actor.appId }, + }; + const copied = comment(457, body); + stateTransport.getComment = async id => { + assert.equal(id, context.commentId); + return structuredClone(managed); + }; + stateTransport.updateComment = async (id, updatedBody) => { + assert.equal(id, context.commentId); + managed.body = updatedBody; + return structuredClone(managed); + }; + const f = fixture({ + comments: [comment(1), managed, copied], + timeline: [{ ...managed, event: "commented" }, { ...copied, event: "commented" }], + runs: { 12345: { id: 12345, status: "in_progress", run_attempt: 1 } }, + }); + const authenticateRecord = record => authenticateWorkflowRecord(stateTransport, context, record); + const readerContext = { ...context, owner: { runId: "99999", attempt: 1 } }; + let loaded; + const readTrustedCheckpoints = async () => { + loaded = await loadCheckpoint(stateTransport, readerContext); + assert.deepEqual(loaded.blockers, []); + return [{ + status: "loaded", state: loaded.checkpoint.state, pending: loaded.checkpoint.pending, + commentId: loaded.commentId, owner: loaded.owner, + }]; + }; + blocked(await snapshot(f, { authenticateRecord }), /checkpoint reader required/i); + const result = await snapshot(f, { authenticateRecord, readTrustedCheckpoints }); + assert.equal(result.status, "complete"); + assert.deepEqual(result.snapshot.input.discussion.comments.map(c => c.id), [1, 457]); + assert.deepEqual(result.snapshot.input.discussion.timeline.map(c => c.id), [457]); + assert.equal(result.snapshot.input.discussion.comments[1].body, body); + assert.equal(runReads, 4); + assert.equal(loaded.checkpoint.pending, null); + assert.deepEqual(result.snapshot.investigations.active[0].owner, loaded.owner); + await reserveWork(stateTransport, context, { + generation: loaded.checkpoint.generation, attemptId: "intake-integration-main", + request: { kind: "source", target: { sourceSha: sha("main") }, endpoint: "main", phase: "main" }, + }); + f.data.timeline[0].body = managed.body; + const pending = await snapshot(f, { authenticateRecord, readTrustedCheckpoints }); + assert.equal(pending.status, "complete"); + assert.equal(pending.snapshot.investigations.active.length, 1); + assert.deepEqual(pending.snapshot.investigations.active[0].owner, loaded.checkpoint.pending.owner); + assert.equal(pending.snapshot.inputDigest, result.snapshot.inputDigest); + assert.equal(await authenticateRecord({ ...managed, user: copied.user }), false); + assert.equal(await authenticateRecord({ ...managed, body: `${body} ` }), false); + stateTransport.getRun = async () => { throw new Error("workflow run provenance unavailable"); }; + blocked(await snapshot(f, { authenticateRecord, readTrustedCheckpoints }), /workflow run provenance unavailable/); +}); + +for (const [name, edit, extra] of [ + ["issue API", f => f.data.fail = route => route.endsWith("/issues/{issue_number}"), {}], + ["later comments API", f => f.data.fail = (route, args) => route.endsWith("/comments") && args.page === 2, {}], + ["timeline API", f => f.data.fail = route => route.endsWith("/timeline"), {}], + ["comment page limit", () => {}, { limits: { commentPages: 1 } }], + ["timeline page limit", f => f.data.timelinePages = [[{ id: 1, event: "labeled" }], [{ id: 2, event: "reopened" }]], { limits: { timelinePages: 1 } }], + ["byte limit", () => {}, { limits: { inputBytes: 10 } }], + ["missing comment ID", f => delete f.data.commentsPages[0][0].id, {}], + ["conflicting duplicate comment", f => f.data.commentsPages[1] = [comment(1, "Changed during pagination")], {}], + ["authentication API", () => {}, { authenticateRecord: async () => { throw new Error("provenance unavailable"); } }], + ["checkpoint API", () => {}, { readTrustedCheckpoints: async () => { throw new Error("checkpoint unavailable"); } }], +]) { + test(`incomplete ${name} blocks without a frozen partial snapshot`, async () => { + const f = fixture({ commentsPages: [[comment(1)], [comment(2)]] }); + edit(f); + blocked(await snapshot(f, extra)); + }); +} + +test("finite safety limits cannot be disabled or increased beyond the hard bounds", async () => { + for (const limits of [{ commentPages: Infinity }, { timelinePages: 0 }, { inputBytes: -1 }, { linkedTargets: 101 }, { unknown: 1 }]) { + blocked(await snapshot(fixture(), { limits }), /limit/i); + } +}); + +test("manifest freezing binds reviewed raw bytes, exact assertions, and dependency identities", async () => { + const f = fixture(); + const plain = await snapshot(f); + const m = manifest(); + m.discussionFingerprint = plain.snapshot.discussionFingerprint; + seal(m); + let bytes = Buffer.from('printfn "ok"'); + const readSource = async source => { + assert.equal(source.path, m.sources[0].path); + return bytes; + }; + const first = await snapshot(f, { reviewedManifest: m, readSource }); + assert.equal(first.status, "complete"); + assert.equal(first.snapshot.input.reproduction.sources[0].bytesBase64, bytes.toString("base64")); + for (const edit of [ + value => value.assertions.pass.stdout = "changed\n", + value => value.packages[0].sha256 = digest("different package"), + value => value.producer.sha256 = digest("different producer"), + value => value.matrix[0].languageVersion = "preview", + ]) { + const changed = structuredClone(m); + edit(changed); + seal(changed); + const next = await snapshot(f, { reviewedManifest: changed, readSource }); + assert.equal(next.status, "complete"); + assert.notEqual(next.snapshot.inputDigest, first.snapshot.inputDigest); + } + bytes = Buffer.from('printfn "ok"\r\n'); + blocked(await snapshot(f, { reviewedManifest: m, readSource }), /source.*hash/i); + m.sources[0].sha256 = digest(bytes); + seal(m); + const changedBytes = await snapshot(f, { reviewedManifest: m, readSource }); + assert.equal(changedBytes.status, "complete"); + assert.notEqual(changedBytes.snapshot.inputDigest, first.snapshot.inputDigest); + f.data.issue.body = "Edited after approval"; + blocked(await snapshot(f, { reviewedManifest: m, readSource }), /discussion/i); +}); + +test("missing, oversized, wrong-issue, or unreadable reviewed sources block", async () => { + const f = fixture(); + const m = manifest(); + m.discussionFingerprint = (await snapshot(f)).snapshot.discussionFingerprint; + seal(m); + blocked(await snapshot(f, { reviewedManifest: m }), /source/i); + blocked(await snapshot(f, { reviewedManifest: m, readSource: async () => { throw new Error("source unavailable"); } })); + blocked(await snapshot(f, { reviewedManifest: m, readSource: async () => Buffer.alloc(20), limits: { sourceBytes: 10 } }), /source/i); + m.issueNumber = 999; + seal(m); + blocked(await snapshot(f, { reviewedManifest: m, readSource: async () => Buffer.from('printfn "ok"') }), /issue/i); +}); + +test("linked investigations use freshly read status and activity rather than old timeline claims", async () => { + const f = fixture({ + comments: [comment(1, "Already investigating forever!"), comment(2, "See https://github.com/dotnet/fsharp/pull/321")], + timeline: [reference(321), reference(322), reference(323, "issues")], + linked: { + 321: issue(321, { state: "closed", merged_at: now }), + 322: issue(322, { state: "open", updated_at: now, merged_at: null }), + 323: issue(323, { state: "open", updated_at: "2026-09-01T00:00:00Z" }), + }, + }); + const result = await snapshot(f); + assert.equal(result.status, "complete"); + assert.deepEqual(result.snapshot.investigations.active.map(x => x.number), [322]); + assert.equal(result.snapshot.investigations.links.length, 3); + assert.equal(result.snapshot.investigations.links.find(x => x.number === 321).state, "closed"); + assert.equal(f.calls.filter(c => c.args.pull_number === 321).length, 1); +}); + +test("unlinked prose and unrelated links do not veto; reciprocal current investigation does", async () => { + const f = fixture({ + comments: [comment(1, "Already investigating! https://github.com/dotnet/fsharp/issues/321")], + linked: { 321: issue(321, { body: "Unrelated open issue" }) }, + }); + let found = await intake().findActiveInvestigations({ ...f.options, issue: f.data.issue, comments: f.data.comments, timeline: [] }); + assert.equal(found.status, "complete"); + assert.equal(found.active.length, 0); + f.data.linked[321].body = "Investigating https://github.com/dotnet/fsharp/issues/123"; + found = await intake().findActiveInvestigations({ ...f.options, issue: f.data.issue, comments: f.data.comments, timeline: [] }); + assert.equal(found.active.length, 1); +}); + +test("required linked reads and linked-target safety limits fail closed", async () => { + const f = fixture({ timeline: [reference(321)] }); + blocked(await snapshot(f), /linked/i); + f.data.timeline.push(reference(322)); + blocked(await snapshot(f, { limits: { linkedTargets: 1 } }), /limit/i); +}); + +test("timeline event and comment identifiers occupy separate namespaces", async () => { + const f = fixture({ comments: [comment(5)], timeline: [ + { ...comment(5), event: "commented" }, + { id: 5, event: "labeled", label: { name: "Regression" } }, + ] }); + const result = await snapshot(f); + assert.equal(result.status, "complete"); + assert.equal(result.snapshot.input.discussion.timeline.length, 2); +}); + +test("a supplied checkpoint reader must return an explicit successful result", async () => { + for (const value of [undefined, null, {}, [{ status: "loaded" }]]) { + blocked(await snapshot(fixture(), { readTrustedCheckpoints: async () => value }), /checkpoint/i); + } +}); + +test("linked PRs require complete current merged status", async () => { + const f = fixture({ timeline: [reference(321)], linked: { 321: issue(321) } }); + blocked(await snapshot(f), /linked.*merged/i); +}); + +test("lost cursor traversal also reaches dense-page overflow candidates", async () => { + const f = fixture({ list: Array.from({ length: 205 }, (_, i) => issue(i + 1)) }); + const seen = new Set(); + for (let scheduleOrdinal = 0; scheduleOrdinal < 750; scheduleOrdinal++) { + const start = f.calls.length; + const found = await intake().discoverEligibleIssues({ ...f.options, issueNumber: undefined, scheduleOrdinal }); + assert.equal(found.status, "complete"); + assert.ok(found.issues.length <= 3); + assert.ok(f.calls.length - start <= 2); + found.issues.forEach(value => seen.add(value.number)); + } + assert.ok(seen.has(4), "lost cursors must not starve the fourth eligible issue"); + assert.ok(seen.has(101), "lost cursors must not starve the second page"); + assert.equal(seen.size, 205); +}); + +test("trusted checkpoint ownership checks the live run, never a forged marker", async () => { + const s = state(); + const f = fixture({ comments: [comment(1, " active")], runs: { + 44: { id: 44, status: "in_progress", run_attempt: 2 }, + } }); + const loaded = { status: "loaded", state: s, commentId: 77, owner: { runId: "44", attempt: 2 } }; + let result = await snapshot(f, { readTrustedCheckpoints: async () => [loaded] }); + assert.equal(result.status, "complete"); + assert.equal(result.snapshot.investigations.active[0].basis, "trusted-checkpoint-active-run"); + assert.equal(f.calls.at(-1).args.attempt_number, 2); + f.data.runs[44].status = "completed"; + result = await snapshot(f, { readTrustedCheckpoints: async () => [loaded] }); + assert.equal(result.snapshot.investigations.active.length, 0); + delete f.data.runs[44]; + blocked(await snapshot(f, { readTrustedCheckpoints: async () => [loaded] }), /run/i); + blocked(await snapshot(f, { readTrustedCheckpoints: async () => [{ status: "blocked", reason: "mandatory evidence missing" }] })); + s.issueNumber = 999; + blocked(await snapshot(f, { readTrustedCheckpoints: async () => [loaded] }), /issue/i); +}); + +test("pending reservation ownership takes precedence over the checkpoint writer", async () => { + const f = fixture({ runs: { 44: { id: 44, status: "in_progress", run_attempt: 2 } } }); + const loaded = { + status: "loaded", state: state(), commentId: 77, + owner: { runId: "99", attempt: 1 }, pending: { owner: { runId: "44", attempt: 2 } }, + }; + const result = await snapshot(f, { readTrustedCheckpoints: async () => [loaded] }); + assert.equal(result.status, "complete"); + assert.equal(result.snapshot.investigations.active[0].owner.runId, "44"); +}); + +for (const [name, patch] of [ + ["opened with label", {}], + ["human late label", { created_at: "2026-09-02T00:00:00Z" }], + ["bot late label", { created_at: "2026-09-03T00:00:00Z", user: { id: 2, login: "bot", type: "Bot" } }], + ["wrong label case", { labels: ["regression"] }], + ["before cutoff", { created_at: "2026-08-31T23:59:59Z" }], + ["closed", { state: "closed" }], + ["pull request", { pull_request: {} }], +]) { + test(`manual, event, scheduled and snapshot selection share exact predicate: ${name}`, async () => { + const value = issue(123, patch); + const f = fixture({ issue: value, list: [value] }); + const expected = isEligibleIssue(value); + for (const trigger of ["manual", "event", "schedule"]) { + const selected = await intake().discoverEligibleIssues({ + ...f.options, issueNumber: trigger === "schedule" ? undefined : 123, trigger, scheduleOrdinal: 0, + }); + assert.equal(selected.status, "complete"); + assert.equal(selected.issues.length, expected ? 1 : 0); + } + assert.equal((await snapshot(f)).status, expected ? "complete" : "ineligible"); + }); +} + +test("created-order REST discovery has hard 2x100/three-candidate bounds and no overflow skips", async () => { + const list = Array.from({ length: 205 }, (_, i) => issue(i + 1)); + const f = fixture({ list }); + const seen = new Set(); + let cursor; + for (let sweep = 0; sweep < 80 && seen.size < list.length; sweep++) { + const start = f.calls.length; + const result = await intake().discoverEligibleIssues({ ...f.options, issueNumber: undefined, cursor, scheduleOrdinal: sweep }); + assert.equal(result.status, "complete"); + assert.ok(result.issues.length <= 3); + assert.ok(f.calls.length - start <= 2); + result.issues.forEach(item => seen.add(item.number)); + cursor = result.nextCursor; + } + assert.equal(seen.size, 205); + for (const { route, args } of f.calls) { + assert.equal(route, "GET /repos/{owner}/{repo}/issues"); + assert.equal(args.sort, "created"); + assert.equal(args.direction, "asc"); + assert.equal(args.per_page, 100); + assert.equal(args.labels, "Regression"); + assert.equal(args.state, "open"); + assert.equal(args.since, undefined); + } +}); + +test("discovery rotates over sparse pages and a lost cursor uses the independent schedule ordinal", async () => { + const list = Array.from({ length: 401 }, (_, i) => issue(i + 1, { labels: i === 400 ? ["Regression"] : [] })); + const f = fixture({ list }); + let cursor; + const seen = new Set(); + for (let sweep = 0; sweep < 20; sweep++) { + const result = await intake().discoverEligibleIssues({ ...f.options, issueNumber: undefined, cursor, scheduleOrdinal: sweep }); + result.issues.forEach(item => seen.add(item.number)); + cursor = result.nextCursor; + } + assert.ok(seen.has(401)); + seen.clear(); + for (let sweep = 0; sweep < 120; sweep++) { + const result = await intake().discoverEligibleIssues({ ...f.options, issueNumber: undefined, scheduleOrdinal: sweep }); + result.issues.forEach(item => seen.add(item.number)); + } + assert.ok(seen.has(401), "losing the cursor each run must not starve later pages"); + f.data.list[0].labels = ["Regression"]; + let recovered = false; + for (let sweep = 120; sweep < 750; sweep++) { + const result = await intake().discoverEligibleIssues({ ...f.options, issueNumber: undefined, scheduleOrdinal: sweep }); + recovered ||= result.issues.some(item => item.number === 1); + } + assert.ok(recovered, "old issues labeled by automation are eventually revisited even without cursor storage"); +}); + +test("discovery returns no candidates or advanced cursor after a required API failure", async () => { + const f = fixture({ + list: Array.from({ length: 201 }, (_, i) => issue(i + 1, { labels: i === 0 ? ["Regression"] : [] })), + fail: (_, args) => args.page === 2, + }); + const result = await intake().discoverEligibleIssues({ ...f.options, issueNumber: undefined, scheduleOrdinal: 0 }); + assert.equal(result.status, "blocked"); + assert.deepEqual(result.issues, []); +}); diff --git a/eng/regression-bisect/state.cjs b/eng/regression-bisect/state.cjs new file mode 100644 index 00000000000..6ed0bc99f14 --- /dev/null +++ b/eng/regression-bisect/state.cjs @@ -0,0 +1,503 @@ +const { createHash, createHmac, timingSafeEqual } = require("node:crypto"); +const { + isEligibleIssue, validateManifest, validatePolicy, validateState, validateObservation, + fingerprintInput: fingerprint, fingerprintEpoch, +} = require("./model.cjs"); + +const MAX_BYTES = 48 * 1024; +const counterNames = ["sourceBuilds", "workerUnits", "reservedMinutes"]; +class StateError extends Error {} +const requireState = (condition, message) => { if (!condition) throw new StateError(message); }; +const equal = (a, b) => fingerprint(a) === fingerprint(b); +const digest = value => typeof value === "string" && /^[a-f0-9]{64}$/.test(value); +const positive = value => Number.isSafeInteger(value) && value > 0; +const timestamp = value => typeof value === "string" && /^\d{4}-\d\d-\d\dT.*Z$/.test(value) && Number.isFinite(Date.parse(value)); +const hashBytes = bytes => createHash("sha256").update(bytes).digest("hex"); +const ownerEqual = (a, b) => a?.runId === b?.runId && a?.attempt === b?.attempt; + +function fields(value, names, description) { + requireState(value && Object.getPrototypeOf(value) === Object.prototype && + equal(Object.keys(value).sort(), names.split(" ").sort()), `invalid ${description} schema`); +} + +// Intern objects, arrays and long strings, including repeated assertion streams. +// References are tagged arrays; original arrays exist only as table entries. +function pack(value) { + const table = [], seen = new Map(); + function encode(v) { + if (v === null || (typeof v !== "object" && !(typeof v === "string" && v.length >= 128))) return v; + const key = fingerprint(v); + if (seen.has(key)) return ["@", seen.get(key)]; + const index = table.length; + seen.set(key, index); + table.push(null); + table[index] = typeof v === "string" ? v : Array.isArray(v) ? v.map(encode) : + Object.fromEntries(Object.entries(v).map(([k, child]) => [k, encode(child)])); + return ["@", index]; + } + return { root: encode(value), table }; +} + +function unpack(packed) { + fields(packed, "root table", "identity table"); + requireState(Array.isArray(packed.table), "invalid identity table"); + let visits = 0; + const active = new Set(); + function decode(value, depth = 0) { + requireState(++visits <= 100000 && depth <= 64, "oversized expanded checkpoint"); + if (!Array.isArray(value)) { + requireState(value === null || typeof value !== "object", "invalid identity reference"); + return value; + } + requireState(value.length === 2 && value[0] === "@" && Number.isSafeInteger(value[1]) && + value[1] >= 0 && value[1] < packed.table.length && !active.has(value[1]), "invalid identity reference"); + active.add(value[1]); + const node = packed.table[value[1]]; + requireState(node !== null && (typeof node === "object" || typeof node === "string"), "invalid identity entry"); + const result = typeof node === "string" ? node : Array.isArray(node) ? node.map(v => decode(v, depth + 1)) : + Object.fromEntries(Object.entries(node).map(([k, v]) => [k, decode(v, depth + 1)])); + active.delete(value[1]); + return result; + } + return decode(packed.root); +} + +function render(envelope) { + return `${envelope.status}\n\n\`\`\`json\n${JSON.stringify(envelope)}\n\`\`\``; +} + +function serializeWorkflowRecord({ kind, status, payload }, context) { + requireState(["checkpoint", "proposal", "result"].includes(kind) && + typeof status === "string" && status.length <= 256 && !/[\r\n`]/.test(status), "invalid record status"); + const key = context.keys[context.keyId]; + requireState(Buffer.isBuffer(key) && key.length >= 32, "authentication key must contain at least 32 bytes"); + const envelope = { + schemaVersion: 1, kind, status, + binding: { + repository: context.repository, issueNumber: context.issueNumber, commentId: context.commentId, + workflowPath: context.workflowPath, workflowRef: context.workflowRef, + workflowRevision: context.workflowRevision, runId: context.owner.runId, + runAttempt: context.owner.attempt, keyId: context.keyId, + }, + payload: pack(payload), + }; + envelope.signature = createHmac("sha256", key).update(fingerprint(envelope)).digest("hex"); + const body = render(envelope); + requireState(Buffer.byteLength(body) < MAX_BYTES, "checkpoint exceeds 48 KiB size limit; execution blocked"); + return body; +} + +function parseRecord(body) { + if (typeof body !== "string" || Buffer.byteLength(body) >= MAX_BYTES) return null; + const match = /^[^\r\n`]*\n\n```json\n([^\r\n]+)\n```$/.exec(body); + if (!match) return null; + try { return JSON.parse(match[1]); } catch (error) { + if (error instanceof SyntaxError) return null; + throw error; + } +} + +async function verifyRun(transport, context, binding) { + const run = await transport.getRun(binding.runId, binding.runAttempt); + requireState(String(run.id) === binding.runId && run.run_attempt === binding.runAttempt && + run.repository?.full_name === context.repository && run.head_repository?.full_name === context.repository && + run.path === context.workflowPath && `refs/heads/${run.head_branch}` === context.workflowRef && + run.head_sha === binding.workflowRevision && context.trustedWorkflowRevisions.includes(run.head_sha), + "workflow run provenance authentication failed"); + return run; +} + +async function authenticatedRecord(transport, context, comment) { + const record = parseRecord(comment?.body); + if (!record || record.schemaVersion !== 1 || !["checkpoint", "proposal", "result"].includes(record.kind) || + typeof record.status !== "string" || !digest(record.signature) || + !equal(Object.keys(record).sort(), ["binding", "kind", "payload", "schemaVersion", "signature", "status"])) return null; + const b = record.binding; + if (!b || !equal(Object.keys(b).sort(), [ + "commentId", "issueNumber", "keyId", "repository", "runAttempt", "runId", + "workflowPath", "workflowRef", "workflowRevision", + ])) return null; + if (comment.id !== context.commentId || b.commentId !== context.commentId || + b.issueNumber !== context.issueNumber || b.repository !== context.repository || + b.workflowPath !== context.workflowPath || b.workflowRef !== context.workflowRef || + !context.trustedWorkflowRevisions.includes(b.workflowRevision) || + !/^\d+$/.test(b.runId) || !positive(b.runAttempt) || + comment.issue_url !== `https://api.github.com/repos/${context.repository}/issues/${context.issueNumber}` || + comment.user?.id !== context.actor.id || comment.user?.login !== context.actor.login || + comment.user?.type !== context.actor.type || comment.performed_via_github_app?.id !== context.actor.appId) return null; + const key = context.keys[b.keyId]; + if (!Buffer.isBuffer(key) || key.length < 32) return null; + const { signature, ...signed } = record; + const expected = createHmac("sha256", key).update(fingerprint(signed)).digest(); + if (!timingSafeEqual(Buffer.from(signature, "hex"), expected) || render(record) !== comment.body) return null; + await verifyRun(transport, context, b); + return record; +} + +async function authenticateWorkflowRecord(transport, context, comment) { + return (await authenticatedRecord(transport, context, comment)) !== null; +} + +function validateEvidence(evidence) { + requireState(Array.isArray(evidence) && evidence.length <= 16, "invalid evidence list"); + for (const item of evidence) { + fields(item, "required sha256 url", "external evidence"); + let url; + try { url = new URL(item.url); } catch (error) { + if (!(error instanceof TypeError)) throw error; + throw new StateError("invalid evidence URL"); + } + requireState(typeof item.required === "boolean" && digest(item.sha256) && + url.protocol === "https:" && !url.username && !url.password && !url.hash, "invalid external evidence"); + } +} + +function validateCheckpoint(cp) { + fields(cp, "schemaVersion generation state inputDigest reproductionDigest policyDigest sequence pending records observations history deadline publicationStatus", "checkpoint"); + requireState(cp.schemaVersion === 1, "unsupported checkpoint schema"); + validateState(cp.state); + requireState(Number.isSafeInteger(cp.generation) && cp.generation >= 0 && timestamp(cp.deadline) && + cp.inputDigest === cp.state.manifest.approvedInputDigest && + cp.reproductionDigest === reproductionDigest(cp.state.manifest) && + cp.policyDigest === fingerprint(cp.state.budget.policy), "checkpoint digest or generation mismatch"); + requireState(Array.isArray(cp.records) && cp.records.length <= cp.state.budget.policy.limits.workerUnits && + cp.sequence === cp.records.length && Array.isArray(cp.history) && + ["unpublished", "pending", "published", "blocked"].includes(cp.publicationStatus), "invalid checkpoint ledger"); + const totals = { sourceBuilds: 0, workerUnits: 0, reservedMinutes: 0 }; + const ids = new Set(); + for (const [index, r] of cp.records.entries()) { + fields(r, "id workItemId request requestDigest owner sequence attempt generation deadline inputDigest epoch reserved status observationDigest", "work record"); + requireState(typeof r.id === "string" && r.id.length > 0 && r.id.length <= 128 && !ids.has(r.id) && + r.sequence === index + 1 && positive(r.attempt) && positive(r.generation) && + timestamp(r.deadline) && digest(r.inputDigest) && digest(r.epoch) && + r.requestDigest === fingerprint(r.request) && digest(r.workItemId) && + ["reserved", "committed", "cancelled", "invalidated"].includes(r.status), "invalid work record identity"); + fields(r.owner, "runId attempt", "work owner"); + requireState(/^\d+$/.test(r.owner.runId) && positive(r.owner.attempt), "invalid work owner"); + fields(r.reserved, "sourceBuilds workerUnits reservedMinutes", "reservation counters"); + requireState(r.reserved.workerUnits === 1 && r.reserved.sourceBuilds === (r.request.kind === "source" ? 1 : 0) && + r.reserved.reservedMinutes === cp.state.budget.policy.limits.workerTimeoutMinutes, "invalid reservation charge"); + ids.add(r.id); + for (const name of counterNames) totals[name] += r.reserved[name]; + requireState(r.status === "committed" ? digest(r.observationDigest) : r.observationDigest === null, "invalid observation identity"); + } + requireState(equal(totals, cp.state.budget.consumed), "consumed budgets must equal the durable reservation ledger"); + const counts = new Map(); + for (const r of cp.records) { + const count = (counts.get(r.workItemId) ?? 0) + 1; + requireState(r.attempt === count, "nonsequential work attempt"); + counts.set(r.workItemId, count); + } + requireState(equal([...counts].map(([workItemId, count]) => ({ workItemId, count })), cp.state.budget.attempts), + "attempt accounting mismatch"); + const reserved = cp.records.filter(r => r.status === "reserved"); + requireState(cp.pending === null ? reserved.length === 0 && cp.state.budget.pendingWork === null : + reserved.length === 1 && equal(reserved[0], cp.pending) && + cp.state.budget.pendingWork?.id === cp.pending.id, "pending reservation mismatch"); + requireState(cp.observations && Object.getPrototypeOf(cp.observations) === Object.prototype, "invalid observation table"); + for (const [id, observation] of Object.entries(cp.observations)) { + fields(observation, "data manifest attemptId evidence valid", "observation record"); + validateObservation(observation.data, observation.manifest); + validateEvidence(observation.evidence); + requireState(fingerprint(observation.data) === id && typeof observation.valid === "boolean" && + cp.records.some(r => r.id === observation.attemptId && r.observationDigest === id), "observation identity mismatch"); + if (observation.valid) requireState(observation.data.manifestDigest === cp.state.approvalFingerprint && + observation.data.provenance.workflowRevision === cp.state.budget.workflowRevision && + observation.data.provenance.policyRevision === cp.state.budget.policyRevision, "stale observation marked valid"); + } + requireState(cp.records.every(r => r.status !== "committed" || cp.observations[r.observationDigest]?.attemptId === r.id), + "missing durable observation"); + requireState(equal(Object.entries(cp.observations).filter(([, o]) => o.valid).map(([id]) => id), + cp.state.budget.authenticatedObservationDigests), "authenticated observation table mismatch"); +} + +function reproductionDigest(manifest) { + const { discussionFingerprint, approvedInputDigest, searchHints, ...reproduction } = manifest; + return fingerprint(reproduction); +} + +function currentState(cp, context) { + const reasons = []; + if (!isEligibleIssue(context.issue) || context.issue.number !== context.issueNumber) reasons.push("issue not eligible"); + let state = null; + try { + validatePolicy(context.policy); + validateManifest(context.manifest); + requireState(context.repository === "dotnet/fsharp" && context.manifest.issueNumber === context.issueNumber && + context.issueFingerprint === context.manifest.discussionFingerprint, "approval does not match complete issue snapshot"); + const budget = { + ...cp.state.budget, policy: context.policy, policyRevision: context.policyRevision, + workflowRevision: context.workflowRevision, authorization: context.authorization, stableSdk: context.stableSdk, + }; + state = { + ...cp.state, manifest: context.manifest, budget, + issueFingerprint: context.issueFingerprint, approvalFingerprint: fingerprint(context.manifest), + pinnedMainSha: context.pinnedMainSha, + epoch: fingerprintEpoch(context.manifest, budget, context.pinnedMainSha), + }; + } catch (error) { + if (!(error instanceof StateError) && error.constructor.name !== "ContractError") throw error; + reasons.push(`approval blocked: ${error.message}`); + } + const stale = state !== null && state.epoch !== cp.state.epoch; + if (stale) reasons.push("stale inputs or current endpoints"); + if (Date.parse(context.now) >= Date.parse(cp.deadline)) reasons.push("allowance deadline exhausted"); + requireState(timestamp(context.now), "invalid controller time"); + return { state, reasons, stale }; +} + +async function evidenceWarnings(transport, observations) { + const warnings = []; + let missingRequired = false; + for (const record of observations) { + for (const evidence of record.evidence) { + // The transport returns null only for confirmed unavailable/expired content. + // API/authentication errors propagate, rather than becoming missing evidence. + const bytes = await transport.readEvidence(evidence.url); + requireState(bytes === null || Buffer.isBuffer(bytes), "evidence transport must return bytes or null"); + if (bytes === null || hashBytes(bytes) !== evidence.sha256) { + warnings.push(`${evidence.required ? "required" : "optional"} missing or expired evidence: ${evidence.url}`); + if (evidence.required) missingRequired = true; + } + } + } + return { warnings, missingRequired }; +} + +async function loadCheckpoint(transport, context) { + const comment = await transport.getComment(context.commentId); + const record = await authenticatedRecord(transport, context, comment); + requireState(record?.kind === "checkpoint", "checkpoint authentication failed"); + const checkpoint = unpack(record.payload); + validateCheckpoint(checkpoint); + requireState(checkpoint.state.repository === context.repository && checkpoint.state.issueNumber === context.issueNumber, + "checkpoint issue authentication failed"); + const current = currentState(checkpoint, context); + const evidence = await evidenceWarnings(transport, Object.values(checkpoint.observations).filter(o => o.valid)); + const blockers = [...current.reasons, ...(evidence.missingRequired ? ["required evidence unavailable; blocked"] : [])]; + return { + checkpoint, commentId: context.commentId, + owner: { runId: record.binding.runId, attempt: record.binding.runAttempt }, + executionAllowed: blockers.length === 0, + publicationAllowed: blockers.length === 0 && checkpoint.state.phase !== "stopped", + blockers, warnings: evidence.warnings, + }; +} + +async function saveCheckpoint(transport, context, checkpoint) { + validateCheckpoint(checkpoint); + const owner = await verifyRun(transport, context, { + runId: context.owner.runId, runAttempt: context.owner.attempt, workflowRevision: context.workflowRevision, + }); + requireState(owner.status === "in_progress", "checkpoint writer must be the active owning run"); + const body = serializeWorkflowRecord({ + kind: "checkpoint", status: `Regression investigation: ${checkpoint.state.phase} (generation ${checkpoint.generation}).`, + payload: checkpoint, + }, context); + const written = await transport.updateComment(context.commentId, body); + requireState(written?.body === body && await authenticateWorkflowRecord(transport, context, written), + "checkpoint write authentication failed"); + return checkpoint; +} + +async function initializeCheckpoint(transport, context, { allowanceId, deadline }) { + const existing = await transport.getComment(context.commentId); + requireState(digest(context.bootstrapDigest) && fingerprint(existing.body) === context.bootstrapDigest && + parseRecord(existing.body) === null, "initialization requires the exact trusted bootstrap placeholder"); + requireState(existing.id === context.commentId && existing.user?.id === context.actor.id && + existing.user?.login === context.actor.login && existing.user?.type === context.actor.type && + existing.performed_via_github_app?.id === context.actor.appId && + existing.issue_url === `https://api.github.com/repos/${context.repository}/issues/${context.issueNumber}`, + "bootstrap author authentication failed"); + requireState(timestamp(deadline) && Date.parse(deadline) > Date.parse(context.now), "invalid allowance deadline"); + const budget = { + schemaVersion: 1, policy: context.policy, policyRevision: context.policyRevision, workflowRevision: context.workflowRevision, + allowanceId, authorization: context.authorization, stableSdk: context.stableSdk, + authenticatedObservationDigests: [], consumed: { sourceBuilds: 0, workerUnits: 0, reservedMinutes: 0 }, + attempts: [], nextWorkItemId: "none", pendingWork: null, + }; + const state = { + schemaVersion: 1, repository: context.repository, issueNumber: context.issueNumber, + issueFingerprint: context.issueFingerprint, approvalFingerprint: fingerprint(context.manifest), + manifest: context.manifest, budget, pinnedMainSha: context.pinnedMainSha, + epoch: fingerprintEpoch(context.manifest, budget, context.pinnedMainSha), phase: "discovery", + endpointEvidence: { stable: null, main: null, good: null }, testedRevisions: [], skips: [], interval: null, + publication: { digest: null, commentId: null }, + }; + const cp = { + schemaVersion: 1, generation: 0, state, inputDigest: context.manifest.approvedInputDigest, + reproductionDigest: reproductionDigest(context.manifest), policyDigest: fingerprint(context.policy), + sequence: 0, pending: null, records: [], observations: {}, history: [], deadline, publicationStatus: "unpublished", + }; + requireState(currentState(cp, context).reasons.length === 0, "initialization blocked by current eligibility or approval"); + return saveCheckpoint(transport, context, structuredClone(cp)); +} + +function checkGeneration(cp, generation) { + requireState(cp.generation === generation, "stale checkpoint generation; controller lock required"); +} + +async function retirePending(transport, context, cp, status) { + if (!cp.pending) return; + const run = await verifyRun(transport, context, { + runId: cp.pending.owner.runId, runAttempt: cp.pending.owner.attempt, + workflowRevision: cp.state.budget.workflowRevision, + }); + requireState(run.status === "completed" && typeof run.conclusion === "string", "pending work has an active or unknown owning run"); + cp.records.find(r => r.id === cp.pending.id).status = status; + cp.pending = null; + cp.state.budget.pendingWork = null; +} + +function validateRequest(request, state) { + fields(request, "kind target endpoint phase", "work request"); + requireState(["source", "released"].includes(request.kind) && + [null, "stable", "main", "good"].includes(request.endpoint) && + ["stable", "main", "good-search", "admission", "bisect", "confirmation"].includes(request.phase), "invalid work request"); + fields(request.target, request.kind === "source" ? "sourceSha" : "sdk", "work target"); + requireState(request.kind === "source" ? /^[a-f0-9]{40}$/.test(request.target.sourceSha) : + equal(request.target.sdk, state.budget.stableSdk), "invalid work target identity"); + requireState(request.endpoint !== "stable" || request.kind === "released", "stable requires released identity"); + requireState(!["main", "good"].includes(request.endpoint) || request.kind === "source", "source endpoint requires source identity"); + requireState(request.endpoint !== "main" || request.target.sourceSha === state.pinnedMainSha, "main request identity mismatch"); +} + +async function reserveWork(transport, context, { generation, attemptId, request }) { + const loaded = await loadCheckpoint(transport, context); + requireState(loaded.executionAllowed, `execution blocked: ${loaded.blockers.join("; ")}`); + const cp = loaded.checkpoint; + validateRequest(request, cp.state); + requireState(typeof attemptId === "string" && attemptId.length > 0 && attemptId.length <= 128, "invalid attempt identity"); + const previous = cp.records.find(r => r.id === attemptId); + if (previous) { + requireState(ownerEqual(previous.owner, context.owner) && previous.requestDigest === fingerprint(request), + "attempt identity reuse conflict"); + requireState(previous.status === "reserved" && cp.pending?.id === attemptId, "attempt already committed, cancelled or invalidated"); + requireState(generation === previous.generation - 1 || generation === cp.generation, "stale checkpoint generation"); + return { ...structuredClone(previous), execute: false }; + } + checkGeneration(cp, generation); + const budget = cp.state.budget; + const workItemId = fingerprint({ kind: request.kind, target: request.target, inputDigest: cp.inputDigest }); + const attempt = (budget.attempts.find(a => a.workItemId === workItemId)?.count ?? 0) + 1; + requireState(attempt <= budget.policy.limits.attemptsPerItem, "work item attempt budget exhausted"); + const reserved = { sourceBuilds: request.kind === "source" ? 1 : 0, workerUnits: 1, reservedMinutes: budget.policy.limits.workerTimeoutMinutes }; + requireState(counterNames.every(name => budget.consumed[name] + reserved[name] <= budget.policy.limits[name]), "cumulative budget exhausted"); + const deadline = new Date(Date.parse(context.now) + reserved.reservedMinutes * 60000).toISOString(); + requireState(Date.parse(deadline) <= Date.parse(cp.deadline), "insufficient time before allowance deadline"); + await retirePending(transport, context, cp, "cancelled"); + for (const name of counterNames) budget.consumed[name] += reserved[name]; + const counter = budget.attempts.find(a => a.workItemId === workItemId); + if (counter) counter.count = attempt; + else budget.attempts.push({ workItemId, count: attempt }); + cp.generation++; + cp.sequence++; + const pending = { + id: attemptId, workItemId, request: structuredClone(request), requestDigest: fingerprint(request), + owner: structuredClone(context.owner), sequence: cp.sequence, attempt, generation: cp.generation, + deadline, inputDigest: cp.inputDigest, epoch: cp.state.epoch, reserved, status: "reserved", observationDigest: null, + }; + cp.pending = pending; + cp.records.push(pending); + budget.nextWorkItemId = workItemId; + budget.pendingWork = { id: attemptId, allowanceId: budget.allowanceId, workItemId, attempt, reserved, status: "reserved" }; + cp.state.phase = request.phase; + await saveCheckpoint(transport, context, cp); + return { ...structuredClone(pending), execute: true }; +} + +async function commitObservation(transport, context, { generation, attemptId, observation, evidence }) { + const loaded = await loadCheckpoint(transport, context); + const cp = loaded.checkpoint; + const record = cp.records.find(r => r.id === attemptId); + requireState(record && ownerEqual(record.owner, context.owner), "observation owner mismatch"); + const observationDigest = fingerprint(observation); + validateEvidence(evidence); + if (record.status === "committed") { + requireState(generation === record.generation, "stale completion generation"); + requireState(record.observationDigest === observationDigest && + equal(cp.observations[observationDigest].evidence, evidence), "observation identity conflict"); + return cp; + } + checkGeneration(cp, generation); + requireState(loaded.executionAllowed, `completion blocked: ${loaded.blockers.join("; ")}`); + requireState(cp.pending?.id === attemptId && record.status === "reserved", "cancelled or invalidated work"); + requireState(Date.parse(context.now) <= Date.parse(record.deadline), "observation arrived after reservation deadline"); + validateObservation(observation, cp.state.manifest); + requireState(observation.kind === record.request.kind && equal(observation.target, record.request.target) && + observation.buildInstanceId === record.id && !observation.cached && + observation.provenance.runId === record.owner.runId && + observation.provenance.workflowRevision === cp.state.budget.workflowRevision && + observation.provenance.policyRevision === cp.state.budget.policyRevision, "observation request or build identity mismatch"); + const availability = await evidenceWarnings(transport, [{ evidence }]); + requireState(!availability.missingRequired, "required evidence missing or digest mismatch"); + cp.observations[observationDigest] = { + data: structuredClone(observation), manifest: structuredClone(cp.state.manifest), + attemptId, evidence: structuredClone(evidence), valid: true, + }; + record.status = "committed"; + record.observationDigest = observationDigest; + cp.state.budget.authenticatedObservationDigests.push(observationDigest); + if (record.request.endpoint !== null) cp.state.endpointEvidence[record.request.endpoint] = observationDigest; + if (observation.kind === "source") { + const tested = { sourceSha: observation.target.sourceSha, observationDigest }; + cp.state.testedRevisions.push(tested); + if (observation.outcome.outcome === "untestable") cp.state.skips.push({ ...tested, reason: observation.outcome.reason }); + } + cp.pending = null; + cp.state.budget.pendingWork = null; + cp.generation++; + return saveCheckpoint(transport, context, cp); +} + +async function invalidateStaleEvidence(transport, context, { generation }) { + const loaded = await loadCheckpoint(transport, context); + const cp = loaded.checkpoint; + checkGeneration(cp, generation); + const current = currentState(cp, context); + const eligible = isEligibleIssue(context.issue) && context.issue.number === context.issueNumber; + if (!current.stale && current.state !== null && eligible && cp.state.phase !== "stopped") return cp; + const phase = !eligible ? "stopped" : current.state === null ? "blocked" : "discovery"; + if (!current.stale && cp.pending === null && cp.state.phase === phase && + (current.state !== null || Object.values(cp.observations).every(o => !o.valid))) return cp; + await retirePending(transport, context, cp, "invalidated"); + const old = cp.state; + if (current.stale || current.state === null) { + const sameInputs = current.state !== null && old.approvalFingerprint === current.state.approvalFingerprint && + old.budget.workflowRevision === current.state.budget.workflowRevision && + old.budget.policyRevision === current.state.budget.policyRevision; + cp.history.push({ + epoch: old.epoch, inputDigest: cp.inputDigest, approvalDigest: old.approvalFingerprint, + policyDigest: cp.policyDigest, workflowRevision: old.budget.workflowRevision, + pinnedMainSha: old.pinnedMainSha, stableSdk: old.budget.stableSdk, + endpointEvidence: old.endpointEvidence, interval: old.interval, publication: old.publication, + reason: current.state === null ? "current approval unavailable" : + sameInputs ? "current endpoints changed" : "approved inputs changed", + }); + cp.state = structuredClone(current.state ?? old); + cp.state.budget.pendingWork = null; + for (const observation of Object.values(cp.observations)) observation.valid &&= sameInputs; + cp.state.budget.authenticatedObservationDigests = Object.entries(cp.observations).filter(([, o]) => o.valid).map(([id]) => id); + cp.state.endpointEvidence = { + stable: sameInputs && equal(old.budget.stableSdk, cp.state.budget.stableSdk) ? old.endpointEvidence.stable : null, + main: sameInputs && old.pinnedMainSha === cp.state.pinnedMainSha ? old.endpointEvidence.main : null, + good: sameInputs ? old.endpointEvidence.good : null, + }; + if (!sameInputs) { cp.state.testedRevisions = []; cp.state.skips = []; } + cp.state.interval = null; + cp.inputDigest = cp.state.manifest.approvedInputDigest; + cp.reproductionDigest = reproductionDigest(cp.state.manifest); + cp.policyDigest = fingerprint(cp.state.budget.policy); + cp.state.publication = { digest: null, commentId: null }; + } + cp.state.phase = phase; + cp.publicationStatus = !eligible || current.state === null ? "blocked" : + cp.state.publication.commentId !== null ? "published" : "unpublished"; + cp.generation++; + return saveCheckpoint(transport, context, cp); +} + +module.exports = { + initializeCheckpoint, serializeWorkflowRecord, authenticateWorkflowRecord, + loadCheckpoint, reserveWork, commitObservation, invalidateStaleEvidence, +}; diff --git a/eng/regression-bisect/state.test.cjs b/eng/regression-bisect/state.test.cjs new file mode 100644 index 00000000000..3c081b2c7d8 --- /dev/null +++ b/eng/regression-bisect/state.test.cjs @@ -0,0 +1,535 @@ +const { test } = require("node:test"); +const assert = require("node:assert/strict"); +const { readFileSync } = require("node:fs"); +const { join } = require("node:path"); +const f = require("./fixtures/contracts.cjs"); +const model = require("./model.cjs"); +const { + initializeCheckpoint, serializeWorkflowRecord, authenticateWorkflowRecord, + loadCheckpoint, reserveWork, commitObservation, invalidateStaleEvidence, +} = require("./state.cjs"); + +const clone = structuredClone; +function harness() { + const a = f.admission(); + const context = { + repository: "dotnet/fsharp", issueNumber: 123, commentId: 456, + actor: { id: 77, login: "github-actions[bot]", type: "Bot", appId: 88 }, + workflowPath: ".github/workflows/regression-bisect.yml", + workflowRef: "refs/heads/main", trustedWorkflowRevisions: [f.sha("workflow")], + keys: { fixture: Buffer.from("synthetic-test-key-not-a-production-secret") }, keyId: "fixture", + owner: { runId: "12345", attempt: 1 }, now: "2026-09-18T00:00:00Z", + issue: { number: 123, state: "open", created_at: "2026-09-01T00:00:00Z", labels: ["Regression"] }, + issueFingerprint: a.manifest.discussionFingerprint, + manifest: a.manifest, authorization: a.budget.authorization, policy: a.budget.policy, + policyRevision: a.budget.policyRevision, workflowRevision: a.budget.workflowRevision, + pinnedMainSha: a.ancestry.pinnedMainSha, stableSdk: a.budget.stableSdk, + }; + const runs = new Map(); + const run = owner => ({ + id: Number(owner.runId), run_attempt: owner.attempt, path: context.workflowPath, + head_sha: context.workflowRevision, head_branch: "main", + repository: { full_name: context.repository }, head_repository: { full_name: context.repository }, + status: "in_progress", conclusion: null, + }); + runs.set("12345/1", run(context.owner)); + let comment = { + id: 456, user: { id: 77, login: "github-actions[bot]", type: "Bot" }, + performed_via_github_app: { id: 88 }, + issue_url: "https://api.github.com/repos/dotnet/fsharp/issues/123", + body: "unused placeholder", + }; + let tail = Promise.resolve(); + const transport = { + getComment: async id => { assert.equal(id, 456); return clone(comment); }, + updateComment: async (id, body) => { assert.equal(id, 456); comment.body = body; return clone(comment); }, + getRun: async (id, attempt) => { + const value = runs.get(`${id}/${attempt}`); + if (!value) throw new Error("run unavailable"); + return clone(value); + }, + readEvidence: async () => Buffer.from("durable evidence"), + }; + return { + context, transport, runs, + get comment() { return comment; }, + set comment(c) { comment = c; }, + // This stands for the controller's whole issue concurrency group, not comment CAS. + lock(action) { + const next = tail.then(action); + tail = next.catch(() => {}); + return next; + }, + async start() { + context.bootstrapDigest = model.fingerprintInput(comment.body); + return initializeCheckpoint(transport, context, { + allowanceId: "maintainer-approved-investigation-1", deadline: "2026-10-01T00:00:00Z", + }); + }, + nextRun() { + runs.get(`${context.owner.runId}/${context.owner.attempt}`).status = "completed"; + runs.get(`${context.owner.runId}/${context.owner.attempt}`).conclusion = "cancelled"; + context.owner = { runId: String(Number(context.owner.runId) + 1), attempt: 1 }; + runs.set(`${context.owner.runId}/1`, run(context.owner)); + }, + }; +} + +const request = (sourceSha = f.sha("main"), endpoint = "main") => ({ + kind: "source", target: { sourceSha }, endpoint, phase: "main", +}); +const reserve = (h, generation = 0, attemptId = "attempt-1", work = request()) => + reserveWork(h.transport, h.context, { generation, attemptId, request: work }); +function completion(h, pending, outcome = "product-failure") { + const data = f.observation(pending.request.kind, outcome, h.context.manifest, pending.request.target.sourceSha); + data.buildInstanceId = pending.id; + data.provenance.runId = pending.owner.runId; + for (const cell of data.cells) { + cell.attempts = cell.attempts.map(a => ({ ...a, execution: { + ...a.execution, ...h.context.manifest.assertions[outcome === "pass" ? "pass" : "failure"], + } })); + cell.outcome = model.classifyAttempts(cell.attempts, h.context.manifest); + for (const artifact of Object.values(cell.artifacts)) artifact.buildInstanceId = pending.id; + } + data.outcome = data.cells[0].outcome; + return { generation: pending.generation, attemptId: pending.id, observation: data, evidence: [] }; +} + +test("checkpoint authenticates exact author, app, comment, issue and live run provenance", async t => { + const mutations = { + author: h => { h.comment.user.id++; }, + app: h => { h.comment.performed_via_github_app.id++; }, + displayName: h => { h.comment.user.login = "impostor"; }, + comment: h => { h.comment.id++; }, + issue: h => { h.comment.issue_url += "4"; }, + body: h => { h.comment.body = h.comment.body.replace("discovery", "complete"); }, + runRepository: h => { h.runs.get("12345/1").repository.full_name = "attacker/fsharp"; }, + runHeadRepository: h => { h.runs.get("12345/1").head_repository.full_name = "attacker/fsharp"; }, + runRef: h => { h.runs.get("12345/1").head_branch = "untrusted"; }, + runRevision: h => { h.runs.get("12345/1").head_sha = f.sha("attacker"); }, + runPath: h => { h.runs.get("12345/1").path = ".github/workflows/other.yml"; }, + runAttempt: h => { h.runs.get("12345/1").run_attempt = 2; }, + key: h => { h.context.keys.fixture = Buffer.from("wrong-key"); }, + }; + for (const [name, mutate] of Object.entries(mutations)) { + await t.test(name, async () => { + const h = harness(); await h.start(); mutate(h); + await assert.rejects(loadCheckpoint(h.transport, h.context), /authentic|provenance/i); + }); + } + const h = harness(); await h.start(); + const loaded = await loadCheckpoint(h.transport, h.context); + assert.equal(loaded.executionAllowed, true); + assert.equal(loaded.commentId, 456); + assert.deepEqual(loaded.owner, { runId: "12345", attempt: 1 }); + assert.equal(loaded.checkpoint.state.budget.consumed.workerUnits, 0); + await assert.rejects(h.start(), /bootstrap|initializ/i); +}); + +test("only authenticated workflow records may be excluded from issue input", async () => { + const h = harness(); await h.start(); + assert.equal(await authenticateWorkflowRecord(h.transport, h.context, h.comment), true); + h.comment.user.id++; + assert.equal(await authenticateWorkflowRecord(h.transport, h.context, h.comment), false); + h.comment.user.id--; + h.comment.body = "\n```json\n{}\n```"; + assert.equal(await authenticateWorkflowRecord(h.transport, h.context, h.comment), false); +}); + +test("malformed, unknown-schema and oversized checkpoints fail closed without replacement", async t => { + for (const body of ["", "```json\n{\n```", "x".repeat(48 * 1024 + 1)]) { + await t.test(`length ${body.length}`, async () => { + const h = harness(); await h.start(); h.comment.body = body; + await assert.rejects(loadCheckpoint(h.transport, h.context)); + assert.equal(h.comment.body, body); + }); + } + const h = harness(); const checkpoint = await h.start(); + checkpoint.schemaVersion = 99; + h.comment.body = serializeWorkflowRecord({ kind: "checkpoint", status: "discovery", payload: checkpoint }, h.context); + await assert.rejects(loadCheckpoint(h.transport, h.context), /schema/i); + h.transport.getComment = async () => { throw new Error("missing authoritative comment"); }; + await assert.rejects(loadCheckpoint(h.transport, h.context), /missing authoritative/); +}); + +test("signed malformed ledgers cannot lose reservations or manufacture authenticated evidence", async t => { + for (const mutate of [ + cp => { cp.extra = true; }, + cp => { cp.state.budget.consumed.workerUnits = 0; }, + cp => { cp.records = []; cp.sequence = 0; }, + cp => { cp.pending = null; cp.state.budget.pendingWork = null; }, + cp => { cp.state.budget.authenticatedObservationDigests = [f.digest("fabricated")]; }, + ]) { + await t.test("invalid ledger", async () => { + const h = harness(); await h.start(); await reserve(h); + const cp = (await loadCheckpoint(h.transport, h.context)).checkpoint; + mutate(cp); + h.comment.body = serializeWorkflowRecord({ kind: "checkpoint", status: "tampered", payload: cp }, h.context); + await assert.rejects(loadCheckpoint(h.transport, h.context)); + }); + } +}); + +test("state is not approval and manual invocation cannot bypass current eligibility", async t => { + for (const change of [ + h => { h.context.issue.labels = ["regression"]; }, + h => { h.context.issue.state = "closed"; }, + h => { h.context.issue.pull_request = {}; }, + h => { h.context.issue.created_at = "2026-08-31T23:59:59Z"; }, + h => { h.context.authorization = null; }, + h => { h.context.authorization.approvalRevision = f.sha("other"); }, + h => { h.context.issueFingerprint = f.digest("correction"); }, + ]) { + await t.test("current context", async () => { + const h = harness(); await h.start(); change(h); + const loaded = await loadCheckpoint(h.transport, h.context); + assert.equal(loaded.executionAllowed, false); + assert.equal(loaded.publicationAllowed, false); + await assert.rejects(reserve(h), /blocked|eligible|approval|stale/i); + }); + } +}); + +test("reservation is durable before execution and duplicate dispatch cannot charge twice", async () => { + const h = harness(); await h.start(); + const first = await reserve(h); + const duplicate = await reserve(h); + assert.equal(first.execute, true); + assert.equal(duplicate.execute, false); + assert.deepEqual({ ...duplicate, execute: true }, first); + const loaded = (await loadCheckpoint(h.transport, h.context)).checkpoint; + assert.equal(loaded.generation, 1); + assert.deepEqual(loaded.state.budget.consumed, { sourceBuilds: 1, workerUnits: 1, reservedMinutes: 110 }); + assert.equal(loaded.pending.id, "attempt-1"); + assert.deepEqual(loaded.pending.request, request()); + assert.equal(loaded.pending.deadline, "2026-09-18T01:50:00.000Z"); + await assert.rejects(reserve(h, 1, "attempt-2"), /active|pending/i); + await assert.rejects(reserve(h, 1, "attempt-1", request(f.sha("other"))), /identity|reuse|conflict/i); +}); + +test("lost write acknowledgement cannot cause duplicate execution under the same reservation", async () => { + const h = harness(); await h.start(); + const update = h.transport.updateComment; + h.transport.updateComment = async (...args) => { await update(...args); throw new Error("response lost"); }; + await assert.rejects(reserve(h), /response lost/); + h.transport.updateComment = update; + const duplicate = await reserve(h); + assert.equal(duplicate.execute, false); + assert.equal((await loadCheckpoint(h.transport, h.context)).checkpoint.state.budget.consumed.workerUnits, 1); +}); + +test("expired reservations cannot issue another execution permission", async () => { + const h = harness(); await h.start(); await reserve(h); + h.context.now = "2026-09-18T02:00:00Z"; + const duplicate = await reserve(h); + assert.equal(duplicate.execute, false); + assert.equal((await loadCheckpoint(h.transport, h.context)).checkpoint.state.budget.attempts[0].count, 1); +}); + +test("transport failure never returns executable work or partially alters caller input", async () => { + const h = harness(); await h.start(); + const before = h.comment.body; + h.transport.updateComment = async () => { throw new Error("checkpoint write failed"); }; + await assert.rejects(reserve(h), /checkpoint write failed/); + assert.equal(h.comment.body, before); +}); + +test("controller lock serializes concurrent requests; stale generation and owner are rejected", async () => { + const h = harness(); await h.start(); + const results = await Promise.allSettled([ + h.lock(() => reserve(h, 0, "one")), + h.lock(() => reserve(h, 0, "two")), + ]); + assert.deepEqual(results.map(r => r.status), ["fulfilled", "rejected"]); + assert.match(results[1].reason.message, /generation/); + const pending = results[0].value; + const done = completion(h, pending); + h.context.owner = { runId: "54321", attempt: 1 }; + await assert.rejects(commitObservation(h.transport, h.context, done), /owner/); + h.context.owner = pending.owner; + done.generation--; + await assert.rejects(commitObservation(h.transport, h.context, done), /generation/); +}); + +test("commit binds exact request, build identity, owner and observation; duplicate completion is idempotent", async () => { + const h = harness(); await h.start(); + const pending = await reserve(h); + const done = completion(h, pending); + const wrongTarget = clone(done); + wrongTarget.observation = f.observation("source", "pass", h.context.manifest, f.sha("other")); + await assert.rejects(commitObservation(h.transport, h.context, wrongTarget), /request|identity|build/); + const checkpoint = await commitObservation(h.transport, h.context, done); + assert.equal(checkpoint.pending, null); + assert.equal(checkpoint.state.endpointEvidence.main, model.fingerprintInput(done.observation)); + assert.equal(checkpoint.state.testedRevisions.length, 1); + assert.equal(checkpoint.records[0].status, "committed"); + assert.deepEqual(await commitObservation(h.transport, h.context, done), checkpoint); + const changed = completion(h, pending, "pass"); + await assert.rejects(commitObservation(h.transport, h.context, changed), /identity|conflict/); + await assert.rejects(commitObservation(h.transport, h.context, { ...done, generation: 999 }), /generation/); + await assert.rejects(reserve(h, 1, "next"), /generation/); +}); + +test("a cancelled owner cannot commit a late worker result", async () => { + const h = harness(); await h.start(); + const pending = await reserve(h); + const done = completion(h, pending); + h.runs.get("12345/1").status = "completed"; + h.runs.get("12345/1").conclusion = "cancelled"; + await assert.rejects(commitObservation(h.transport, h.context, done), /active|owner|completed/); + assert.equal((await loadCheckpoint(h.transport, h.context)).checkpoint.pending.id, pending.id); +}); + +test("all three cancellation points preserve charges and replay only uncommitted work", async t => { + for (const point of ["before-observation", "before-commit", "after-commit"]) { + await t.test(point, async () => { + const h = harness(); await h.start(); + const pending = await reserve(h); + const done = completion(h, pending); + if (point === "after-commit") await commitObservation(h.transport, h.context, done); + h.nextRun(); + const loaded = (await loadCheckpoint(h.transport, h.context)).checkpoint; + if (point === "after-commit") { + assert.equal(loaded.pending, null); + assert.equal(Object.keys(loaded.observations).length, 1); + const next = await reserve(h, loaded.generation, "next", request(f.sha("good"), "good")); + assert.equal(next.sequence, 2); + } else { + const retry = await reserve(h, loaded.generation, "retry"); + assert.equal(retry.sequence, 2); + assert.equal(retry.attempt, 2); + assert.equal((await loadCheckpoint(h.transport, h.context)).checkpoint.records[0].status, "cancelled"); + await assert.rejects(commitObservation(h.transport, h.context, done), /owner|generation|cancelled/); + } + assert.deepEqual((await loadCheckpoint(h.transport, h.context)).checkpoint.state.budget.consumed, + { sourceBuilds: 2, workerUnits: 2, reservedMinutes: 220 }); + }); + } +}); + +test("active, unknown or unreadable owning run cannot be reclaimed even after deadline", async t => { + for (const status of ["in_progress", "queued", "waiting", "requested", "pending", "unknown", "unreadable"]) { + await t.test(status, async () => { + const h = harness(); await h.start(); await reserve(h); + h.context.now = "2026-09-19T00:00:00Z"; + h.runs.get("12345/1").status = status; + if (status === "unreadable") h.transport.getRun = async () => { throw new Error("run unavailable"); }; + await assert.rejects(reserve(h, 1, "retry"), /active|run|pending/); + assert.equal(h.comment.body.includes("retry"), false); + }); + } +}); + +test("two attempts per item, cumulative reservation limit and absolute deadline survive redispatch", async () => { + const h = harness(); await h.start(); await reserve(h); h.nextRun(); + await reserve(h, 1, "retry"); h.nextRun(); + await assert.rejects(reserve(h, 2, "third"), /attempt/); + let generation = 2; + for (let i = 0; i < 4; i++) { + const p = await reserve(h, generation, `different-${i}`, request(f.sha(`revision-${i}`), null)); + generation = p.generation; + h.nextRun(); + } + await assert.rejects(reserve(h, generation, "exhausted", request(f.sha("last"), null)), /budget/); + const totals = (await loadCheckpoint(h.transport, h.context)).checkpoint.state.budget.consumed; + assert.equal(totals.reservedMinutes, 660); + h.context.now = "2026-10-02T00:00:00Z"; + await assert.rejects(reserve(h, generation, "late", request(f.sha("last"), null)), /deadline/); +}); + +test("input changes and label toggles invalidate evidence but never replenish an allowance", async () => { + const h = harness(); await h.start(); + const p = await reserve(h); + await commitObservation(h.transport, h.context, completion(h, p)); + const before = (await loadCheckpoint(h.transport, h.context)).checkpoint.state.budget.consumed; + h.context.issue.labels = []; + let cp = await invalidateStaleEvidence(h.transport, h.context, { generation: 2 }); + assert.equal(cp.state.phase, "stopped"); + assert.equal(Object.keys(cp.observations).length, 1); + h.context.issue.labels = ["Regression"]; + h.context.manifest.discussionFingerprint = f.digest("later correction"); + h.context.issueFingerprint = h.context.manifest.discussionFingerprint; + f.seal(h.context.manifest); + h.context.authorization.manifestDigest = model.fingerprintInput(h.context.manifest); + h.context.authorization.inputDigest = h.context.manifest.approvedInputDigest; + cp = await invalidateStaleEvidence(h.transport, h.context, { generation: cp.generation }); + assert.deepEqual(cp.state.budget.consumed, before); + assert.equal(cp.state.budget.allowanceId, "maintainer-approved-investigation-1"); + assert.deepEqual(cp.state.endpointEvidence, { stable: null, main: null, good: null }); + assert.equal(Object.values(cp.observations)[0].valid, false); + assert.equal(cp.state.publication.commentId, null); + assert.equal(cp.publicationStatus, "unpublished"); + assert.equal((await loadCheckpoint(h.transport, h.context)).executionAllowed, true); +}); + +test("invalidating an active run cannot release its reservation; stopped publication retains its audit trail", async () => { + const h = harness(); await h.start(); await reserve(h); + h.context.issue.labels = []; + await assert.rejects(invalidateStaleEvidence(h.transport, h.context, { generation: 1 }), /active/); + const active = await loadCheckpoint(h.transport, h.context); + assert.equal(active.executionAllowed, false); + assert.notEqual(active.checkpoint.pending, null); + h.nextRun(); + const cp = await invalidateStaleEvidence(h.transport, h.context, { generation: 1 }); + assert.equal(cp.pending, null); + assert.equal(cp.records[0].status, "invalidated"); + assert.equal(cp.state.budget.consumed.workerUnits, 1); +}); + +test("withdrawn approval invalidates evidence and label removal retains published result identity", async () => { + const h = harness(); await h.start(); + const p = await reserve(h); + const published = await commitObservation(h.transport, h.context, completion(h, p)); + published.state.publication = { digest: f.digest("result"), commentId: 789 }; + published.publicationStatus = "published"; + h.comment.body = serializeWorkflowRecord({ kind: "checkpoint", status: "published", payload: published }, h.context); + h.context.issue.state = "closed"; + let cp = await invalidateStaleEvidence(h.transport, h.context, { generation: 2 }); + assert.deepEqual(cp.state.publication, published.state.publication); + assert.equal(cp.publicationStatus, "blocked"); + h.context.issue.state = "open"; + h.context.authorization = null; + cp = await invalidateStaleEvidence(h.transport, h.context, { generation: cp.generation }); + assert.equal(cp.state.phase, "blocked"); + assert.equal(Object.values(cp.observations)[0].valid, false); + assert.deepEqual(cp.state.budget.authenticatedObservationDigests, []); + assert.deepEqual(cp.state.endpointEvidence, { stable: null, main: null, good: null }); + assert.ok(cp.history.some(entry => entry.publication.commentId === 789)); +}); + +test("observation size overflow leaves the charged reservation durable and blocks completion", async () => { + const h = harness(); await h.start(); + const p = await reserve(h); const done = completion(h, p); + done.evidence = Array.from({ length: 16 }, (_, i) => ({ + required: false, sha256: f.digest("durable evidence"), + url: `https://example.com/${i}/${String(i).repeat(4000)}`, + })); + await assert.rejects(commitObservation(h.transport, h.context, done), /size|48/); + const cp = (await loadCheckpoint(h.transport, h.context)).checkpoint; + assert.equal(cp.pending.id, p.id); + assert.equal(cp.state.budget.consumed.workerUnits, 1); + assert.deepEqual(cp.observations, {}); +}); + +test("untestable observations retain ordered tested and skipped reasons and full evidence", async () => { + const h = harness(); await h.start(); + const p = await reserve(h); + const done = completion(h, p); + for (const cell of done.observation.cells) { + cell.attempts.forEach(a => { a.execution.status = "timeout"; }); + cell.outcome = model.classifyAttempts(cell.attempts, h.context.manifest); + } + done.observation.outcome = done.observation.cells[0].outcome; + const cp = await commitObservation(h.transport, h.context, done); + assert.equal(cp.state.skips[0].reason, "timeout"); + assert.equal(cp.state.skips[0].observationDigest, cp.state.testedRevisions[0].observationDigest); +}); + +test("changed approval, reproduction, producer and orchestration invalidate dependent pending work", async t => { + for (const change of [ + h => { h.context.manifest.sources[0].sha256 = f.digest("corrected source"); }, + h => { h.context.manifest.assertions.failure.stderr = "corrected assertion"; }, + h => { h.context.manifest.producer.sha256 = f.digest("new producer"); }, + h => { h.context.policyRevision = f.sha("new approval"); h.context.authorization.approvalRevision = h.context.policyRevision; }, + h => { h.context.workflowRevision = f.sha("new workflow"); h.context.trustedWorkflowRevisions.push(h.context.workflowRevision); }, + ]) { + await t.test("changed dependency", async () => { + const h = harness(); await h.start(); const p = await reserve(h); const done = completion(h, p); + h.nextRun(); change(h); + h.runs.get(`${h.context.owner.runId}/1`).head_sha = h.context.workflowRevision; + f.seal(h.context.manifest); + h.context.authorization.manifestDigest = model.fingerprintInput(h.context.manifest); + h.context.authorization.inputDigest = h.context.manifest.approvedInputDigest; + assert.equal((await loadCheckpoint(h.transport, h.context)).executionAllowed, false); + const cp = await invalidateStaleEvidence(h.transport, h.context, { generation: 1 }); + assert.equal(cp.pending, null); + assert.equal(cp.records[0].status, "invalidated"); + assert.equal(cp.state.budget.consumed.workerUnits, 1); + await assert.rejects(commitObservation(h.transport, h.context, done)); + }); + } +}); + +test("moving main and stable start endpoint epochs while retaining exact historical source evidence", async () => { + const h = harness(); await h.start(); + const p = await reserve(h, 0, "good", request(f.sha("good"), "good")); + const done = completion(h, p, "pass"); + await commitObservation(h.transport, h.context, done); + const old = (await loadCheckpoint(h.transport, h.context)).checkpoint; + h.context.pinnedMainSha = f.sha("new main"); + let cp = await invalidateStaleEvidence(h.transport, h.context, { generation: old.generation }); + assert.notEqual(cp.state.epoch, old.state.epoch); + assert.equal(cp.state.endpointEvidence.good, model.fingerprintInput(done.observation)); + assert.equal(cp.observations[model.fingerprintInput(done.observation)].valid, true); + assert.equal(cp.state.testedRevisions.length, 1); + assert.equal(cp.state.interval, null); + h.context.stableSdk.releaseIndexDigest = f.digest("new release metadata"); + cp = await invalidateStaleEvidence(h.transport, h.context, { generation: cp.generation }); + assert.equal(cp.state.endpointEvidence.good, model.fingerprintInput(done.observation)); + assert.equal(cp.history.length, 2); +}); + +test("mandatory external evidence loss blocks reuse; optional expired logs are disclosed", async t => { + for (const required of [true, false]) { + await t.test(required ? "mandatory" : "optional", async () => { + const h = harness(); await h.start(); + const p = await reserve(h); + const done = completion(h, p); + done.evidence = [{ required, sha256: f.digest("durable evidence"), url: "https://github.com/dotnet/fsharp/actions/runs/12345/artifacts/1" }]; + await commitObservation(h.transport, h.context, done); + h.transport.readEvidence = async () => null; + const loaded = await loadCheckpoint(h.transport, h.context); + assert.equal(loaded.executionAllowed, !required); + assert.equal(loaded.publicationAllowed, !required); + assert.match(loaded.warnings.join(), /missing.*evidence|expired/i); + assert.equal(loaded.checkpoint.state.budget.consumed.workerUnits, 1); + }); + } +}); + +test("external evidence hashes and malformed observations cannot be accepted", async () => { + const h = harness(); await h.start(); + const p = await reserve(h); const done = completion(h, p); + done.evidence = [{ required: true, sha256: f.digest("not the data"), url: "https://example.com/evidence" }]; + await assert.rejects(commitObservation(h.transport, h.context, done), /evidence/); + done.evidence = []; done.observation.cells[0].attempts = []; + await assert.rejects(commitObservation(h.transport, h.context, done), /attempt|evidence/); + assert.notEqual((await loadCheckpoint(h.transport, h.context)).checkpoint.pending, null); +}); + +test("repeated identity/evidence tables compact below 48 KiB without dropping observations", async () => { + const h = harness(); + h.context.manifest.assertions.failure.stderr = "x".repeat(12000); + f.seal(h.context.manifest); + h.context.authorization.manifestDigest = model.fingerprintInput(h.context.manifest); + h.context.authorization.inputDigest = h.context.manifest.approvedInputDigest; + await h.start(); + const p = await reserve(h); + const done = completion(h, p); + await commitObservation(h.transport, h.context, done); + assert.ok(Buffer.byteLength(h.comment.body) < 48 * 1024); + const cp = (await loadCheckpoint(h.transport, h.context)).checkpoint; + assert.deepEqual(cp.observations[model.fingerprintInput(done.observation)].data, done.observation); + cp.history.push({ reason: "x".repeat(48 * 1024) }); + assert.throws(() => serializeWorkflowRecord({ kind: "checkpoint", status: "blocked", payload: cp }, h.context), /size|48/); +}); + +test("serialized checkpoint fixture replays reservation, observation, cancellation and resume", async () => { + const h = harness(); + h.comment.body = readFileSync(join(__dirname, "fixtures", "checkpoint.txt"), "utf8").trimEnd(); + const initial = await loadCheckpoint(h.transport, h.context); + assert.equal(initial.checkpoint.generation, 0); + const first = await reserve(h); + await commitObservation(h.transport, h.context, completion(h, first)); + const uncommitted = await reserve(h, 2, "cancelled", request(f.sha("good"), "good")); + completion(h, uncommitted, "pass"); + h.nextRun(); + const resumed = await reserve(h, 3, "resumed", request(f.sha("good"), "good")); + const cp = await commitObservation(h.transport, h.context, completion(h, resumed, "pass")); + assert.equal(Object.keys(cp.observations).length, 2); + assert.equal(cp.records[1].status, "cancelled"); + assert.deepEqual(cp.state.budget.consumed, { sourceBuilds: 3, workerUnits: 3, reservedMinutes: 330 }); + assert.equal(cp.generation, 5); +}); + +module.exports = { harness }; From c2ef90845a2720fb4fcdbb520176478861f80c8f Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 00:38:18 +0200 Subject: [PATCH 07/99] Prevent rollback of authenticated regression checkpoints Bind checkpoint generations and exact bodies to independent signed commit-status receipts. Recover interrupted writes only through authenticated predecessor links, without refunding reservations. Populate endpoint and interval tests to enforce selective invalidation and retained historical evidence. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 44 ++++- eng/regression-bisect/fixtures/checkpoint.txt | 2 +- eng/regression-bisect/fixtures/contracts.cjs | 23 ++- eng/regression-bisect/intake.test.cjs | 20 +- eng/regression-bisect/state.cjs | 98 +++++++++- eng/regression-bisect/state.test.cjs | 175 +++++++++++++++--- 6 files changed, 317 insertions(+), 45 deletions(-) diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index f3a1576452f..39fef51efd5 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -277,6 +277,10 @@ The trusted controller supplies `context`: * `repository`, `issueNumber`, and the exact authoritative `commentId`; `actor: { id, login, type, appId }` are pinned numeric GitHub author/app identities, not just the shared Actions bot display name. +* `checkpointAnchorSha` is the immutable trusted commit SHA from the initial + checkpoint creation receipt. Retain that same anchor across main advances, + policy changes, redispatch, and key rotation; never derive it from issue text, + the current comment, or a disposable cache. * `workflowPath`, `workflowRef` (for example `refs/heads/main`), `workflowRevision`, and `trustedWorkflowRevisions`. Old orchestration revisions may be retained only after independent maintainer review. @@ -298,6 +302,7 @@ The transport implements these asynchronous methods: | `updateComment(commentId, body)` | The full updated comment. The helper verifies the returned body and provenance before granting execution. | | `getRun(runId, attempt)` | The **specific run attempt**, with REST `id`, `run_attempt`, `path`, `head_sha`, `head_branch`, `repository.full_name`, `head_repository.full_name`, `status`, and `conclusion`. Do not silently substitute the latest rerun. | | `readEvidence(url)` | Verified downloaded bytes as a `Buffer`, or `null` for confirmed missing/expired content. Authentication, rate-limit, and network errors throw. The controller restricts downloads to its evidence stores. | +| `request(route, params)` | Octokit-style REST `{ data, headers, status? }` for listing **all** commit statuses on the pinned anchor and appending a status. Pass through pagination headers and errors; never use the combined/latest-status endpoint or a cached status list. | The versioned envelope has a human-readable authenticated status, a compact payload, and HMAC-SHA256 over its canonical contents. Its signed binding includes @@ -320,6 +325,41 @@ present. Initialization refuses an existing envelope. There is deliberately no automatic reset/new-allowance operation; ordinary issue edits, label toggles, redispatches, and cancellations cannot replenish any counter. +### Independent freshness + +A valid HMAC authenticates an old body too. Every checkpoint write therefore +appends a signed freshness receipt to GitHub's commit-status history on +`checkpointAnchorSha`, under `regression-bisect//`. +Receipts bind the repository, issue, comment, anchor, generation, exact body +digest, and key identity. Reads verify the numeric writer identity and signature +and select the **highest authenticated generation across every page**, not the +most recently posted status. Reposting an old receipt cannot lower that mark. +The status is only an anti-rollback witness; the single issue comment still +contains the authoritative observations, reservation ledger, and publication +state. No memory branch or product-content writer is introduced. + +The controller needs `statuses: read` for normal loads and `statuses: write` for +transitions/recovery, in addition to its issue-comment permissions. The +[commit-status REST API](https://docs.github.com/en/rest/commits/statuses) +appends statuses rather than editing/deleting them and does not depend on +Actions log/artifact retention. Keep the anchor commit reachable and retain +verification keys for existing receipts. Unavailable history, invalid receipts, +conflicting digests, or more than ten pages of 100 statuses block work. +Exhausting GitHub's per-context status limit likewise blocks, never resets the +allowance. This context must not be configured as a required product-build check. + +The signed checkpoint includes `previousDigest`, the exact predecessor body +hash (the trusted bootstrap digest for generation zero). A transition writes +the comment, then its receipt, and grants execution only after both succeed. +Under the issue lock, `loadCheckpoint` can finish the receipt write for exactly +one authenticated successor whose predecessor matches the independent mark. +This preserves observations when cancellation interrupts the two writes; it +does not grant a second execution permission. An older body, a changed body at +the same generation, a generation gap, or a wrong predecessor fails closed. +An interrupted initial write can only be completed by `initializeCheckpoint` +with the original bootstrap receipt and the identical initial allowance. +Missing receipts are never synthesized from arbitrary loaded state. + ### Controller transaction Hold one issue-scoped controller lock across **read, reserve, worker, commit, @@ -329,12 +369,12 @@ lock. All state-transition calls reread the authoritative comment. | Export | Behavior | | --- | --- | -| `loadCheckpoint(transport, context)` | Returns `{ checkpoint, commentId, owner, executionAllowed, publicationAllowed, blockers, warnings }`. Invalid authentication/schema throws; stale approvals/inputs or required evidence loss explicitly block reuse. | +| `loadCheckpoint(transport, context)` | Returns `{ checkpoint, bodyDigest, commentId, owner, executionAllowed, publicationAllowed, blockers, warnings }`. Authenticates freshness and can finish an interrupted receipt write under the issue lock. Invalid authentication/schema/freshness throws; stale approvals/inputs or required evidence loss explicitly block reuse. | | `reserveWork(transport, context, { generation, attemptId, request })` | Durably charges and saves the exact request before returning a reservation with `execute: true`. Duplicate requests return `execute: false`; they **must not launch another worker**. Conflicting attempt IDs, stale generations, active pending owners, exhausted counters, and deadlines fail closed. | | `commitObservation(transport, context, { generation, attemptId, observation, evidence })` | Checks ownership, request/build/provenance and the shared observation contract, then commits all decisive evidence before clearing pending work. Exact duplicate completion is idempotent. | | `invalidateStaleEvidence(transport, context, { generation })` | Invalidates changed input/approval/producer/orchestration dependencies without erasing history or charges. Main/stable changes start a new endpoint epoch while retaining exact-input historical source observations. Closed/unlabeled issues stop publication; existing result identities remain auditable. | | `serializeWorkflowRecord({ kind, status, payload }, context)` | Signs a `checkpoint`, `proposal`, or `result` envelope. Low-level serialization is not approval or a storage transition. | -| `authenticateWorkflowRecord(transport, context, comment)` | Returns true only for a matching signed record and fetched run provenance. Forged records return false; required API failures throw. Bind the expected record ID before using this as intake's exclusion callback. | +| `authenticateWorkflowRecord(transport, context, comment)` | Returns true only for a matching signed record and fetched run provenance, not its freshness. Forged records return false; required API failures throw. Bind the expected record ID before using this as intake's exclusion callback, and use `loadCheckpoint` in the mandatory trusted-checkpoint reader to detect rollback before freezing. | A work request is `{ kind, target, endpoint, phase }`: `kind`/`target` use the observation shape; diff --git a/eng/regression-bisect/fixtures/checkpoint.txt b/eng/regression-bisect/fixtures/checkpoint.txt index 12a3864c8fa..12ead961515 100644 --- a/eng/regression-bisect/fixtures/checkpoint.txt +++ b/eng/regression-bisect/fixtures/checkpoint.txt @@ -1,5 +1,5 @@ Regression investigation: discovery (generation 0). ```json -{"schemaVersion":1,"kind":"checkpoint","status":"Regression investigation: discovery (generation 0).","binding":{"repository":"dotnet/fsharp","issueNumber":123,"commentId":456,"workflowPath":".github/workflows/regression-bisect.yml","workflowRef":"refs/heads/main","workflowRevision":"899531e0f885501b5b8520a9bd95bc83345eaa5b","runId":"12345","runAttempt":1,"keyId":"fixture"},"payload":{"root":["@",0],"table":[{"schemaVersion":1,"generation":0,"state":["@",1],"inputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a","reproductionDigest":"91815101645c42988cb8f780311ed49cd7c179071d76d17298a1c6810a965138","policyDigest":"deb550568d7b07a6d49bcbc92690ee602abc201b76329631f0dee9387b2ea09f","sequence":0,"pending":null,"records":["@",28],"observations":["@",32],"history":["@",28],"deadline":"2026-10-01T00:00:00Z","publicationStatus":"unpublished"},{"schemaVersion":1,"repository":"dotnet/fsharp","issueNumber":123,"issueFingerprint":"ba79e2bbb07eaf308944401cb8a62139d3eed051e25c88279ce781531fef37f5","approvalFingerprint":"16503092fdfbb9143595739dc023ca6e201f80203f63950dd884370b09878dcb","manifest":["@",2],"budget":["@",18],"pinnedMainSha":"b28b7af69320201d1cf206ebf28373980add1451","epoch":"09eea1ee5aea50efd7ac23639a5050c834e1824d056cbcc4b054532e8c9a02a3","phase":"discovery","endpointEvidence":["@",30],"testedRevisions":["@",28],"skips":["@",28],"interval":null,"publication":["@",31]},{"schemaVersion":1,"repository":"dotnet/fsharp","issueNumber":123,"discussionFingerprint":"ba79e2bbb07eaf308944401cb8a62139d3eed051e25c88279ce781531fef37f5","kind":"fsi","sources":["@",3],"assertions":["@",5],"matrix":["@",8],"timeoutSeconds":300,"producer":["@",11],"packages":["@",12],"searchHints":["@",14],"approvedInputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a"},[["@",4]],{"path":"fixtures/repro.fsx","sha256":"e1c5329a6b6f65fb0733f62c334e0bf4bb91cfa449f5eb213dc67f0d40b440af"},{"pass":["@",6],"failure":["@",7]},{"exitCode":0,"stdout":"ok\n","stderr":""},{"exitCode":1,"stdout":"","stderr":"expected regression\n"},[["@",9]],{"id":"linux-x64","os":"linux","architecture":"x64","project":null,"tfm":"net10.0","configuration":"Release","languageVersion":"10.0","command":["@",10]},["fsi","fixtures/repro.fsx"],{"name":"repro","version":"1.0.0","sha256":"d1990f97a1f4fe7b8e86da8c03c9dd3f7d536c14aa4a6f9eedab07f34cd961b0"},[["@",13]],{"id":"Example","version":"1.0.0","sha256":"bc4a71180870f7945155fbb02f4b0a2e3faa2a62d6d31b7039013055ed19869a"},{"sdkVersions":["@",15],"sourceShas":["@",16],"urls":["@",17]},["9.0.100"],["fc19318dd13128ce14344d066510a982269c241b"],["https://github.com/dotnet/fsharp"],{"schemaVersion":1,"policy":["@",19],"policyRevision":"9f00fad98badc1330e637a0a015d704113498420","workflowRevision":"899531e0f885501b5b8520a9bd95bc83345eaa5b","allowanceId":"maintainer-approved-investigation-1","authorization":["@",24],"stableSdk":["@",25],"authenticatedObservationDigests":["@",28],"consumed":["@",29],"attempts":["@",28],"nextWorkItemId":"none","pendingWork":null},{"schemaVersion":1,"repository":"dotnet/fsharp","selection":["@",20],"authorization":["@",21],"accounting":["@",22],"limits":["@",23],"evidenceBytesPerStream":16384},{"state":"open","label":"Regression","createdAtOrAfter":"2026-09-01T00:00:00Z"},{"mode":"reviewed-manifests-only","policySource":"pinned-trusted-default-branch","approvalDirectory":".github/regression-bisect/approved","assets":"reviewed-repository-paths-and-sha256"},{"scope":"issue-allowance","reserveBeforeExecution":true,"retriesCharged":true,"cancellationRefunds":false,"inputEditsResetAllowance":false,"labelChangesResetAllowance":false,"dispatchResetsAllowance":false,"newAllowanceRequires":"maintainer-authorization"},{"sourceBuilds":24,"workerUnits":40,"reservedMinutes":720,"attemptsPerItem":2,"assertionRepetitions":2,"buildTimeoutMinutes":90,"reproductionTimeoutSeconds":300,"workerTimeoutMinutes":110,"discoveryIssues":3,"discoveryPages":2,"discoveryPageSize":100},{"repository":"dotnet/fsharp","issueNumber":123,"approvalPath":".github/regression-bisect/approved/123.json","approvalRevision":"9f00fad98badc1330e637a0a015d704113498420","manifestDigest":"16503092fdfbb9143595739dc023ca6e201f80203f63950dd884370b09878dcb","inputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a"},{"version":"10.0.100","origin":"official-release-index","latest":true,"verified":true,"releaseIndexUrl":"https://builds.dotnet.microsoft.com/dotnet/release-metadata/releases-index.json","releaseIndexDigest":"2a21ef65c47867ae1b2056a246bf7c001b12ade326bd0e1fcdfb81af140e65b7","archives":["@",26]},[["@",27]],{"os":"linux","architecture":"x64","sha256":"9d9e20df50a502c04ee629c1282495a65a279635168510572f846de604da046e"},[],{"sourceBuilds":0,"workerUnits":0,"reservedMinutes":0},{"stable":null,"main":null,"good":null},{"digest":null,"commentId":null},{}]},"signature":"d815fab6dc3815c19a9f221efb0216087b01acf3deedfb3961b7318ae36a0ad4"} +{"schemaVersion":1,"kind":"checkpoint","status":"Regression investigation: discovery (generation 0).","binding":{"repository":"dotnet/fsharp","issueNumber":123,"commentId":456,"workflowPath":".github/workflows/regression-bisect.yml","workflowRef":"refs/heads/main","workflowRevision":"899531e0f885501b5b8520a9bd95bc83345eaa5b","runId":"12345","runAttempt":1,"keyId":"fixture"},"payload":{"root":["@",0],"table":[{"schemaVersion":1,"generation":0,"previousDigest":"657b8c3151a6f2496623f2e58482895222b1808b30eee9cc032e1431a85a9347","state":["@",1],"inputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a","reproductionDigest":"91815101645c42988cb8f780311ed49cd7c179071d76d17298a1c6810a965138","policyDigest":"deb550568d7b07a6d49bcbc92690ee602abc201b76329631f0dee9387b2ea09f","sequence":0,"pending":null,"records":["@",28],"observations":["@",32],"history":["@",28],"deadline":"2026-10-01T00:00:00Z","publicationStatus":"unpublished"},{"schemaVersion":1,"repository":"dotnet/fsharp","issueNumber":123,"issueFingerprint":"ba79e2bbb07eaf308944401cb8a62139d3eed051e25c88279ce781531fef37f5","approvalFingerprint":"16503092fdfbb9143595739dc023ca6e201f80203f63950dd884370b09878dcb","manifest":["@",2],"budget":["@",18],"pinnedMainSha":"b28b7af69320201d1cf206ebf28373980add1451","epoch":"09eea1ee5aea50efd7ac23639a5050c834e1824d056cbcc4b054532e8c9a02a3","phase":"discovery","endpointEvidence":["@",30],"testedRevisions":["@",28],"skips":["@",28],"interval":null,"publication":["@",31]},{"schemaVersion":1,"repository":"dotnet/fsharp","issueNumber":123,"discussionFingerprint":"ba79e2bbb07eaf308944401cb8a62139d3eed051e25c88279ce781531fef37f5","kind":"fsi","sources":["@",3],"assertions":["@",5],"matrix":["@",8],"timeoutSeconds":300,"producer":["@",11],"packages":["@",12],"searchHints":["@",14],"approvedInputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a"},[["@",4]],{"path":"fixtures/repro.fsx","sha256":"e1c5329a6b6f65fb0733f62c334e0bf4bb91cfa449f5eb213dc67f0d40b440af"},{"pass":["@",6],"failure":["@",7]},{"exitCode":0,"stdout":"ok\n","stderr":""},{"exitCode":1,"stdout":"","stderr":"expected regression\n"},[["@",9]],{"id":"linux-x64","os":"linux","architecture":"x64","project":null,"tfm":"net10.0","configuration":"Release","languageVersion":"10.0","command":["@",10]},["fsi","fixtures/repro.fsx"],{"name":"repro","version":"1.0.0","sha256":"d1990f97a1f4fe7b8e86da8c03c9dd3f7d536c14aa4a6f9eedab07f34cd961b0"},[["@",13]],{"id":"Example","version":"1.0.0","sha256":"bc4a71180870f7945155fbb02f4b0a2e3faa2a62d6d31b7039013055ed19869a"},{"sdkVersions":["@",15],"sourceShas":["@",16],"urls":["@",17]},["9.0.100"],["fc19318dd13128ce14344d066510a982269c241b"],["https://github.com/dotnet/fsharp"],{"schemaVersion":1,"policy":["@",19],"policyRevision":"9f00fad98badc1330e637a0a015d704113498420","workflowRevision":"899531e0f885501b5b8520a9bd95bc83345eaa5b","allowanceId":"maintainer-approved-investigation-1","authorization":["@",24],"stableSdk":["@",25],"authenticatedObservationDigests":["@",28],"consumed":["@",29],"attempts":["@",28],"nextWorkItemId":"none","pendingWork":null},{"schemaVersion":1,"repository":"dotnet/fsharp","selection":["@",20],"authorization":["@",21],"accounting":["@",22],"limits":["@",23],"evidenceBytesPerStream":16384},{"state":"open","label":"Regression","createdAtOrAfter":"2026-09-01T00:00:00Z"},{"mode":"reviewed-manifests-only","policySource":"pinned-trusted-default-branch","approvalDirectory":".github/regression-bisect/approved","assets":"reviewed-repository-paths-and-sha256"},{"scope":"issue-allowance","reserveBeforeExecution":true,"retriesCharged":true,"cancellationRefunds":false,"inputEditsResetAllowance":false,"labelChangesResetAllowance":false,"dispatchResetsAllowance":false,"newAllowanceRequires":"maintainer-authorization"},{"sourceBuilds":24,"workerUnits":40,"reservedMinutes":720,"attemptsPerItem":2,"assertionRepetitions":2,"buildTimeoutMinutes":90,"reproductionTimeoutSeconds":300,"workerTimeoutMinutes":110,"discoveryIssues":3,"discoveryPages":2,"discoveryPageSize":100},{"repository":"dotnet/fsharp","issueNumber":123,"approvalPath":".github/regression-bisect/approved/123.json","approvalRevision":"9f00fad98badc1330e637a0a015d704113498420","manifestDigest":"16503092fdfbb9143595739dc023ca6e201f80203f63950dd884370b09878dcb","inputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a"},{"version":"10.0.100","origin":"official-release-index","latest":true,"verified":true,"releaseIndexUrl":"https://builds.dotnet.microsoft.com/dotnet/release-metadata/releases-index.json","releaseIndexDigest":"2a21ef65c47867ae1b2056a246bf7c001b12ade326bd0e1fcdfb81af140e65b7","archives":["@",26]},[["@",27]],{"os":"linux","architecture":"x64","sha256":"9d9e20df50a502c04ee629c1282495a65a279635168510572f846de604da046e"},[],{"sourceBuilds":0,"workerUnits":0,"reservedMinutes":0},{"stable":null,"main":null,"good":null},{"digest":null,"commentId":null},{}]},"signature":"d9f7ba4f55dafa5ae32fcb37b5a9be6e85dbbfe424b7b1b48b68385b166e533b"} ``` diff --git a/eng/regression-bisect/fixtures/contracts.cjs b/eng/regression-bisect/fixtures/contracts.cjs index 9999e141488..94dc798288f 100644 --- a/eng/regression-bisect/fixtures/contracts.cjs +++ b/eng/regression-bisect/fixtures/contracts.cjs @@ -112,4 +112,25 @@ function state() { }; } -module.exports = { policy, manifest, attempt, observation, admission, state, seal, sha, digest }; +function statusTransport(context) { + const statuses = []; + return { + statuses, + async request(route, args) { + if (args.owner !== "dotnet" || args.repo !== "fsharp" || + (args.sha ?? args.ref) !== context.checkpointAnchorSha) throw new Error("unexpected freshness repository or anchor"); + if (route === "POST /repos/{owner}/{repo}/statuses/{sha}") { + const status = { ...structuredClone(args), id: statuses.length + 1, creator: structuredClone(context.actor) }; + statuses.push(status); + return { data: structuredClone(status) }; + } + if (route !== "GET /repos/{owner}/{repo}/commits/{ref}/statuses") throw new Error("unexpected freshness route"); + return { + data: structuredClone(statuses.toReversed().slice((args.page - 1) * 100, args.page * 100)), + headers: { link: args.page * 100 < statuses.length ? '; rel="next"' : "" }, + }; + }, + }; +} + +module.exports = { policy, manifest, attempt, observation, admission, state, seal, sha, digest, statusTransport }; diff --git a/eng/regression-bisect/intake.test.cjs b/eng/regression-bisect/intake.test.cjs index 8970466a446..da043c1c3d6 100644 --- a/eng/regression-bisect/intake.test.cjs +++ b/eng/regression-bisect/intake.test.cjs @@ -1,6 +1,6 @@ const test = require("node:test"); const assert = require("node:assert/strict"); -const { manifest, seal, digest, state, sha } = require("./fixtures/contracts.cjs"); +const { manifest, seal, digest, state, sha, statusTransport } = require("./fixtures/contracts.cjs"); const { fingerprintInput, isEligibleIssue } = require("./model.cjs"); const intake = () => require("./intake.cjs"); const repository = "dotnet/fsharp"; @@ -156,7 +156,7 @@ test("authenticated records require an explicit trusted checkpoint reader before }); test("real checkpoint authentication excludes only the pinned signed workflow record", async () => { - const { authenticateWorkflowRecord, loadCheckpoint, reserveWork } = require("./state.cjs"); + const { initializeCheckpoint, authenticateWorkflowRecord, loadCheckpoint, reserveWork } = require("./state.cjs"); const body = require("node:fs").readFileSync(require.resolve("./fixtures/checkpoint.txt"), "utf8").trimEnd(); const actor = { id: 77, login: "github-actions[bot]", type: "Bot", appId: 88 }; const baseline = state(); @@ -164,6 +164,7 @@ test("real checkpoint authentication excludes only the pinned signed workflow re repository, issueNumber: 123, commentId: 456, actor, workflowPath: ".github/workflows/regression-bisect.yml", workflowRef: "refs/heads/main", workflowRevision: sha("workflow"), trustedWorkflowRevisions: [sha("workflow")], + checkpointAnchorSha: sha("workflow"), bootstrapDigest: fingerprintInput("unused placeholder"), keys: { fixture: Buffer.from("synthetic-test-key-not-a-production-secret") }, keyId: "fixture", owner: { runId: "12345", attempt: 1 }, now, issue: issue(), policy: baseline.budget.policy, policyRevision: baseline.budget.policyRevision, @@ -171,7 +172,7 @@ test("real checkpoint authentication excludes only the pinned signed workflow re authorization: baseline.budget.authorization, stableSdk: baseline.budget.stableSdk, pinnedMainSha: baseline.pinnedMainSha, }; let runReads = 0; - const stateTransport = { async getRun(runId, attempt) { + const stateTransport = { request: statusTransport(context).request, async getRun(runId, attempt) { assert.equal(runId, "12345"); assert.equal(attempt, 1); runReads++; @@ -182,7 +183,7 @@ test("real checkpoint authentication excludes only the pinned signed workflow re }; } }; const managed = { - ...comment(456, body), issue_url: "https://api.github.com/repos/dotnet/fsharp/issues/123", + ...comment(456, "unused placeholder"), issue_url: "https://api.github.com/repos/dotnet/fsharp/issues/123", user: { id: actor.id, login: actor.login, type: actor.type }, performed_via_github_app: { id: actor.appId }, }; const copied = comment(457, body); @@ -195,6 +196,11 @@ test("real checkpoint authentication excludes only the pinned signed workflow re managed.body = updatedBody; return structuredClone(managed); }; + await initializeCheckpoint(stateTransport, context, { + allowanceId: "maintainer-approved-investigation-1", deadline: "2026-10-01T00:00:00Z", + }); + assert.equal(managed.body, body); + runReads = 0; const f = fixture({ comments: [comment(1), managed, copied], timeline: [{ ...managed, event: "commented" }, { ...copied, event: "commented" }], @@ -230,6 +236,12 @@ test("real checkpoint authentication excludes only the pinned signed workflow re assert.equal(pending.snapshot.investigations.active.length, 1); assert.deepEqual(pending.snapshot.investigations.active[0].owner, loaded.checkpoint.pending.owner); assert.equal(pending.snapshot.inputDigest, result.snapshot.inputDigest); + const currentBody = managed.body; + managed.body = body; + f.data.timeline[0].body = body; + blocked(await snapshot(f, { authenticateRecord, readTrustedCheckpoints }), /freshness|rollback/i); + managed.body = currentBody; + f.data.timeline[0].body = currentBody; assert.equal(await authenticateRecord({ ...managed, user: copied.user }), false); assert.equal(await authenticateRecord({ ...managed, body: `${body} ` }), false); stateTransport.getRun = async () => { throw new Error("workflow run provenance unavailable"); }; diff --git a/eng/regression-bisect/state.cjs b/eng/regression-bisect/state.cjs index 6ed0bc99f14..a44b24e8d0c 100644 --- a/eng/regression-bisect/state.cjs +++ b/eng/regression-bisect/state.cjs @@ -153,10 +153,10 @@ function validateEvidence(evidence) { } function validateCheckpoint(cp) { - fields(cp, "schemaVersion generation state inputDigest reproductionDigest policyDigest sequence pending records observations history deadline publicationStatus", "checkpoint"); + fields(cp, "schemaVersion generation previousDigest state inputDigest reproductionDigest policyDigest sequence pending records observations history deadline publicationStatus", "checkpoint"); requireState(cp.schemaVersion === 1, "unsupported checkpoint schema"); validateState(cp.state); - requireState(Number.isSafeInteger(cp.generation) && cp.generation >= 0 && timestamp(cp.deadline) && + requireState(Number.isSafeInteger(cp.generation) && cp.generation >= 0 && digest(cp.previousDigest) && timestamp(cp.deadline) && cp.inputDigest === cp.state.manifest.approvedInputDigest && cp.reproductionDigest === reproductionDigest(cp.state.manifest) && cp.policyDigest === fingerprint(cp.state.budget.policy), "checkpoint digest or generation mismatch"); @@ -264,6 +264,66 @@ async function evidenceWarnings(transport, observations) { return { warnings, missingRequired }; } +function freshnessStatus(context, generation, bodyDigest, keyId = context.keyId) { + requireState(context.repository === "dotnet/fsharp" && positive(context.issueNumber) && positive(context.commentId) && + /^[a-f0-9]{40}$/.test(context.checkpointAnchorSha), "invalid trusted freshness anchor"); + const key = context.keys[keyId]; + requireState(/^[\w-]{1,64}$/.test(keyId) && Buffer.isBuffer(key) && key.length >= 32, "freshness authentication key unavailable"); + const name = `regression-bisect/${context.issueNumber}/${context.commentId}`; + const description = `checkpoint:${generation}:${bodyDigest}`; + const url = `https://github.com/${context.repository}/issues/${context.issueNumber}#checkpoint-${keyId}:`; + const signature = createHmac("sha256", key).update(fingerprint({ + anchor: context.checkpointAnchorSha, name, description, url, + })).digest("hex"); + return { state: "success", context: name, description, target_url: url + signature }; +} + +async function readFreshness(transport, context) { + const expected = freshnessStatus(context, 0, ""); + let head = null; + for (let page = 1; page <= 10; page++) { + const response = await transport.request("GET /repos/{owner}/{repo}/commits/{ref}/statuses", { + owner: "dotnet", repo: "fsharp", ref: context.checkpointAnchorSha, per_page: 100, page, + }); + requireState(response && (!response.status || response.status === 200) && + Array.isArray(response.data) && response.data.length <= 100, "incomplete freshness response"); + for (const status of response.data) { + if (status.context !== expected.context || status.creator?.id !== context.actor.id) continue; + const parts = /^checkpoint:(0|[1-9]\d*):([a-f0-9]{64})$/.exec(status.description); + const key = /#checkpoint-([\w-]{1,64}):[a-f0-9]{64}$/.exec(status.target_url); + requireState(parts && key && Number.isSafeInteger(Number(parts[1])) && + positive(status.id) && status.creator.login === context.actor.login && + status.creator.type === context.actor.type, "freshness receipt authentication failed"); + const receipt = { generation: Number(parts[1]), bodyDigest: parts[2] }; + const signed = freshnessStatus(context, receipt.generation, receipt.bodyDigest, key[1]); + requireState(Object.entries(signed).every(([k, v]) => status[k] === v), "freshness receipt authentication failed"); + requireState(head?.generation !== receipt.generation || head.bodyDigest === receipt.bodyDigest, + "conflicting freshness receipts"); + if (head === null || receipt.generation > head.generation) head = receipt; + } + const next = response.headers?.link === undefined ? response.data.length === 100 : + /;\s*rel="?next"?/.test(response.headers.link); + if (!next) return head; + } + throw new StateError("freshness page limit exceeded; history incomplete"); +} + +async function writeFreshness(transport, context, generation, bodyDigest) { + const run = await verifyRun(transport, context, { + runId: context.owner.runId, runAttempt: context.owner.attempt, workflowRevision: context.workflowRevision, + }); + requireState(run.status === "in_progress", "freshness writer must be the active owning run"); + const status = freshnessStatus(context, generation, bodyDigest); + const response = await transport.request("POST /repos/{owner}/{repo}/statuses/{sha}", { + owner: "dotnet", repo: "fsharp", sha: context.checkpointAnchorSha, ...status, + }); + const written = response?.data; + requireState((!response?.status || response.status === 201) && positive(written?.id) && + written.creator?.id === context.actor.id && written.creator?.login === context.actor.login && + written.creator?.type === context.actor.type && + Object.entries(status).every(([k, v]) => written[k] === v), "freshness write authentication failed"); +} + async function loadCheckpoint(transport, context) { const comment = await transport.getComment(context.commentId); const record = await authenticatedRecord(transport, context, comment); @@ -272,11 +332,20 @@ async function loadCheckpoint(transport, context) { validateCheckpoint(checkpoint); requireState(checkpoint.state.repository === context.repository && checkpoint.state.issueNumber === context.issueNumber, "checkpoint issue authentication failed"); + const head = await readFreshness(transport, context); + const bodyDigest = hashBytes(comment.body); + requireState(head !== null, "checkpoint freshness receipt missing; initialization or repair required"); + if (checkpoint.generation !== head.generation || bodyDigest !== head.bodyDigest) { + requireState(checkpoint.generation === head.generation + 1 && checkpoint.previousDigest === head.bodyDigest, + "checkpoint freshness rollback or predecessor mismatch"); + // Complete an interrupted comment -> receipt write, under the controller lock. + await writeFreshness(transport, context, checkpoint.generation, bodyDigest); + } const current = currentState(checkpoint, context); const evidence = await evidenceWarnings(transport, Object.values(checkpoint.observations).filter(o => o.valid)); const blockers = [...current.reasons, ...(evidence.missingRequired ? ["required evidence unavailable; blocked"] : [])]; return { - checkpoint, commentId: context.commentId, + checkpoint, bodyDigest, commentId: context.commentId, owner: { runId: record.binding.runId, attempt: record.binding.runAttempt }, executionAllowed: blockers.length === 0, publicationAllowed: blockers.length === 0 && checkpoint.state.phase !== "stopped", @@ -284,7 +353,8 @@ async function loadCheckpoint(transport, context) { }; } -async function saveCheckpoint(transport, context, checkpoint) { +async function saveCheckpoint(transport, context, checkpoint, previousDigest = checkpoint.previousDigest) { + checkpoint.previousDigest = previousDigest; validateCheckpoint(checkpoint); const owner = await verifyRun(transport, context, { runId: context.owner.runId, runAttempt: context.owner.attempt, workflowRevision: context.workflowRevision, @@ -297,13 +367,14 @@ async function saveCheckpoint(transport, context, checkpoint) { const written = await transport.updateComment(context.commentId, body); requireState(written?.body === body && await authenticateWorkflowRecord(transport, context, written), "checkpoint write authentication failed"); + await writeFreshness(transport, context, checkpoint.generation, hashBytes(body)); return checkpoint; } async function initializeCheckpoint(transport, context, { allowanceId, deadline }) { + requireState(await readFreshness(transport, context) === null, "initialization forbidden: freshness history already exists"); const existing = await transport.getComment(context.commentId); - requireState(digest(context.bootstrapDigest) && fingerprint(existing.body) === context.bootstrapDigest && - parseRecord(existing.body) === null, "initialization requires the exact trusted bootstrap placeholder"); + requireState(digest(context.bootstrapDigest), "initialization requires the trusted bootstrap digest"); requireState(existing.id === context.commentId && existing.user?.id === context.actor.id && existing.user?.login === context.actor.login && existing.user?.type === context.actor.type && existing.performed_via_github_app?.id === context.actor.appId && @@ -325,11 +396,18 @@ async function initializeCheckpoint(transport, context, { allowanceId, deadline publication: { digest: null, commentId: null }, }; const cp = { - schemaVersion: 1, generation: 0, state, inputDigest: context.manifest.approvedInputDigest, + schemaVersion: 1, generation: 0, previousDigest: context.bootstrapDigest, state, inputDigest: context.manifest.approvedInputDigest, reproductionDigest: reproductionDigest(context.manifest), policyDigest: fingerprint(context.policy), sequence: 0, pending: null, records: [], observations: {}, history: [], deadline, publicationStatus: "unpublished", }; requireState(currentState(cp, context).reasons.length === 0, "initialization blocked by current eligibility or approval"); + if (fingerprint(existing.body) !== context.bootstrapDigest || parseRecord(existing.body) !== null) { + const record = await authenticatedRecord(transport, context, existing); + requireState(record?.kind === "checkpoint" && equal(unpack(record.payload), cp), + "initialization requires the exact trusted bootstrap placeholder or interrupted initial checkpoint"); + await writeFreshness(transport, context, 0, hashBytes(existing.body)); + return cp; + } return saveCheckpoint(transport, context, structuredClone(cp)); } @@ -402,7 +480,7 @@ async function reserveWork(transport, context, { generation, attemptId, request budget.nextWorkItemId = workItemId; budget.pendingWork = { id: attemptId, allowanceId: budget.allowanceId, workItemId, attempt, reserved, status: "reserved" }; cp.state.phase = request.phase; - await saveCheckpoint(transport, context, cp); + await saveCheckpoint(transport, context, cp, loaded.bodyDigest); return { ...structuredClone(pending), execute: true }; } @@ -447,7 +525,7 @@ async function commitObservation(transport, context, { generation, attemptId, ob cp.pending = null; cp.state.budget.pendingWork = null; cp.generation++; - return saveCheckpoint(transport, context, cp); + return saveCheckpoint(transport, context, cp, loaded.bodyDigest); } async function invalidateStaleEvidence(transport, context, { generation }) { @@ -494,7 +572,7 @@ async function invalidateStaleEvidence(transport, context, { generation }) { cp.publicationStatus = !eligible || current.state === null ? "blocked" : cp.state.publication.commentId !== null ? "published" : "unpublished"; cp.generation++; - return saveCheckpoint(transport, context, cp); + return saveCheckpoint(transport, context, cp, loaded.bodyDigest); } module.exports = { diff --git a/eng/regression-bisect/state.test.cjs b/eng/regression-bisect/state.test.cjs index 3c081b2c7d8..a3b66eb99f0 100644 --- a/eng/regression-bisect/state.test.cjs +++ b/eng/regression-bisect/state.test.cjs @@ -17,6 +17,7 @@ function harness() { actor: { id: 77, login: "github-actions[bot]", type: "Bot", appId: 88 }, workflowPath: ".github/workflows/regression-bisect.yml", workflowRef: "refs/heads/main", trustedWorkflowRevisions: [f.sha("workflow")], + checkpointAnchorSha: f.sha("workflow"), bootstrapDigest: model.fingerprintInput("unused placeholder"), keys: { fixture: Buffer.from("synthetic-test-key-not-a-production-secret") }, keyId: "fixture", owner: { runId: "12345", attempt: 1 }, now: "2026-09-18T00:00:00Z", issue: { number: 123, state: "open", created_at: "2026-09-01T00:00:00Z", labels: ["Regression"] }, @@ -40,7 +41,9 @@ function harness() { body: "unused placeholder", }; let tail = Promise.resolve(); + const freshness = f.statusTransport(context); const transport = { + request: freshness.request, getComment: async id => { assert.equal(id, 456); return clone(comment); }, updateComment: async (id, body) => { assert.equal(id, 456); comment.body = body; return clone(comment); }, getRun: async (id, attempt) => { @@ -51,9 +54,14 @@ function harness() { readEvidence: async () => Buffer.from("durable evidence"), }; return { - context, transport, runs, + context, transport, runs, statuses: freshness.statuses, get comment() { return comment; }, set comment(c) { comment = c; }, + checkpoint(cp) { + cp.previousDigest = f.digest(comment.body); + cp.generation++; + comment.body = serializeWorkflowRecord({ kind: "checkpoint", status: "fixture transition", payload: cp }, context); + }, // This stands for the controller's whole issue concurrency group, not comment CAS. lock(action) { const next = tail.then(action); @@ -61,7 +69,6 @@ function harness() { return next; }, async start() { - context.bootstrapDigest = model.fingerprintInput(comment.body); return initializeCheckpoint(transport, context, { allowanceId: "maintainer-approved-investigation-1", deadline: "2026-10-01T00:00:00Z", }); @@ -87,6 +94,7 @@ function completion(h, pending, outcome = "product-failure") { for (const cell of data.cells) { cell.attempts = cell.attempts.map(a => ({ ...a, execution: { ...a.execution, ...h.context.manifest.assertions[outcome === "pass" ? "pass" : "failure"], + status: outcome === "untestable" ? "timeout" : a.execution.status, } })); cell.outcome = model.classifyAttempts(cell.attempts, h.context.manifest); for (const artifact of Object.values(cell.artifacts)) artifact.buildInstanceId = pending.id; @@ -337,6 +345,107 @@ test("two attempts per item, cumulative reservation limit and absolute deadline await assert.rejects(reserve(h, generation, "late", request(f.sha("last"), null)), /deadline/); }); +test("restoring an authentic old body to the same comment cannot roll back evidence or budgets", async () => { + const h = harness(); await h.start(); + const initialBody = h.comment.body; + const first = await reserve(h); + await commitObservation(h.transport, h.context, completion(h, first)); + for (let i = 0; i < 5; i++) { + const cp = (await loadCheckpoint(h.transport, h.context)).checkpoint; + await reserve(h, cp.generation, `charged-${i}`, request(f.sha(`charged-${i}`), null)); + h.nextRun(); + } + const latestBody = h.comment.body; + const latest = (await loadCheckpoint(h.transport, h.context)).checkpoint; + assert.equal(latest.state.budget.consumed.reservedMinutes, 660); + assert.equal(Object.keys(latest.observations).length, 1); + await assert.rejects(reserve(h, latest.generation, "exhausted", request(f.sha("last"), null)), /budget/); + h.comment.body = initialBody; + await assert.rejects(loadCheckpoint(h.transport, h.context), /freshness|rollback/i); + await assert.rejects(reserve(h, 0, "rollback"), /freshness|rollback/i); + await assert.rejects(invalidateStaleEvidence(h.transport, h.context, { generation: 0 }), /freshness|rollback/i); + h.comment.body = "unused placeholder"; + await assert.rejects(h.start(), /freshness|initializ/i); + h.comment.body = latestBody; + assert.deepEqual((await loadCheckpoint(h.transport, h.context)).checkpoint, latest); +}); + +test("freshness receipts cannot be forged, omitted, truncated or replaced by a newer replay", async t => { + for (const [name, change] of [ + ["missing", h => { h.statuses.length = 0; }], + ["writer", h => { h.statuses[0].creator.id++; }], + ["signature", h => { h.statuses[0].target_url += "0"; }], + ["generation", h => { h.statuses[0].description = h.statuses[0].description.replace("checkpoint:0:", "checkpoint:1:"); }], + ["status", h => { h.statuses[0].state = "pending"; }], + ["wrong issue", h => { h.statuses[0].target_url = h.statuses[0].target_url.replace("/123#", "/124#"); }], + ["wrong comment", h => { h.statuses[0].context = "regression-bisect/123/789"; }], + ["wrong anchor", h => { h.context.checkpointAnchorSha = f.sha("another anchor"); }], + ["API failure", h => { h.transport.request = async () => { throw new Error("freshness unavailable"); }; }], + ["HTTP error", h => { h.transport.request = async () => ({ status: 403, data: [] }); }], + ["incomplete pagination", h => { h.transport.request = async () => ({ data: [], headers: { link: '; rel="next"' } }); }], + ["same-generation replacement", async h => { + const cp = (await loadCheckpoint(h.transport, h.context)).checkpoint; + h.comment.body = serializeWorkflowRecord({ kind: "checkpoint", status: "different authentic body", + payload: cp }, h.context); + }], + ]) { + await t.test(name, async () => { + const h = harness(); await h.start(); + await change(h); + await assert.rejects(loadCheckpoint(h.transport, h.context), /freshness/i); + }); + } + const h = harness(); await h.start(); + const initialBody = h.comment.body, oldReceipt = clone(h.statuses[0]); + await reserve(h); + for (let i = 0; i < 101; i++) h.statuses.push({ ...clone(oldReceipt), id: i + 10 }); + const receiptCount = h.statuses.length; + assert.equal((await loadCheckpoint(h.transport, h.context)).checkpoint.state.budget.consumed.workerUnits, 1); + assert.equal(h.statuses.length, receiptCount); + h.comment.body = initialBody; + await assert.rejects(loadCheckpoint(h.transport, h.context), /freshness|rollback/i); +}); + +test("freshness write cancellation recovers only the signed immediate successor without refund or execution", async t => { + for (const stage of ["initialization", "reservation", "observation", "receipt-acknowledgement"]) { + await t.test(stage, async () => { + const h = harness(); + let pending; + if (stage !== "initialization") await h.start(); + if (stage === "observation") pending = await reserve(h); + const send = h.transport.request; + h.transport.request = async (route, args) => { + if (route.startsWith("POST")) { + if (stage === "receipt-acknowledgement") await send(route, args); + throw new Error("cancelled before freshness acknowledgement"); + } + return send(route, args); + }; + await assert.rejects(stage === "initialization" ? h.start() : stage === "observation" + ? commitObservation(h.transport, h.context, completion(h, pending)) : reserve(h), /cancelled/); + h.transport.request = send; + h.nextRun(); + if (stage === "initialization") await h.start(); + const cp = (await loadCheckpoint(h.transport, h.context)).checkpoint; + assert.equal(cp.state.budget.consumed.workerUnits, stage === "initialization" ? 0 : 1); + assert.equal(Object.keys(cp.observations).length, stage === "observation" ? 1 : 0); + if (cp.pending) { + const retry = await reserve(h, cp.generation, "retry"); + assert.equal(retry.attempt, 2); + assert.equal((await loadCheckpoint(h.transport, h.context)).checkpoint.state.budget.consumed.workerUnits, 2); + } + }); + } + for (const gap of [false, true]) { + const h = harness(); const cp = await h.start(); + h.checkpoint(cp); + if (gap) cp.generation++; + else cp.previousDigest = f.digest("unrelated checkpoint"); + h.comment.body = serializeWorkflowRecord({ kind: "checkpoint", status: "invalid predecessor", payload: cp }, h.context); + await assert.rejects(loadCheckpoint(h.transport, h.context), /freshness|predecessor/i); + } +}); + test("input changes and label toggles invalidate evidence but never replenish an allowance", async () => { const h = harness(); await h.start(); const p = await reserve(h); @@ -382,9 +491,9 @@ test("withdrawn approval invalidates evidence and label removal retains publishe const published = await commitObservation(h.transport, h.context, completion(h, p)); published.state.publication = { digest: f.digest("result"), commentId: 789 }; published.publicationStatus = "published"; - h.comment.body = serializeWorkflowRecord({ kind: "checkpoint", status: "published", payload: published }, h.context); + h.checkpoint(published); h.context.issue.state = "closed"; - let cp = await invalidateStaleEvidence(h.transport, h.context, { generation: 2 }); + let cp = await invalidateStaleEvidence(h.transport, h.context, { generation: published.generation }); assert.deepEqual(cp.state.publication, published.state.publication); assert.equal(cp.publicationStatus, "blocked"); h.context.issue.state = "open"; @@ -414,12 +523,7 @@ test("observation size overflow leaves the charged reservation durable and block test("untestable observations retain ordered tested and skipped reasons and full evidence", async () => { const h = harness(); await h.start(); const p = await reserve(h); - const done = completion(h, p); - for (const cell of done.observation.cells) { - cell.attempts.forEach(a => { a.execution.status = "timeout"; }); - cell.outcome = model.classifyAttempts(cell.attempts, h.context.manifest); - } - done.observation.outcome = done.observation.cells[0].outcome; + const done = completion(h, p, "untestable"); const cp = await commitObservation(h.transport, h.context, done); assert.equal(cp.state.skips[0].reason, "timeout"); assert.equal(cp.state.skips[0].observationDigest, cp.state.testedRevisions[0].observationDigest); @@ -450,23 +554,39 @@ test("changed approval, reproduction, producer and orchestration invalidate depe } }); -test("moving main and stable start endpoint epochs while retaining exact historical source evidence", async () => { - const h = harness(); await h.start(); - const p = await reserve(h, 0, "good", request(f.sha("good"), "good")); - const done = completion(h, p, "pass"); - await commitObservation(h.transport, h.context, done); - const old = (await loadCheckpoint(h.transport, h.context)).checkpoint; - h.context.pinnedMainSha = f.sha("new main"); - let cp = await invalidateStaleEvidence(h.transport, h.context, { generation: old.generation }); - assert.notEqual(cp.state.epoch, old.state.epoch); - assert.equal(cp.state.endpointEvidence.good, model.fingerprintInput(done.observation)); - assert.equal(cp.observations[model.fingerprintInput(done.observation)].valid, true); - assert.equal(cp.state.testedRevisions.length, 1); - assert.equal(cp.state.interval, null); - h.context.stableSdk.releaseIndexDigest = f.digest("new release metadata"); - cp = await invalidateStaleEvidence(h.transport, h.context, { generation: cp.generation }); - assert.equal(cp.state.endpointEvidence.good, model.fingerprintInput(done.observation)); - assert.equal(cp.history.length, 2); +test("moving main or stable clears only that endpoint and the interval, retaining historical evidence", async t => { + for (const changed of ["main", "stable"]) { + await t.test(changed, async () => { + const h = harness(); let old = await h.start(); + for (const endpoint of ["stable", "main", "good", null]) { + const work = endpoint === "stable" + ? { kind: "released", target: { sdk: h.context.stableSdk }, endpoint, phase: "stable" } + : request(f.sha(endpoint ?? "skipped"), endpoint); + const p = await reserve(h, old.generation, endpoint ?? "skipped", work); + old = await commitObservation(h.transport, h.context, + completion(h, p, endpoint === "good" ? "pass" : endpoint === null ? "untestable" : "product-failure")); + } + old.state.interval = { goodSha: f.sha("good"), badSha: f.sha("main"), gitLog: ["verified ancestry"] }; + old.state.publication = { digest: f.digest("result"), commentId: 789 }; + old.publicationStatus = "published"; + h.checkpoint(old); + if (changed === "main") h.context.pinnedMainSha = f.sha("new main"); + else h.context.stableSdk.releaseIndexDigest = f.digest("new release metadata"); + const cp = await invalidateStaleEvidence(h.transport, h.context, { generation: old.generation }); + assert.notEqual(cp.state.epoch, old.state.epoch); + assert.deepEqual(cp.state.endpointEvidence, { ...old.state.endpointEvidence, [changed]: null }); + assert.equal(cp.state.interval, null); + assert.deepEqual(cp.state.publication, { digest: null, commentId: null }); + assert.equal(cp.publicationStatus, "unpublished"); + for (const field of ["observations", "records", "sequence"]) assert.deepEqual(cp[field], old[field]); + for (const field of ["testedRevisions", "skips", "budget"]) assert.deepEqual(cp.state[field], + field === "budget" ? { ...old.state.budget, stableSdk: h.context.stableSdk } : old.state[field]); + assert.equal(cp.history.length, 1); + assert.deepEqual(cp.history[0].endpointEvidence, old.state.endpointEvidence); + assert.deepEqual(cp.history[0].interval, old.state.interval); + assert.deepEqual(cp.history[0].publication, old.state.publication); + }); + } }); test("mandatory external evidence loss blocks reuse; optional expired logs are disclosed", async t => { @@ -516,6 +636,7 @@ test("repeated identity/evidence tables compact below 48 KiB without dropping ob test("serialized checkpoint fixture replays reservation, observation, cancellation and resume", async () => { const h = harness(); + await h.start(); h.comment.body = readFileSync(join(__dirname, "fixtures", "checkpoint.txt"), "utf8").trimEnd(); const initial = await loadCheckpoint(h.transport, h.context); assert.equal(initial.checkpoint.generation, 0); From 13048d6531af3ea2cb1188a3a984cb75a612e9e3 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 01:00:38 +0200 Subject: [PATCH 08/99] Recover interrupted checkpoint initialization before current input checks Authenticate and restore the original generation-zero receipt without rebuilding state from changed inputs. Preserve its allowance and deadline, and keep current eligibility and approval checks at load and reservation time. Cover both initialization interruption points across input and eligibility changes, reject forged/noninitial recovery, and document the recovery contract. Scoped validation: 421 Node tests, seven syntax checks, isolated checkpoint replay, and diff checks passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 14 +++- eng/regression-bisect/state.cjs | 24 +++++-- eng/regression-bisect/state.test.cjs | 96 ++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 10 deletions(-) diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index 39fef51efd5..65e4c573fb4 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -321,7 +321,8 @@ missed. `initializeCheckpoint(transport, context, { allowanceId, deadline })` is only for the initial maintainer-authorized allowance, under the issue lock. `context.bootstrapDigest` must come from the trusted creation receipt for the exact initial placeholder, not from hashing whichever comment is currently -present. Initialization refuses an existing envelope. There is deliberately no +present. Initialization refuses an existing envelope except its authenticated, +interrupted generation-zero checkpoint described below. There is deliberately no automatic reset/new-allowance operation; ordinary issue edits, label toggles, redispatches, and cancellations cannot replenish any counter. @@ -357,8 +358,15 @@ This preserves observations when cancellation interrupts the two writes; it does not grant a second execution permission. An older body, a changed body at the same generation, a generation gap, or a wrong predecessor fails closed. An interrupted initial write can only be completed by `initializeCheckpoint` -with the original bootstrap receipt and the identical initial allowance. -Missing receipts are never synthesized from arbitrary loaded state. +with the original bootstrap receipt and the identical initial allowance ID and +deadline. Recovery authenticates and validates the original generation-zero +checkpoint and appends its missing receipt without rewriting the comment or +reconstructing state from today's inputs. It preserves the original allowance +even when the discussion, approval, endpoints, or eligibility changed, or its +deadline expired. The controller then uses `loadCheckpoint` and +`invalidateStaleEvidence` to apply current context; recovering a receipt is not +execution or publication permission. Missing receipts are never synthesized +from arbitrary loaded state. ### Controller transaction diff --git a/eng/regression-bisect/state.cjs b/eng/regression-bisect/state.cjs index a44b24e8d0c..5bf199706b4 100644 --- a/eng/regression-bisect/state.cjs +++ b/eng/regression-bisect/state.cjs @@ -380,6 +380,23 @@ async function initializeCheckpoint(transport, context, { allowanceId, deadline existing.performed_via_github_app?.id === context.actor.appId && existing.issue_url === `https://api.github.com/repos/${context.repository}/issues/${context.issueNumber}`, "bootstrap author authentication failed"); + if (fingerprint(existing.body) !== context.bootstrapDigest || parseRecord(existing.body) !== null) { + const record = await authenticatedRecord(transport, context, existing); + requireState(record?.kind === "checkpoint", "interrupted initialization authentication failed"); + const cp = unpack(record.payload); + validateCheckpoint(cp); + requireState(cp.state.repository === context.repository && cp.state.issueNumber === context.issueNumber && + cp.state.budget.workflowRevision === record.binding.workflowRevision && + cp.generation === 0 && cp.previousDigest === context.bootstrapDigest && cp.sequence === 0 && + cp.history.length === 0 && cp.state.phase === "discovery" && cp.state.interval === null && + cp.state.budget.nextWorkItemId === "none" && cp.publicationStatus === "unpublished" && + equal(cp.state.publication, { digest: null, commentId: null }) && + cp.state.budget.allowanceId === allowanceId && cp.deadline === deadline, + "initialization requires the original interrupted checkpoint and allowance"); + // Recover the original bytes; current inputs gate execution when this checkpoint is loaded. + await writeFreshness(transport, context, 0, hashBytes(existing.body)); + return cp; + } requireState(timestamp(deadline) && Date.parse(deadline) > Date.parse(context.now), "invalid allowance deadline"); const budget = { schemaVersion: 1, policy: context.policy, policyRevision: context.policyRevision, workflowRevision: context.workflowRevision, @@ -401,13 +418,6 @@ async function initializeCheckpoint(transport, context, { allowanceId, deadline sequence: 0, pending: null, records: [], observations: {}, history: [], deadline, publicationStatus: "unpublished", }; requireState(currentState(cp, context).reasons.length === 0, "initialization blocked by current eligibility or approval"); - if (fingerprint(existing.body) !== context.bootstrapDigest || parseRecord(existing.body) !== null) { - const record = await authenticatedRecord(transport, context, existing); - requireState(record?.kind === "checkpoint" && equal(unpack(record.payload), cp), - "initialization requires the exact trusted bootstrap placeholder or interrupted initial checkpoint"); - await writeFreshness(transport, context, 0, hashBytes(existing.body)); - return cp; - } return saveCheckpoint(transport, context, structuredClone(cp)); } diff --git a/eng/regression-bisect/state.test.cjs b/eng/regression-bisect/state.test.cjs index a3b66eb99f0..fde13ce06af 100644 --- a/eng/regression-bisect/state.test.cjs +++ b/eng/regression-bisect/state.test.cjs @@ -123,6 +123,9 @@ test("checkpoint authenticates exact author, app, comment, issue and live run pr await t.test(name, async () => { const h = harness(); await h.start(); mutate(h); await assert.rejects(loadCheckpoint(h.transport, h.context), /authentic|provenance/i); + h.statuses.length = 0; + await assert.rejects(h.start(), /authentic|provenance/i); + assert.equal(h.statuses.length, 0); }); } const h = harness(); await h.start(); @@ -446,6 +449,99 @@ test("freshness write cancellation recovers only the signed immediate successor } }); +test("interrupted initialization recovers original state before checking changed current inputs", async t => { + const changes = { + main: h => { h.context.pinnedMainSha = f.sha("new main"); }, + stable: h => { h.context.stableSdk.releaseIndexDigest = f.digest("new release metadata"); }, + discussion: h => { h.context.issueFingerprint = f.digest("unapproved correction"); }, + reproduction: h => { + h.context.manifest.sources[0].sha256 = f.digest("corrected reproduction"); + f.seal(h.context.manifest); + h.context.authorization.manifestDigest = model.fingerprintInput(h.context.manifest); + h.context.authorization.inputDigest = h.context.manifest.approvedInputDigest; + }, + approval: h => { h.context.authorization = null; }, + manifest: h => { h.context.manifest = null; }, + policy: h => { h.context.policyRevision = f.sha("new approval"); h.context.authorization.approvalRevision = h.context.policyRevision; }, + workflow: h => { + h.context.workflowRevision = f.sha("new workflow"); + h.context.trustedWorkflowRevisions.push(h.context.workflowRevision); + h.runs.get(`${h.context.owner.runId}/1`).head_sha = h.context.workflowRevision; + }, + label: h => { h.context.issue.labels = []; }, + closed: h => { h.context.issue.state = "closed"; }, + deadline: h => { h.context.now = "2026-10-02T00:00:00Z"; }, + }; + for (const point of ["comment-acknowledgement", "receipt"]) { + for (const [name, change] of Object.entries(changes)) { + await t.test(`${point}: ${name}`, async () => { + const h = harness(); + const original = await harness().start(); + const { updateComment, request: send } = h.transport; + if (point === "comment-acknowledgement") { + h.transport.updateComment = async (...args) => { + await updateComment(...args); + throw new Error("initialization interrupted"); + }; + } else { + h.transport.request = async (route, args) => { + if (route.startsWith("POST")) throw new Error("initialization interrupted"); + return send(route, args); + }; + } + await assert.rejects(h.start(), /initialization interrupted/); + Object.assign(h.transport, { updateComment, request: send }); + const originalBody = h.comment.body; + assert.equal(h.statuses.length, 0); + h.nextRun(); + change(h); + assert.deepEqual(await h.lock(() => h.start()), original); + assert.equal(h.comment.body, originalBody); + assert.equal(h.statuses.length, 1); + const loaded = await loadCheckpoint(h.transport, h.context); + assert.deepEqual(loaded.checkpoint, original); + assert.deepEqual(loaded.owner, { runId: "12345", attempt: 1 }); + assert.equal(loaded.executionAllowed, false); + assert.equal(loaded.publicationAllowed, false); + await assert.rejects(reserve(h), /blocked/); + const cp = await invalidateStaleEvidence(h.transport, h.context, { generation: 0 }); + assert.deepEqual(cp.state.budget.consumed, original.state.budget.consumed); + assert.equal(cp.state.budget.allowanceId, original.state.budget.allowanceId); + assert.equal(cp.deadline, original.deadline); + assert.equal((await loadCheckpoint(h.transport, h.context)).executionAllowed, + ["main", "stable", "reproduction", "policy", "workflow"].includes(name)); + }); + } + } +}); + +test("initialization recovery rejects malformed, noninitial or changed-allowance records without writing", async t => { + const changes = { + schema: cp => { cp.schemaVersion++; }, + issue: cp => { cp.state.issueNumber++; }, + generation: cp => { cp.generation++; }, + bootstrap: cp => { cp.previousDigest = f.digest("different placeholder"); }, + history: cp => { cp.history.push({ reason: "not initial state" }); }, + phase: cp => { cp.state.phase = "stopped"; }, + allowance: cp => { cp.state.budget.allowanceId = "different allowance"; }, + deadline: cp => { cp.deadline = "2026-11-01T00:00:00Z"; }, + }; + for (const [name, change] of Object.entries(changes)) { + await t.test(name, async () => { + const h = harness(); + const cp = await h.start(); + change(cp); + h.comment.body = serializeWorkflowRecord({ kind: "checkpoint", status: "initialization", payload: cp }, h.context); + h.statuses.length = 0; + h.nextRun(); + const body = h.comment.body; + await assert.rejects(h.lock(() => h.start())); + assert.equal(h.comment.body, body); + assert.equal(h.statuses.length, 0); + }); + } +}); + test("input changes and label toggles invalidate evidence but never replenish an allowance", async () => { const h = harness(); await h.start(); const p = await reserve(h); From 04908faace1d5b55e54904c4b4ee6180a6614efc Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 01:26:45 +0200 Subject: [PATCH 09/99] Prevent cursor-loss starvation and incomplete regression intake Hold fallback page/offset selections for expanding schedule windows so nonconsecutive bounded-gap ordinals cannot alias discovery. Reject timeline cross-references without resolvable issue identities before freezing intake. Cover missing/redacted cross-references, missing/corrupt cursor recovery at schedule strides 1 and 60, and exact large-ordinal boundaries. All 429 scoped Node tests, seven syntax checks, isolated checkpoint replay, and diff checks passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 19 ++++++-- eng/regression-bisect/intake.cjs | 26 ++++++---- eng/regression-bisect/intake.test.cjs | 70 +++++++++++++++++++++------ 3 files changed, 87 insertions(+), 28 deletions(-) diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index 65e4c573fb4..64a6f54e38b 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -215,8 +215,10 @@ activity, and trusted checkpoints. Suppression requires an open/unmerged target active within 14 days and a timeline cross-reference or verified backlink; an arbitrary "already investigating" comment is not a permanent veto. The controller must inspect `snapshot.investigations.active` before dispatch. -The result records links, status, and the basis for suppression; unreadable -required targets block intake rather than silently disappearing. +The result records links, status, and the basis for suppression. Every timeline +cross-reference must identify a resolvable issue/PR target; missing/redacted +source identities and unreadable required targets block intake rather than +silently disappearing. To supply trusted checkpoints, adapt `loadCheckpoint()` into `readTrustedCheckpoints` records @@ -249,9 +251,16 @@ eligible creation dates so later bot labeling, including issues created exactly at the cutoff, is eventually discovered. Supply a monotonic schedule ordinal independently of the cursor cache (for -example, the scheduled UTC slot number from a fixed epoch). Lost or corrupt -cursors use expanding triangular page rounds and rotating three-item offsets; -repeated cache loss must not always return page one's first three issues. +example, the scheduled UTC slot number from a fixed epoch). Scheduled calls must +continue with bounded gaps between ordinals; consecutive slots are not required. +Lost or corrupt cursors use expanding rounds: round `r` visits pages `1..r+1`, +holding each page for `r+1` ordinal slots, at offset `3 * (r % 34)`. Round `r` +starts at `r * (r+1) * (2*r+1) / 6`. Once the dwell time exceeds the schedule gap, +every page/offset in subsequent rounds is visited, including all three-item +slots on dense pages. This prevents fixed cadences such as `2 + 60 * sweep` +from aliasing the rotation and permanently skipping eligible issues. +Fallback recovery deliberately repeats candidates and is slower than retaining +the cursor; sustained unbounded scheduling outages cannot guarantee progress. The cursor is only a discovery optimization. Its loss never initializes a new allowance, removes a checkpoint, or changes consumed budgets. Event/manual paths and discovery candidates still enter the same serialized issue diff --git a/eng/regression-bisect/intake.cjs b/eng/regression-bisect/intake.cjs index 013eec6a316..2948e095c04 100644 --- a/eng/regression-bisect/intake.cjs +++ b/eng/regression-bisect/intake.cjs @@ -215,8 +215,8 @@ async function findActiveInvestigations(options) { requireValue(targets.size <= limits.linkedTargets, "linked target limit exceeded"); }; for (const event of options.timeline) { - if (event.event === "cross-referenced" && event.source?.issue) { - const link = issueLink(event.source.issue.html_url); + if (event.event === "cross-referenced") { + const link = issueLink(event.source?.issue?.html_url); requireValue(link, "required linked timeline target is unreadable"); add(link, "timeline-cross-reference"); } @@ -303,11 +303,18 @@ async function findActiveInvestigations(options) { function restartCursor(ordinal) { requireValue(Number.isSafeInteger(ordinal) && ordinal >= 0, "scheduleOrdinal must be a nonnegative safe integer"); - // Triangular rounds revisit pages 1; 1,2; 1,2,3; ... . Each round rotates - // the three-item starting slot, so repeated cursor loss starves neither a - // later page nor the remaining eligible candidates on a densely eligible page. - const round = Math.floor((Math.sqrt(8 * ordinal + 1) - 1) / 2); - return { version: 1, page: ordinal - round * (round + 1) / 2 + 1, offset: (round % 34) * 3 }; + // Round r visits pages 1..r+1, holding each for r+1 schedule slots. + // Growing dwell times prevent bounded gaps between ordinals from aliasing + // the page/offset rotation. Integer boundaries avoid large-ordinal rounding. + const slot = BigInt(ordinal); + const start = r => r * (r + 1n) * (2n * r + 1n) / 6n; + let round = BigInt(Math.floor(Math.cbrt(3 * ordinal))); + while (start(round) > slot) round--; + while (start(round + 1n) <= slot) round++; + return { + version: 1, page: Number((slot - start(round)) / (round + 1n)) + 1, + offset: Number(round % 34n) * 3, + }; } /** @@ -317,8 +324,9 @@ function restartCursor(ordinal) { * its page/offset resumes before any overflow candidate, then wraps after the end. * The cursor is discovery optimization, never issue state or deduplication. * Missing/corrupt cursor restarts using scheduleOrdinal, a monotonic independent - * schedule slot (not a counter stored with the cursor). Ordinal defaults to zero - * for a one-off restart; scheduled callers MUST supply it to guarantee recovery. + * schedule slot with bounded gaps (not a counter stored with the cursor). + * Expanding page dwell times tolerate nonconsecutive slots. Ordinal defaults to + * zero for a one-off restart; scheduled callers MUST supply it for recovery. * A shortened page restarts its offset; failures return no candidates and do not * advance the supplied cursor. Candidate eligibility must be rechecked at intake. */ diff --git a/eng/regression-bisect/intake.test.cjs b/eng/regression-bisect/intake.test.cjs index da043c1c3d6..c0f1184eced 100644 --- a/eng/regression-bisect/intake.test.cjs +++ b/eng/regression-bisect/intake.test.cjs @@ -362,6 +362,29 @@ test("required linked reads and linked-target safety limits fail closed", async blocked(await snapshot(f, { limits: { linkedTargets: 1 } }), /limit/i); }); +for (const [name, source] of [ + ["missing source", undefined], + ["null source", null], + ["missing source issue", { type: "issue" }], + ["null source issue", { type: "issue", issue: null }], + ["missing target URL", { type: "issue", issue: { number: 321 } }], + ["invalid target URL", { type: "issue", issue: { html_url: "https://example.com/pull/321" } }], +]) { + test(`unresolved cross-reference blocks the inventory and freeze: ${name}`, async () => { + const f = fixture({ timelinePages: [ + [{ id: 1, event: "labeled" }], + [{ id: 2, event: "cross-referenced", ...(source === undefined ? {} : { source }) }], + ] }); + blocked(await snapshot(f), /required linked timeline target/i); + const inventory = await intake().findActiveInvestigations({ + ...f.options, issue: f.data.issue, comments: [], timeline: f.data.timelinePages[1], + }); + blocked(inventory, /required linked timeline target/i); + assert.equal(inventory.links, undefined); + assert.equal(inventory.active, undefined); + }); +} + test("timeline event and comment identifiers occupy separate namespaces", async () => { const f = fixture({ comments: [comment(5)], timeline: [ { ...comment(5), event: "commented" }, @@ -383,20 +406,39 @@ test("linked PRs require complete current merged status", async () => { blocked(await snapshot(f), /linked.*merged/i); }); -test("lost cursor traversal also reaches dense-page overflow candidates", async () => { - const f = fixture({ list: Array.from({ length: 205 }, (_, i) => issue(i + 1)) }); - const seen = new Set(); - for (let scheduleOrdinal = 0; scheduleOrdinal < 750; scheduleOrdinal++) { - const start = f.calls.length; - const found = await intake().discoverEligibleIssues({ ...f.options, issueNumber: undefined, scheduleOrdinal }); - assert.equal(found.status, "complete"); - assert.ok(found.issues.length <= 3); - assert.ok(f.calls.length - start <= 2); - found.issues.forEach(value => seen.add(value.number)); +for (const [stride, startOrdinal, sweeps] of [[1, 0, 18000], [60, 2, 6000]]) { + test(`lost cursor traversal reaches every dense-page candidate with schedule stride ${stride}`, async () => { + const f = fixture({ list: Array.from({ length: 205 }, (_, i) => issue(i + 1)) }); + const seen = new Set(); + for (let sweep = 0; sweep < sweeps && seen.size < f.data.list.length; sweep++) { + const start = f.calls.length; + const found = await intake().discoverEligibleIssues({ + ...f.options, issueNumber: undefined, scheduleOrdinal: startOrdinal + stride * sweep, + cursor: sweep % 2 ? { version: 1, page: -1, offset: 0 } : undefined, + }); + assert.equal(found.status, "complete"); + assert.ok(found.issues.length <= 3); + assert.ok(f.calls.length - start <= 2); + found.issues.forEach(value => seen.add(value.number)); + } + assert.ok(seen.has(1), "lost cursors must revisit issues created exactly at the cutoff"); + assert.ok(seen.has(4), "lost cursors must not starve the fourth eligible issue"); + assert.ok(seen.has(101), "lost cursors must not starve the second page"); + assert.equal(seen.size, 205); + }); +} + +test("lost cursor round boundaries remain exact for large safe schedule ordinals", async () => { + for (const round of [1n, 34n, 300000n]) { + const boundary = Number(round * (round + 1n) * (2n * round + 1n) / 6n); + for (const [scheduleOrdinal, expectedPage] of [[boundary - 1, Number(round)], [boundary, 1]]) { + const f = fixture(); + const result = await intake().discoverEligibleIssues({ ...f.options, issueNumber: undefined, scheduleOrdinal }); + assert.equal(result.status, "complete"); + assert.equal(f.calls[0].args.page, expectedPage); + assert.deepEqual(result.nextCursor, { version: 1, page: 1, offset: 0 }); + } } - assert.ok(seen.has(4), "lost cursors must not starve the fourth eligible issue"); - assert.ok(seen.has(101), "lost cursors must not starve the second page"); - assert.equal(seen.size, 205); }); test("trusted checkpoint ownership checks the live run, never a forged marker", async () => { @@ -499,7 +541,7 @@ test("discovery rotates over sparse pages and a lost cursor uses the independent assert.ok(seen.has(401), "losing the cursor each run must not starve later pages"); f.data.list[0].labels = ["Regression"]; let recovered = false; - for (let sweep = 120; sweep < 750; sweep++) { + for (let sweep = 120; sweep < 15000 && !recovered; sweep++) { const result = await intake().discoverEligibleIssues({ ...f.options, issueNumber: undefined, scheduleOrdinal: sweep }); recovered ||= result.issues.some(item => item.number === 1); } From 261a6b484880e1cfbcfc84aaa813e4b7221843b9 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 02:06:07 +0200 Subject: [PATCH 10/99] Acquire exact released and revision-specific regression toolchains Resolve stable SDKs from official metadata, verify isolated archive and F# runtime identities through the reused installer, and retain coupled source-toolchain transitions. Add bounded real-acquisition smoke and public-feed policy coverage. Validation: 304 required model/toolchain tests and 516 related Node tests passed; seven CJS syntax checks and staged diff check passed. Real Windows x64 smoke acquired and verified SDK 10.0.401 with runtime 10.0.12, then cleaned temporary storage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/policies/regression-bisect.json | 10 +- eng/regression-bisect/README.md | 128 ++++- eng/regression-bisect/fixtures/checkpoint.txt | 2 +- eng/regression-bisect/model.cjs | 29 +- eng/regression-bisect/toolchain-installer.cjs | 295 ++++++++++ .../toolchain-installer.test.cjs | 284 ++++++++++ eng/regression-bisect/toolchain-io.cjs | 233 ++++++++ eng/regression-bisect/toolchains.cjs | 390 +++++++++++++ eng/regression-bisect/toolchains.smoke.cjs | 54 ++ eng/regression-bisect/toolchains.test.cjs | 535 ++++++++++++++++++ 10 files changed, 1949 insertions(+), 11 deletions(-) create mode 100644 eng/regression-bisect/toolchain-installer.cjs create mode 100644 eng/regression-bisect/toolchain-installer.test.cjs create mode 100644 eng/regression-bisect/toolchain-io.cjs create mode 100644 eng/regression-bisect/toolchains.cjs create mode 100644 eng/regression-bisect/toolchains.smoke.cjs create mode 100644 eng/regression-bisect/toolchains.test.cjs diff --git a/.github/policies/regression-bisect.json b/.github/policies/regression-bisect.json index 7ad53c8d6c2..95f3d460f4d 100644 --- a/.github/policies/regression-bisect.json +++ b/.github/policies/regression-bisect.json @@ -35,5 +35,13 @@ "discoveryPages": 2, "discoveryPageSize": 100 }, - "evidenceBytesPerStream": 16384 + "evidenceBytesPerStream": 16384, + "toolchains": { + "publicFeeds": [ + "https://builds.dotnet.microsoft.com/dotnet", + "https://ci.dot.net/public", + "https://dotnetcli.blob.core.windows.net/dotnet", + "https://download.visualstudio.microsoft.com/download/pr" + ] + } } diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index 64a6f54e38b..5a6a27218d0 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -1,16 +1,18 @@ # Regression investigation contracts This directory defines version 1 of the deterministic input, evidence, state, -and admission contracts, complete issue intake, and authenticated issue-comment -checkpoints. It does not activate a workflow, execute reproductions, or spawn -subprocesses. GitHub access is injected by the trusted controller; tests use +and admission contracts, complete issue intake, authenticated issue-comment +checkpoints, and isolated exact toolchain acquisition. Acquisition runs the +official installer and fixed product-identity probes, not reproductions, source +builds, or bisection. It does not activate a workflow. +GitHub access is injected by the trusted controller; tests use synthetic records and an in-memory transport, never live issue mutations. Records under `fixtures` are not reproduction approvals. Run the dependency-free suite from the repository root: ```powershell -node --test eng\regression-bisect\model.test.cjs eng\regression-bisect\intake.test.cjs eng\regression-bisect\state.test.cjs +node --test eng\regression-bisect\model.test.cjs eng\regression-bisect\intake.test.cjs eng\regression-bisect\state.test.cjs eng\regression-bisect\toolchains.test.cjs eng\regression-bisect\toolchain-installer.test.cjs ``` ## Trust boundary @@ -49,6 +51,8 @@ path-qualified error. Unknown or missing fields, unsupported versions, and malformed identities are rejected. `classifyAttempts` instead maps incomplete attempt evidence to an explicit `untestable` outcome; `admitBisection` returns `{ decision, reason }`, with `decision` exactly `admit`, `stop`, or `blocked`. +An admitted interval with different verified source build toolchains also +returns `toolchainTransitions` for coupled compiler/toolchain attribution. `fixtures/contracts.cjs` provides executable examples of every record shape. @@ -122,7 +126,16 @@ can be `null` only in untestable evidence with the corresponding checks failed. The observation binds the complete manifest, approved input, discussion, matrix, producer, and package fingerprints. Admission also compares runner environments -across all endpoints and SDK/runtime identities across the two source endpoints. +across all endpoints and **reproduction** SDK/runtime identities across the two +source endpoints. A source cell can additionally carry `buildToolchain`: +`sourceSha`, `globalJsonDigest`, `evidenceDigest`, `requestedSdk`, +`requestedTools`, `requestedCompilerRuntime`, verified `sdk` and +`compilerRuntime` product identities, and `verified: true`, `supported: true`. +Requested/resolved versions and source SHA must agree. Both source endpoints +must supply this field or both omit it; legacy observations retain the original +SDK/runtime equality checks. Different valid build toolchains are not evidence +that the compiler source change alone caused the regression: admission retains +both records in each coupled `toolchainTransitions` entry. Cached source observations are reusable only with these same inputs and trusted provenance. Different source revisions naturally have different artifact hashes. @@ -169,7 +182,110 @@ Only the orchestration engine may eventually spawn `git bisect`. It must call this gate immediately before the first operation and again after any evidence-invalidating change, using the current authenticated context and charged reservation. This contract suite does not claim to prove an engine's -subprocess behavior; there is no engine or subprocess execution here. +subprocess behavior; no bisection engine is implemented here. + +## Exact toolchain acquisition + +`toolchains.cjs` exports four operations. All options are trusted-controller +inputs, never instructions extracted from an issue. Each operation requires the +validated maintainer `policy`. Resolution and acquisition also require +`platform: { os, rid, architecture }`, either directly or in the frozen toolchain. +Supported architectures are `x64` and `arm64`; RIDs are `win-*`, `linux-*`, +`linux-musl-*`, or `osx-*`, matching that OS and architecture. + +| Operation | Result | +| --- | --- | +| `resolveLatestStableSdk({ policy, platform, signal? }, dependencies?)` | `{ status: "resolved", toolchain, attempts }`; freezes the run clock and raw official index/channel metadata bytes and SHA-256 hashes. Inspects both `sdk` and `sdks`, orders stable versions numerically across LTS/STS channels, excludes future releases, and selects exactly one matching ZIP/tar archive. | +| `readRevisionToolchain({ policy, platform, sourceDirectory, sourceSha, runtimeConfigs? }, dependencies?)` | Preserves raw `global.json`, both requested SDK declarations, original SDK-selection settings, optional historical build/TFM files, and exact runtimeconfig requirements. Each config is `{ path, role: "compiler" \| "fsi" \| "sample" }`. No source file is rewritten. | +| `acquireToolchain({ policy, toolchain, storageDirectory, helperRoot?, signal? }, dependencies?)` | `{ status: "acquired", identity, evidencePath, launch, attempts }`; uses a unique child of existing storage, persists complete identity/metadata evidence, and retains verified archives. `helperRoot` is a repository root, defaulting to the historical source for source acquisition and this trusted checkout for released acquisition. | +| `verifyToolchainIdentity({ policy, toolchain, identity })` | `{ status: "verified" }` only for matching requested/resolved SDK, host/SDK paths, RID/architecture, archive hashes, bundled F# identities, and all exact runtime requirements. The acquisition request binds `toolchain.installationRoot`; evidence cannot nominate a different root. This validates captured evidence, not the authenticity of arbitrary JSON. | + +Failures return `{ status: "untestable", outcome, error: { stage, code, message }, +attempts }`, not reproduction pass/failure. Missing historical metadata, +unavailable/checksum-less archives, unsupported installer flags or declarations, +SDK/tool declaration disagreement, changed source files, and unfulfilled runtime +requirements fail closed. Historical previews are allowed, but never selected +as released stable. Public preview fallback uses an official archive SHA-512 +sidecar; missing checksum provenance cannot become success through a host SDK. +Exact `global.json.tools.runtimes` arrays (`dotnet`, `aspnetcore`, +`windowsdesktop`, optionally suffixed with the matching architecture) become +separate build-runtime requirements. Unevaluated MSBuild version expressions +and foreign-architecture requirements are explicitly untestable, not guessed. + +The shared boundaries are injectable: `fs` (`node:fs/promises`), `clock` +(`() => Date`), `http(url, { destination?, method? })`, `run(executable, args, +{ cwd, env, timeoutMs })`, `installer`, and `capture`. HTTP returns status, +final URL, redirect hops and raw bytes, or streams a large download to +`destination`. Every production redirect is checked against the policy's +credential-free public-feed allowlist; downloads, processes and smoke runs +are bounded. Process execution receives a newly constructed environment, not +the caller's secrets, private feeds, credential providers or user profile. + +`toolchain-installer.cjs` calls the existing Arcade `GetDotNetInstallScript` +helper with a freshly verified official-script cache. It verifies actual +script parameter support, then invokes the official installer with exact +version/architecture and archive-retention flags. A loopback-only, bounded +feed serves the already downloaded and SHA-512-verified archive to that +installer. This transport boundary records the **actual public download's** +redirects/digests without reimplementing extraction or trusting a separate HEAD +request. The official retained archive is hashed again. Supplemental runtime +installation uses the official skip-non-versioned-files flag to preserve the +SDK host. Arcade-owned files are neither edited nor copied. + +`launch` contains the explicit executable, historical build working directory, +isolated environment, and a separate sample directory with exact SDK selection +and roll-forward disabled. Source search paths that could select a different +installation are rejected. CLI home and all NuGet caches are unique per +acquisition. Source `global.json` remains unchanged; no TFM is overridden. +Callers must retain these launch settings and choose reviewed public NuGet +sources explicitly when they eventually execute an approved reproduction. + +Identity captures `--info`, `--list-sdks`, `--list-runtimes`, resolved SDK base +path, real host/file paths and hashes, actual process architecture, F# compiler/ +FSI/Core assembly and file versions, dependency-manifest identity, and bundled +Core nuspec/package identity when present. A fixed FSI probe runs only after +archive verification. SDK-bundled runtime, supplemental sample requirements, +and local compiler execution runtimes are separate records. Supplied local +compiler/FSI runtimeconfigs are verified using the corresponding built DLL's +`--help` and host trace with the exact runtime; sample code is never executed. +Only acquisition with both local compiler and FSI execution evidence emits the +`buildToolchain` admission record. An SDK acquired before a source build does +not attest the identity of a compiler that has not been built yet. + +### Bounded opt-in acquisition smoke + +```powershell +node eng\regression-bisect\toolchains.smoke.cjs --run --storage "$env:TEMP" +``` + +The smoke resolves current official metadata, acquires one stable SDK, verifies +the real archive and running identities, prints a JSON result, and removes only +its uniquely created child directory. It has a ten-minute cancellation deadline. +Exit 0 means actual acquisition/identity success; exit 2 records the exact +block, never mocked download success. This is not a source build or live issue run. + +On 2026-09-19 at 00:02 UTC, that command completed successfully on Windows x64: +requested/resolved SDK `10.0.401`, RID `win-x64`, bundled/executing runtime +`10.0.12`, compiler and FSI assembly `15.2.401.0`, Core assembly `10.1.0.0`, +and bundled package `FSharp.Core/10.1.401`. The archive's final public URL was +`https://builds.dotnet.microsoft.com/dotnet/Sdk/10.0.401/dotnet-sdk-10.0.401-win-x64.zip` +(no redirects). Official and downloaded SHA-512 were both +`24b670ad3d923bfcf47df6c3b034152398b42f6dbc388e10d783aee1cfb5e5817d399fc0ae2a12cfa822a55e61d34830ccb15c50ef6efee437ab874bb7c79430`; +archive SHA-256 was +`c1b96dea223e1ab2af8a51187e0cc96338d14c68834be1656136be18953cea09`. +The official PowerShell install script SHA-256 was +`e8b873e18a81e5c4cd8ab69d84dac8fead291d50b3c44633cd7fddad709a13d6`. +Temporary acquisition storage was cleaned. These are recorded smoke results, +not production version constants. + +Validation also completed `node --check` for all seven changed/new `.cjs` +files and the required +`node --test eng\regression-bisect\model.test.cjs eng\regression-bisect\toolchains.test.cjs` +command (304 passed). The broader suite above passed all 516 tests, including +intake, authenticated state replay and installer-boundary coverage. The signed checkpoint fixture +was regenerated through its existing harness for the new pinned public-feed +policy; an old-policy checkpoint is not silently accepted as a new-policy one. +Root compiler Release builds/tests are not this Node adapter's acceptance gate. ## Complete intake and bounded discovery diff --git a/eng/regression-bisect/fixtures/checkpoint.txt b/eng/regression-bisect/fixtures/checkpoint.txt index 12ead961515..432d66f8863 100644 --- a/eng/regression-bisect/fixtures/checkpoint.txt +++ b/eng/regression-bisect/fixtures/checkpoint.txt @@ -1,5 +1,5 @@ Regression investigation: discovery (generation 0). ```json -{"schemaVersion":1,"kind":"checkpoint","status":"Regression investigation: discovery (generation 0).","binding":{"repository":"dotnet/fsharp","issueNumber":123,"commentId":456,"workflowPath":".github/workflows/regression-bisect.yml","workflowRef":"refs/heads/main","workflowRevision":"899531e0f885501b5b8520a9bd95bc83345eaa5b","runId":"12345","runAttempt":1,"keyId":"fixture"},"payload":{"root":["@",0],"table":[{"schemaVersion":1,"generation":0,"previousDigest":"657b8c3151a6f2496623f2e58482895222b1808b30eee9cc032e1431a85a9347","state":["@",1],"inputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a","reproductionDigest":"91815101645c42988cb8f780311ed49cd7c179071d76d17298a1c6810a965138","policyDigest":"deb550568d7b07a6d49bcbc92690ee602abc201b76329631f0dee9387b2ea09f","sequence":0,"pending":null,"records":["@",28],"observations":["@",32],"history":["@",28],"deadline":"2026-10-01T00:00:00Z","publicationStatus":"unpublished"},{"schemaVersion":1,"repository":"dotnet/fsharp","issueNumber":123,"issueFingerprint":"ba79e2bbb07eaf308944401cb8a62139d3eed051e25c88279ce781531fef37f5","approvalFingerprint":"16503092fdfbb9143595739dc023ca6e201f80203f63950dd884370b09878dcb","manifest":["@",2],"budget":["@",18],"pinnedMainSha":"b28b7af69320201d1cf206ebf28373980add1451","epoch":"09eea1ee5aea50efd7ac23639a5050c834e1824d056cbcc4b054532e8c9a02a3","phase":"discovery","endpointEvidence":["@",30],"testedRevisions":["@",28],"skips":["@",28],"interval":null,"publication":["@",31]},{"schemaVersion":1,"repository":"dotnet/fsharp","issueNumber":123,"discussionFingerprint":"ba79e2bbb07eaf308944401cb8a62139d3eed051e25c88279ce781531fef37f5","kind":"fsi","sources":["@",3],"assertions":["@",5],"matrix":["@",8],"timeoutSeconds":300,"producer":["@",11],"packages":["@",12],"searchHints":["@",14],"approvedInputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a"},[["@",4]],{"path":"fixtures/repro.fsx","sha256":"e1c5329a6b6f65fb0733f62c334e0bf4bb91cfa449f5eb213dc67f0d40b440af"},{"pass":["@",6],"failure":["@",7]},{"exitCode":0,"stdout":"ok\n","stderr":""},{"exitCode":1,"stdout":"","stderr":"expected regression\n"},[["@",9]],{"id":"linux-x64","os":"linux","architecture":"x64","project":null,"tfm":"net10.0","configuration":"Release","languageVersion":"10.0","command":["@",10]},["fsi","fixtures/repro.fsx"],{"name":"repro","version":"1.0.0","sha256":"d1990f97a1f4fe7b8e86da8c03c9dd3f7d536c14aa4a6f9eedab07f34cd961b0"},[["@",13]],{"id":"Example","version":"1.0.0","sha256":"bc4a71180870f7945155fbb02f4b0a2e3faa2a62d6d31b7039013055ed19869a"},{"sdkVersions":["@",15],"sourceShas":["@",16],"urls":["@",17]},["9.0.100"],["fc19318dd13128ce14344d066510a982269c241b"],["https://github.com/dotnet/fsharp"],{"schemaVersion":1,"policy":["@",19],"policyRevision":"9f00fad98badc1330e637a0a015d704113498420","workflowRevision":"899531e0f885501b5b8520a9bd95bc83345eaa5b","allowanceId":"maintainer-approved-investigation-1","authorization":["@",24],"stableSdk":["@",25],"authenticatedObservationDigests":["@",28],"consumed":["@",29],"attempts":["@",28],"nextWorkItemId":"none","pendingWork":null},{"schemaVersion":1,"repository":"dotnet/fsharp","selection":["@",20],"authorization":["@",21],"accounting":["@",22],"limits":["@",23],"evidenceBytesPerStream":16384},{"state":"open","label":"Regression","createdAtOrAfter":"2026-09-01T00:00:00Z"},{"mode":"reviewed-manifests-only","policySource":"pinned-trusted-default-branch","approvalDirectory":".github/regression-bisect/approved","assets":"reviewed-repository-paths-and-sha256"},{"scope":"issue-allowance","reserveBeforeExecution":true,"retriesCharged":true,"cancellationRefunds":false,"inputEditsResetAllowance":false,"labelChangesResetAllowance":false,"dispatchResetsAllowance":false,"newAllowanceRequires":"maintainer-authorization"},{"sourceBuilds":24,"workerUnits":40,"reservedMinutes":720,"attemptsPerItem":2,"assertionRepetitions":2,"buildTimeoutMinutes":90,"reproductionTimeoutSeconds":300,"workerTimeoutMinutes":110,"discoveryIssues":3,"discoveryPages":2,"discoveryPageSize":100},{"repository":"dotnet/fsharp","issueNumber":123,"approvalPath":".github/regression-bisect/approved/123.json","approvalRevision":"9f00fad98badc1330e637a0a015d704113498420","manifestDigest":"16503092fdfbb9143595739dc023ca6e201f80203f63950dd884370b09878dcb","inputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a"},{"version":"10.0.100","origin":"official-release-index","latest":true,"verified":true,"releaseIndexUrl":"https://builds.dotnet.microsoft.com/dotnet/release-metadata/releases-index.json","releaseIndexDigest":"2a21ef65c47867ae1b2056a246bf7c001b12ade326bd0e1fcdfb81af140e65b7","archives":["@",26]},[["@",27]],{"os":"linux","architecture":"x64","sha256":"9d9e20df50a502c04ee629c1282495a65a279635168510572f846de604da046e"},[],{"sourceBuilds":0,"workerUnits":0,"reservedMinutes":0},{"stable":null,"main":null,"good":null},{"digest":null,"commentId":null},{}]},"signature":"d9f7ba4f55dafa5ae32fcb37b5a9be6e85dbbfe424b7b1b48b68385b166e533b"} +{"schemaVersion":1,"kind":"checkpoint","status":"Regression investigation: discovery (generation 0).","binding":{"repository":"dotnet/fsharp","issueNumber":123,"commentId":456,"workflowPath":".github/workflows/regression-bisect.yml","workflowRef":"refs/heads/main","workflowRevision":"899531e0f885501b5b8520a9bd95bc83345eaa5b","runId":"12345","runAttempt":1,"keyId":"fixture"},"payload":{"root":["@",0],"table":[{"schemaVersion":1,"generation":0,"previousDigest":"657b8c3151a6f2496623f2e58482895222b1808b30eee9cc032e1431a85a9347","state":["@",1],"inputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a","reproductionDigest":"91815101645c42988cb8f780311ed49cd7c179071d76d17298a1c6810a965138","policyDigest":"328aa42989c22950aea4d267acb68a4849932bbdb8ea4d118604f083e7d20738","sequence":0,"pending":null,"records":["@",30],"observations":["@",34],"history":["@",30],"deadline":"2026-10-01T00:00:00Z","publicationStatus":"unpublished"},{"schemaVersion":1,"repository":"dotnet/fsharp","issueNumber":123,"issueFingerprint":"ba79e2bbb07eaf308944401cb8a62139d3eed051e25c88279ce781531fef37f5","approvalFingerprint":"16503092fdfbb9143595739dc023ca6e201f80203f63950dd884370b09878dcb","manifest":["@",2],"budget":["@",18],"pinnedMainSha":"b28b7af69320201d1cf206ebf28373980add1451","epoch":"09eea1ee5aea50efd7ac23639a5050c834e1824d056cbcc4b054532e8c9a02a3","phase":"discovery","endpointEvidence":["@",32],"testedRevisions":["@",30],"skips":["@",30],"interval":null,"publication":["@",33]},{"schemaVersion":1,"repository":"dotnet/fsharp","issueNumber":123,"discussionFingerprint":"ba79e2bbb07eaf308944401cb8a62139d3eed051e25c88279ce781531fef37f5","kind":"fsi","sources":["@",3],"assertions":["@",5],"matrix":["@",8],"timeoutSeconds":300,"producer":["@",11],"packages":["@",12],"searchHints":["@",14],"approvedInputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a"},[["@",4]],{"path":"fixtures/repro.fsx","sha256":"e1c5329a6b6f65fb0733f62c334e0bf4bb91cfa449f5eb213dc67f0d40b440af"},{"pass":["@",6],"failure":["@",7]},{"exitCode":0,"stdout":"ok\n","stderr":""},{"exitCode":1,"stdout":"","stderr":"expected regression\n"},[["@",9]],{"id":"linux-x64","os":"linux","architecture":"x64","project":null,"tfm":"net10.0","configuration":"Release","languageVersion":"10.0","command":["@",10]},["fsi","fixtures/repro.fsx"],{"name":"repro","version":"1.0.0","sha256":"d1990f97a1f4fe7b8e86da8c03c9dd3f7d536c14aa4a6f9eedab07f34cd961b0"},[["@",13]],{"id":"Example","version":"1.0.0","sha256":"bc4a71180870f7945155fbb02f4b0a2e3faa2a62d6d31b7039013055ed19869a"},{"sdkVersions":["@",15],"sourceShas":["@",16],"urls":["@",17]},["9.0.100"],["fc19318dd13128ce14344d066510a982269c241b"],["https://github.com/dotnet/fsharp"],{"schemaVersion":1,"policy":["@",19],"policyRevision":"9f00fad98badc1330e637a0a015d704113498420","workflowRevision":"899531e0f885501b5b8520a9bd95bc83345eaa5b","allowanceId":"maintainer-approved-investigation-1","authorization":["@",26],"stableSdk":["@",27],"authenticatedObservationDigests":["@",30],"consumed":["@",31],"attempts":["@",30],"nextWorkItemId":"none","pendingWork":null},{"schemaVersion":1,"repository":"dotnet/fsharp","selection":["@",20],"authorization":["@",21],"accounting":["@",22],"limits":["@",23],"evidenceBytesPerStream":16384,"toolchains":["@",24]},{"state":"open","label":"Regression","createdAtOrAfter":"2026-09-01T00:00:00Z"},{"mode":"reviewed-manifests-only","policySource":"pinned-trusted-default-branch","approvalDirectory":".github/regression-bisect/approved","assets":"reviewed-repository-paths-and-sha256"},{"scope":"issue-allowance","reserveBeforeExecution":true,"retriesCharged":true,"cancellationRefunds":false,"inputEditsResetAllowance":false,"labelChangesResetAllowance":false,"dispatchResetsAllowance":false,"newAllowanceRequires":"maintainer-authorization"},{"sourceBuilds":24,"workerUnits":40,"reservedMinutes":720,"attemptsPerItem":2,"assertionRepetitions":2,"buildTimeoutMinutes":90,"reproductionTimeoutSeconds":300,"workerTimeoutMinutes":110,"discoveryIssues":3,"discoveryPages":2,"discoveryPageSize":100},{"publicFeeds":["@",25]},["https://builds.dotnet.microsoft.com/dotnet","https://ci.dot.net/public","https://dotnetcli.blob.core.windows.net/dotnet","https://download.visualstudio.microsoft.com/download/pr"],{"repository":"dotnet/fsharp","issueNumber":123,"approvalPath":".github/regression-bisect/approved/123.json","approvalRevision":"9f00fad98badc1330e637a0a015d704113498420","manifestDigest":"16503092fdfbb9143595739dc023ca6e201f80203f63950dd884370b09878dcb","inputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a"},{"version":"10.0.100","origin":"official-release-index","latest":true,"verified":true,"releaseIndexUrl":"https://builds.dotnet.microsoft.com/dotnet/release-metadata/releases-index.json","releaseIndexDigest":"2a21ef65c47867ae1b2056a246bf7c001b12ade326bd0e1fcdfb81af140e65b7","archives":["@",28]},[["@",29]],{"os":"linux","architecture":"x64","sha256":"9d9e20df50a502c04ee629c1282495a65a279635168510572f846de604da046e"},[],{"sourceBuilds":0,"workerUnits":0,"reservedMinutes":0},{"stable":null,"main":null,"good":null},{"digest":null,"commentId":null},{}]},"signature":"f271b6c7deff0f7c41a118baf7a71f0bb43aa32bf35b7347b01c3710f88150a2"} ``` diff --git a/eng/regression-bisect/model.cjs b/eng/regression-bisect/model.cjs index f94c4614b25..dc53bb5d7c2 100644 --- a/eng/regression-bisect/model.cjs +++ b/eng/regression-bisect/model.cjs @@ -229,13 +229,20 @@ const artifact = shape({ sha256: digest, origin: oneOf(["local-build", "released-sdk"]), sourceSha: nullable(sourceSha), buildInstanceId: text, }); -const observationCell = shape({ +const buildToolchain = shape({ + sourceSha, globalJsonDigest: digest, evidenceDigest: digest, + requestedSdk: pinnedVersion, requestedTools: pinnedVersion, requestedCompilerRuntime: pinnedVersion, + sdk: productIdentity, compilerRuntime: productIdentity, + verified: literal(true), supported: literal(true), +}); +const observationCell = (v, p) => object(v, { id: text, command: list(text, 2, 2), environment: shape({ os, architecture, imageDigest: digest }), sdk: nullable(productIdentity), runtime: nullable(productIdentity), artifacts: shape({ compiler: nullable(artifact), tasks: nullable(artifact), targets: nullable(artifact), core: nullable(artifact) }), producer: identity, attempts: list(attemptSchema, 1, trustedPolicy.limits.assertionRepetitions), outcome: validateOutcome, -}); + ...(v && Object.hasOwn(v, "buildToolchain") ? { buildToolchain } : {}), +}, p); function validateObservation(observation, manifest, options = { finalConfirmation: null }) { validateManifest(manifest); @@ -265,6 +272,13 @@ function validateObservation(observation, manifest, options = { finalConfirmatio check(expected && equal(cell.command, expected.command) && cell.environment.os === expected.os && cell.environment.architecture === expected.architecture, "observation.cells", "matrix or command mismatch"); check(equal(cell.producer, manifest.producer), "observation.cells.producer", "producer mismatch"); + if (cell.buildToolchain) { + const build = cell.buildToolchain; + check(observation.kind === "source" && build.sourceSha === observation.target.sourceSha && + build.requestedSdk === build.sdk.version && build.requestedTools === build.sdk.version && + build.requestedCompilerRuntime === build.compilerRuntime.version, + "observation.cells.buildToolchain", "wrong source declaration or resolved build SDK"); + } for (const [name, missing] of Object.entries({ sdk: cell.sdk === null, runtime: cell.runtime === null, identity: Object.values(cell.artifacts).some(a => a === null), @@ -369,11 +383,17 @@ function admitBisection(input) { check(main.target.sourceSha === ancestry.pinnedMainSha && main.target.sourceSha === ancestry.badSha && good.target.sourceSha === ancestry.goodSha && ancestry.goodSha !== ancestry.badSha && ancestry.isAncestor, "admission", "good must be a strict ancestor of pinned main"); + const toolchainTransitions = []; for (const cell of main.cells) { const older = good.cells.find(c => c.id === cell.id); const released = stable.cells.find(c => c.id === cell.id); check(equal(cell.environment, older.environment) && equal(cell.environment, released.environment) && equal(cell.sdk, older.sdk) && equal(cell.runtime, older.runtime), "admission", "incomparable environment or source toolchain"); + check(!!cell.buildToolchain === !!older.buildToolchain, "admission", "incomplete revision-specific toolchain evidence"); + if (cell.buildToolchain && (!equal(cell.buildToolchain.sdk, older.buildToolchain.sdk) || + !equal(cell.buildToolchain.compilerRuntime, older.buildToolchain.compilerRuntime))) { + toolchainTransitions.push({ cellId: cell.id, good: older.buildToolchain, main: cell.buildToolchain }); + } } if ([stable, main, good].some(o => o.outcome.outcome === "untestable")) { return { decision: "blocked", reason: "untestable endpoint evidence" }; @@ -387,7 +407,10 @@ function admitBisection(input) { budget.consumed.reservedMinutes + limits.workerTimeoutMinutes <= limits.reservedMinutes && (budget.attempts.find(a => a.workItemId === budget.nextWorkItemId)?.count ?? 0) < limits.attemptsPerItem, "admission", "insufficient remaining budget"); - return { decision: "admit", reason: "verified stable failure, pinned-main failure, and comparable ancestor pass" }; + return { + decision: "admit", reason: "verified stable failure, pinned-main failure, and comparable ancestor pass", + ...(toolchainTransitions.length ? { toolchainTransitions } : {}), + }; } catch (error) { if (!(error instanceof ContractError)) throw error; return { decision: "blocked", reason: error.message }; diff --git a/eng/regression-bisect/toolchain-installer.cjs b/eng/regression-bisect/toolchain-installer.cjs new file mode 100644 index 00000000000..4645562a733 --- /dev/null +++ b/eng/regression-bisect/toolchain-installer.cjs @@ -0,0 +1,295 @@ +'use strict'; + +const crypto = require('node:crypto'); +const http = require('node:http'); +const path = require('node:path'); +const { pipeline } = require('node:stream/promises'); + +const marker = 'INSTALLER_HELPER_RESULT='; +const helperPS = ` +$ErrorActionPreference = 'Stop' +$prepareMachine = $false +$ci = $false +$restore = $false +$runtimeSourceFeed = '' +$runtimeSourceFeedKey = '' +$useInstalledDotNetCli = $false +$disableConfigureToolsetImport = $true +$dotnetInstallScriptVersion = 'v1' +. $env:RB_HELPER_PATH +$scriptPath = GetDotNetInstallScript $env:RB_INSTALLER_CACHE +$tokens = $null +$parseErrors = $null +$ast = [System.Management.Automation.Language.Parser]::ParseFile($scriptPath, [ref]$tokens, [ref]$parseErrors) +if ($parseErrors.Count -ne 0 -or $null -eq $ast.ParamBlock) { throw 'Unsupported installer parameter block' } +$names = @($ast.ParamBlock.Parameters | ForEach-Object { $_.Name.VariablePath.UserPath }) +Write-Output ('INSTALLER_HELPER_RESULT=' + (@{ path = [string]$scriptPath; parameters = $names } | ConvertTo-Json -Compress)) +`; +const helperSH = ` +set -euo pipefail +prepare_machine=false +ci=false +restore=false +runtime_source_feed='' +runtime_source_feed_key='' +use_installed_dotnet_cli=false +disable_configure_toolset_import=true +dotnetInstallScriptVersion=v1 +. "$RB_HELPER_PATH" +GetDotNetInstallScript "$RB_INSTALLER_CACHE" +printf 'INSTALLER_HELPER_RESULT=%s\\n' "$_GetDotNetInstallScript" +`; + +const hash = (bytes, algorithm) => crypto.createHash(algorithm).update(bytes).digest('hex'); + +async function hashFile(fs, filename) { + const sha256 = crypto.createHash('sha256'); + const sha512 = crypto.createHash('sha512'); + const file = await fs.open(filename, 'r'); + try { + for await (const chunk of file.createReadStream({ autoClose: false })) { + sha256.update(chunk); + sha512.update(chunk); + } + return { sha256: sha256.digest('hex'), sha512: sha512.digest('hex') }; + } finally { + await file.close(); + } +} + +function publicUrl(value) { + const url = new URL(value); + if (url.protocol !== 'https:' || url.username || url.password || url.hash) + throw new Error('Acquisition requires a public HTTPS URL without credentials or fragment'); + return url; +} + +function isolatedEnvironment(env, windows) { + if (!env || typeof env !== 'object') throw new Error('An isolated environment is required'); + const allowed = new Set([ + 'SYSTEMROOT', 'WINDIR', 'PATH', 'HOME', 'USERPROFILE', 'LOCALAPPDATA', 'APPDATA', + 'TEMP', 'TMP', 'TMPDIR', 'DOTNET_CLI_HOME', 'NUGET_PACKAGES', 'LANG', 'LC_ALL', + ]); + const result = {}; + for (const [key, value] of Object.entries(env)) + if (allowed.has(key.toUpperCase()) && typeof value === 'string') result[key] = value; + if (windows) { + const systemRoot = Object.entries(result).find(([key]) => key.toUpperCase() === 'SYSTEMROOT')?.[1]; + if (!systemRoot || !path.win32.isAbsolute(systemRoot)) throw new Error('Absolute env.SystemRoot is required on Windows'); + result.SystemRoot = systemRoot; + result.PSModulePath = path.win32.join(systemRoot, 'System32', 'WindowsPowerShell', 'v1.0', 'Modules'); + } + result.NO_PROXY = result.no_proxy = '127.0.0.1,localhost'; + result.DOTNET_CLI_TELEMETRY_OPTOUT = '1'; + result.DOTNET_SKIP_FIRST_TIME_EXPERIENCE = '1'; + result.DOTNET_MULTILEVEL_LOOKUP = '0'; + return result; +} + +// This is a transport for already verified bytes, not an extractor or a public-feed +// fallback implementation. Only the official installer unpacks the archive. The +// caller's HTTP boundary validates every public hop and streams the original GET. +async function startFeed(fs, download, archiveName, productDirectory, version, product) { + const prefix = `/${crypto.randomBytes(24).toString('hex')}`; + const directory = `${prefix}/${productDirectory}/${version}/`; + const archiveRoute = directory + archiveName; + const versionRoutes = new Set([ + `${directory}productVersion.txt`, + `${directory}${product === 'sdk' ? 'sdk' : product === 'dotnet' ? 'runtime' : product}-productVersion.txt`, + ]); + const size = (await fs.stat(download)).size; + let requests = 0; + let active = 0; + let downloads = 0; + let error; + const server = http.createServer(async (req, res) => { + res.setHeader('Connection', 'close'); + if (++requests > 64 || active >= 4) { res.writeHead(429).end(); return; } + if (!['GET', 'HEAD'].includes(req.method)) { res.writeHead(405).end(); return; } + if (req.url !== archiveRoute && !versionRoutes.has(req.url)) { res.writeHead(404).end(); return; } + active++; + try { + if (versionRoutes.has(req.url)) { + res.setHeader('Content-Type', 'text/plain'); + res.end(req.method === 'HEAD' ? undefined : `${version}\n`); + } else { + res.setHeader('Content-Type', 'application/octet-stream'); + res.setHeader('Content-Length', size); + if (req.method === 'HEAD') res.end(); + else { + const file = await fs.open(download, 'r'); + try { + await pipeline(file.createReadStream({ autoClose: false }), res); + downloads++; + } finally { + await file.close(); + } + } + } + } catch (cause) { + error = cause; + res.destroy(); + } finally { + active--; + } + }); + server.requestTimeout = server.headersTimeout = 30_000; + server.timeout = 60_000; + server.maxHeadersCount = 32; + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + return { + url: `http://127.0.0.1:${server.address().port}${prefix}`, + verify() { + if (error) throw new Error(`Loopback archive delivery failed: ${error.message}`, { cause: error }); + if (!downloads) throw new Error('Official installer did not download the verified archive'); + }, + close: () => new Promise((resolve, reject) => { + server.close(cause => cause ? reject(cause) : resolve()); + server.closeAllConnections(); + }), + }; +} + +/** + * helperRoot contains Arcade tools.ps1/tools.sh; root is the isolated install directory. + * fs is node:fs/promises-compatible. http performs policy-checked public GETs (streaming + * to destination when supplied), and run is the caller's bounded, trusted process + * boundary. No process.env is inherited. The acquisition receipt is persisted at + * archivePath + '.provenance.json' before any acquired script executes. + */ +async function installArchive(options, deps) { + const { helperRoot, root, archivePath, product, version, platform, archive, cwd } = options; + const fs = deps.fs || require('node:fs/promises'); + if (typeof deps.http !== 'function' || typeof deps.run !== 'function') + throw new Error('Policy-constrained http and trusted run boundaries are required'); + const os = ({ win32: 'win', windows: 'win', darwin: 'osx', macos: 'osx' })[platform?.os] || platform?.os; + const windows = os === 'win'; + if (!['win', 'linux', 'linux-musl', 'osx', 'freebsd', 'rhel.6'].includes(os)) + throw new Error('Unsupported installer OS'); + if (!['sdk', 'dotnet', 'aspnetcore', 'windowsdesktop'].includes(product) || (product === 'windowsdesktop' && !windows)) + throw new Error('Unsupported installer product'); + if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(version)) + throw new Error('An exact SDK/runtime version is required, not a channel or latest'); + if (!['x64', 'x86', 'arm', 'arm64', 's390x', 'ppc64le', 'loongarch64', 'riscv64'].includes(platform.architecture)) + throw new Error('Unsupported installer architecture'); + if (!archive || !/^[0-9a-f]{128}$/i.test(archive.sha512)) + throw new Error('Official archive SHA512 is required'); + const archiveName = decodeURIComponent(publicUrl(archive.url).pathname.split('/').pop()); + if (!/^[A-Za-z0-9][A-Za-z0-9_.-]*\.(zip|tar\.gz)$/.test(archiveName) || (!windows && !archiveName.endsWith('.tar.gz'))) + throw new Error('Unsupported official archive filename'); + for (const [name, value] of Object.entries({ helperRoot, root, archivePath, cwd })) + if (typeof value !== 'string' || !path.isAbsolute(value) || /[\r\n\0]/.test(value)) + throw new Error(`Absolute ${name} is required`); + const env = isolatedEnvironment(options.env, windows); + if (windows) env.DOTNET_INSTALL_SKIP_TAR = archiveName.endsWith('.zip') ? '1' : '0'; + const extension = windows ? 'ps1' : 'sh'; + const installerUrl = `https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.${extension}`; + const helperPath = path.join(helperRoot, `tools.${extension}`); + const executable = windows + ? path.win32.join(env.SystemRoot, 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe') + : '/bin/bash'; + const shellArgs = windows ? ['-NoLogo', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass'] : ['--noprofile', '--norc']; + const attempts = []; + const logs = []; + const download = `${archivePath}.download`; + let feed; + let receipt; + async function get(url, destination) { + const attempt = { url, version, status: 'pending' }; + attempts.push(attempt); + try { + const response = await deps.http(url, { method: 'GET', ...(destination ? { destination } : {}) }); + attempt.status = response.status; + if (response.status !== 200) throw new Error(`Acquisition HTTP ${response.status}: ${url}`); + publicUrl(response.url); + if (!Array.isArray(response.redirects)) throw new Error('HTTP acquisition omitted redirect provenance'); + return response; + } catch (error) { + attempt.error = error.message; + throw error; + } + } + async function run(args, timeoutMs) { + const result = await deps.run(executable, [...shellArgs, ...args], { cwd, env: { ...env }, timeoutMs }); + logs.push({ stdout: result.stdout || '', stderr: result.stderr || '', exitCode: result.exitCode }); + if (result.exitCode !== 0) throw new Error(`Official installer process exited ${result.exitCode}: ${result.stderr || result.stdout || ''}`); + return result.stdout || ''; + } + try { + await fs.mkdir(path.dirname(archivePath), { recursive: true }); + await fs.mkdir(root, { recursive: true }); + const response = await get(archive.url, download); + const digests = await hashFile(fs, download); + if (digests.sha512 !== archive.sha512.toLowerCase()) throw new Error('Downloaded archive SHA512 does not match official metadata'); + for (const name of ['sha256', 'sha512']) + if (response[name] && response[name].toLowerCase() !== digests[name]) + throw new Error(`HTTP ${name} digest does not match downloaded archive bytes`); + const scriptResponse = await get(installerUrl); + if (!Buffer.isBuffer(scriptResponse.bytes) || scriptResponse.bytes.length === 0 || scriptResponse.bytes.length > 1_048_576) + throw new Error('Official installer script response is missing or oversized'); + const scriptDigest = hash(scriptResponse.bytes, 'sha256'); + if (scriptResponse.sha256 && scriptResponse.sha256.toLowerCase() !== scriptDigest) + throw new Error('Official installer script digest mismatch'); + const helperDigest = hash(await fs.readFile(helperPath), 'sha256'); + const cache = await fs.mkdtemp(path.join(path.dirname(archivePath), '.installer-')); + const scriptPath = path.join(cache, `dotnet-install.${extension}`); + await fs.writeFile(scriptPath, scriptResponse.bytes, { flag: 'wx' }); + if (!windows) await fs.writeFile(path.join(cache, '.dotnet-install.timestamp'), `${Math.floor(Date.now() / 1000)}\n`, { flag: 'wx' }); + receipt = { + archive: { ...archive, sha256: digests.sha256, actualSha512: digests.sha512, redirectUrl: response.url, redirects: response.redirects }, + installer: { + url: installerUrl, sha256: scriptDigest, helperPath, helperSha256: helperDigest, + path: scriptPath, redirectUrl: scriptResponse.url, redirects: scriptResponse.redirects, + }, + attempts, + }; + await fs.writeFile(`${archivePath}.provenance.json`, JSON.stringify(receipt, null, 2)); + Object.assign(env, { RB_HELPER_PATH: helperPath, RB_INSTALLER_CACHE: cache, RB_INSTALLER_PATH: scriptPath }); + const helperOutput = await run(windows ? ['-Command', helperPS] : ['-c', helperSH], 60_000); + const markers = helperOutput.split(/\r?\n/).filter(line => line.startsWith(marker)); + if (markers.length !== 1) throw new Error('Repository helper did not return exactly one installer path'); + const output = markers[0].slice(marker.length); + const helperResult = windows ? JSON.parse(output) : { path: output }; + if (path.resolve(helperResult.path) !== path.resolve(scriptPath)) throw new Error('Repository helper returned an unexpected installer path'); + if (hash(await fs.readFile(scriptPath), 'sha256') !== scriptDigest || + hash(await fs.readFile(helperPath), 'sha256') !== helperDigest) + throw new Error('Verified installer script or repository helper changed'); + const required = windows + ? ['Version', 'InstallDir', 'Architecture', 'NoPath', 'KeepZip', 'ZipPath', 'AzureFeed'] + : ['version', 'install-dir', 'architecture', 'no-path', 'keep-zip', 'zip-path', 'azure-feed', 'os']; + if (product !== 'sdk') required.push(...(windows ? ['Runtime', 'SkipNonVersionedFiles'] : ['runtime', 'skip-non-versioned-files'])); + const supported = windows + ? new Set(helperResult.parameters.map(name => name.toLowerCase())) + : new Set((await run([scriptPath, '--help'], 30_000)).match(/--[a-z][a-z-]*/g)?.map(name => name.slice(2)) || []); + for (const flag of required) + if (!supported.has(flag.toLowerCase())) throw new Error(`Official installer does not support required flag ${flag}`); + const productDirectory = product === 'sdk' ? 'Sdk' : product === 'aspnetcore' ? 'aspnetcore/Runtime' + : product === 'windowsdesktop' && Number(version.split('.')[0]) >= 5 ? 'WindowsDesktop' : 'Runtime'; + feed = await startFeed(fs, download, archiveName, productDirectory, version, product); + const args = windows + ? ['-File', scriptPath, '-Version', version, '-InstallDir', root, '-Architecture', platform.architecture, + '-NoPath', '-KeepZip', '-ZipPath', archivePath, '-AzureFeed', feed.url] + : [scriptPath, '--version', version, '--install-dir', root, '--architecture', platform.architecture, + '--no-path', '--keep-zip', '--zip-path', archivePath, '--azure-feed', feed.url, '--os', os]; + if (product !== 'sdk') args.push(windows ? '-Runtime' : '--runtime', product, windows ? '-SkipNonVersionedFiles' : '--skip-non-versioned-files'); + await run(args, 1_200_000); + feed.verify(); + const retained = await hashFile(fs, archivePath); + if (retained.sha512 !== digests.sha512 || retained.sha256 !== digests.sha256) + throw new Error('Retained archive digest does not match verified download'); + return { ...receipt, logs }; + } catch (error) { + error.attempts = attempts; + error.logs = logs; + if (receipt) error.acquisition = receipt; + throw error; + } finally { + if (feed) await feed.close(); + await fs.rm(download, { force: true }); + } +} + +module.exports = { installArchive }; diff --git a/eng/regression-bisect/toolchain-installer.test.cjs b/eng/regression-bisect/toolchain-installer.test.cjs new file mode 100644 index 00000000000..d14b0d25ae5 --- /dev/null +++ b/eng/regression-bisect/toolchain-installer.test.cjs @@ -0,0 +1,284 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { execFile } = require('node:child_process'); +const crypto = require('node:crypto'); +const fs = require('node:fs/promises'); +const http = require('node:http'); +const path = require('node:path'); +const { test } = require('node:test'); +const { promisify } = require('node:util'); +const { installArchive } = require('./toolchain-installer.cjs'); + +const digest = (bytes, algorithm) => crypto.createHash(algorithm).update(bytes).digest('hex'); +const parameters = ['Version', 'InstallDir', 'Architecture', 'Runtime', 'NoPath', 'KeepZip', 'ZipPath', 'AzureFeed', 'SkipNonVersionedFiles']; +const help = '--version --install-dir --architecture --runtime --no-path --keep-zip --zip-path --azure-feed --os --skip-non-versioned-files'; +const payload = Buffer.from('verified archive bytes'); + +function request(url, method = 'GET') { + return new Promise((resolve, reject) => { + const req = http.request(url, { method, agent: false }, response => { + const chunks = []; + response.on('data', chunk => chunks.push(chunk)); + response.on('end', () => resolve({ status: response.statusCode, bytes: Buffer.concat(chunks) })); + }); + req.on('error', reject); + req.end(); + }); +} + +async function fixture(t, { product = 'sdk', os = 'win', architecture = 'x64', version = '11.0.100-preview.3.26159.112' } = {}) { + const directory = await fs.mkdtemp(path.join(__dirname, '.installer-test-')); + t.after(() => fs.rm(directory, { recursive: true, force: true })); + const windows = os === 'win'; + const rid = `${os}-${architecture}`; + const filename = `${product === 'sdk' ? 'dotnet-sdk' : product === 'dotnet' ? 'dotnet-runtime' : `${product}-runtime`}-${version}-${rid}.${windows ? 'zip' : 'tar.gz'}`; + const helperRoot = path.join(directory, 'eng', 'common'); + await fs.mkdir(helperRoot, { recursive: true }); + const helperPath = path.join(helperRoot, windows ? 'tools.ps1' : 'tools.sh'); + await fs.writeFile(helperPath, '# trusted repository helper'); + const options = { + helperRoot, root: path.join(directory, 'sdk'), archivePath: path.join(directory, filename), + product, version, platform: { os, rid, architecture }, + archive: { name: `${product}-${rid}`, url: `https://builds.dotnet.microsoft.com/dotnet/${filename}`, sha512: digest(payload, 'sha512') }, + cwd: directory, + env: { + SystemRoot: 'C:\\Windows', PATH: 'trusted-system-path', + HOME: directory, USERPROFILE: directory, TEMP: directory, TMP: directory, + BASH_ENV: 'untrusted', ENV: 'untrusted', GH_TOKEN: 'secret', + HTTP_PROXY: 'untrusted', DOTNET_STARTUP_HOOKS: 'untrusted', PSModulePath: 'untrusted', + }, + }; + const state = { calls: [], requests: [], parameters, help, payload, feed: null, hook: null }; + const script = Buffer.from(windows + ? `[CmdletBinding()]\nparam(${parameters.map(name => `[string]$${name}`).join(',')})\n` + : '#!/bin/bash\n# Official script fixture\n'); + const deps = { + fs, + http: async (url, transport) => { + state.requests.push({ url, ...transport }); + if (transport.destination) { + await fs.writeFile(transport.destination, state.payload); + return { + status: 200, sha256: digest(state.payload, 'sha256'), sha512: digest(state.payload, 'sha512'), + url: 'https://ci.dot.net/public/archive', redirects: ['https://ci.dot.net/public/archive'], + }; + } + return { bytes: script, url, redirects: [], status: 200 }; + }, + run: async (executable, args, execution) => { + state.calls.push({ executable, args, ...execution }); + assert.equal(execution.cwd, directory); + assert.ok(execution.timeoutMs > 0 && execution.timeoutMs <= 1_200_000); + for (const key of ['GH_TOKEN', 'BASH_ENV', 'ENV', 'HTTP_PROXY', 'DOTNET_STARTUP_HOOKS']) + assert.equal(execution.env[key], undefined, key); + assert.notEqual(execution.env.PSModulePath, 'untrusted'); + const provenance = JSON.parse(await fs.readFile(`${options.archivePath}.provenance.json`, 'utf8')); + assert.equal(provenance.archive.actualSha512, digest(payload, 'sha512')); + assert.equal(provenance.archive.redirectUrl, 'https://ci.dot.net/public/archive'); + if (args.some(arg => arg.includes('GetDotNetInstallScript'))) { + assert.ok(args.some(arg => arg.includes(windows ? '$prepareMachine = $false' : 'prepare_machine=false'))); + assert.ok(args.some(arg => arg.includes(windows ? '$ci = $false' : 'ci=false'))); + assert.ok(!args.some(arg => arg.includes('InitializeDotNetCli'))); + assert.deepEqual(await fs.readFile(execution.env.RB_INSTALLER_PATH), script); + if (!windows) { + const stamp = Number(await fs.readFile(path.join(execution.env.RB_INSTALLER_CACHE, '.dotnet-install.timestamp'), 'utf8')); + assert.ok(Math.abs(stamp - Date.now() / 1000) < 10); + } + if (state.mutateScript) await fs.appendFile(execution.env.RB_INSTALLER_PATH, 'changed'); + return { + exitCode: 0, stderr: '', + stdout: `helper chatter\nINSTALLER_HELPER_RESULT=${windows ? JSON.stringify({ + path: state.wrongPath || execution.env.RB_INSTALLER_PATH, parameters: state.parameters, + }) : state.wrongPath || execution.env.RB_INSTALLER_PATH}\n`, + }; + } + if (args.includes('--help')) return { exitCode: 0, stdout: state.help, stderr: '' }; + const flag = name => args[args.indexOf(windows ? `-${name}` : `--${name.replace(/[A-Z]/g, (c, i) => `${i ? '-' : ''}${c.toLowerCase()}`)}`) + 1]; + assert.equal(flag('Version'), options.version); + assert.equal(flag('InstallDir'), options.root); + assert.equal(flag('Architecture'), options.platform.architecture); + assert.equal(flag('ZipPath'), options.archivePath); + assert.ok(args.includes(windows ? '-KeepZip' : '--keep-zip')); + assert.ok(args.includes(windows ? '-NoPath' : '--no-path')); + assert.equal(args.includes(windows ? '-SkipNonVersionedFiles' : '--skip-non-versioned-files'), product !== 'sdk'); + assert.equal(args.includes(windows ? '-Runtime' : '--runtime'), product !== 'sdk'); + if (product !== 'sdk') assert.equal(flag('Runtime'), product); + if (windows) { + assert.equal(executable, 'C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe'); + assert.ok(args.includes('-NoProfile')); + assert.equal(execution.env.DOTNET_INSTALL_SKIP_TAR, '1'); + } else { + assert.equal(executable, '/bin/bash'); + assert.deepEqual(args.slice(0, 2), ['--noprofile', '--norc']); + assert.equal(flag('Os'), os); + } + state.feed = flag('AzureFeed'); + assert.equal(new URL(state.feed).hostname, '127.0.0.1'); + const productDir = product === 'sdk' ? 'Sdk' : product === 'aspnetcore' ? 'aspnetcore/Runtime' + : product === 'windowsdesktop' && Number(version.split('.')[0]) >= 5 ? 'WindowsDesktop' : 'Runtime'; + state.archiveUrl = `${state.feed}/${productDir}/${version}/${filename}`; + const productVersion = await request(`${state.feed}/${productDir}/${version}/productVersion.txt`); + assert.equal(productVersion.status, 200); + assert.equal(productVersion.bytes.toString().trim(), version); + assert.equal((await request(state.archiveUrl, 'HEAD')).status, 200); + assert.deepEqual((await request(state.archiveUrl)).bytes, payload); + if (state.hook) return state.hook({ execution, args, flag }); + await fs.copyFile(`${options.archivePath}.download`, options.archivePath); + return { exitCode: 0, stdout: 'installed', stderr: '' }; + }, + }; + return { options, deps, state, script, helperPath }; +} + +for (const settings of [ + { product: 'sdk' }, { product: 'dotnet' }, { product: 'aspnetcore' }, { product: 'windowsdesktop' }, + { product: 'windowsdesktop', version: '3.1.32' }, + { product: 'sdk', os: 'linux' }, { product: 'dotnet', os: 'linux-musl' }, + { product: 'aspnetcore', os: 'osx' }, { product: 'sdk', architecture: 'arm64' }, +]) { + test(`installs exact ${settings.product} ${settings.os || 'win'} ${settings.architecture || 'x64'} ${settings.version || 'preview'}`, async t => { + const { options, deps, state, script, helperPath } = await fixture(t, settings); + const result = await installArchive(options, deps); + assert.equal(result.archive.sha256, digest(payload, 'sha256')); + assert.equal(result.archive.actualSha512, options.archive.sha512); + assert.equal(result.archive.name, options.archive.name); + assert.equal(result.archive.url, options.archive.url); + assert.equal(result.archive.redirectUrl, 'https://ci.dot.net/public/archive'); + assert.deepEqual(result.archive.redirects, ['https://ci.dot.net/public/archive']); + assert.equal(result.installer.sha256, digest(script, 'sha256')); + assert.equal(result.installer.helperPath, helperPath); + assert.equal(result.installer.helperSha256, digest(await fs.readFile(helperPath), 'sha256')); + assert.ok(result.attempts.some(attempt => attempt.url === options.archive.url && attempt.version === options.version && attempt.status === 200)); + assert.equal(state.requests.length, 2); + assert.ok(state.requests.every(req => req.method === 'GET')); + assert.ok(state.calls.every(call => call.executable === ((settings.os || 'win') === 'win' + ? 'C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe' : '/bin/bash'))); + assert.deepEqual(await fs.readFile(options.archivePath), payload); + await assert.rejects(request(state.archiveUrl), /ECONNREFUSED|ECONNRESET/); + await assert.rejects(fs.stat(`${options.archivePath}.download`), { code: 'ENOENT' }); + }); +} + +for (const os of ['win', 'linux']) { + for (const product of ['sdk', 'dotnet']) { + test(`missing nonversioned-file protection is ${product === 'sdk' ? 'allowed' : 'rejected'} for ${product} on ${os}`, async t => { + const { options, deps, state } = await fixture(t, { os, product }); + state.parameters = parameters.filter(name => name !== 'SkipNonVersionedFiles'); + state.help = help.replace('--skip-non-versioned-files', '--skip-non-versioned-files-unsupported'); + if (product === 'sdk') await installArchive(options, deps); + else { + await assert.rejects(installArchive(options, deps), /support.*SkipNonVersionedFiles|support.*skip-non-versioned-files/i); + assert.equal(state.feed, null); + } + }); + } +} + +test('rejects wrong SHA512 before any process runs', async t => { + const { options, deps, state } = await fixture(t); + state.payload = Buffer.from('wrong archive'); + await assert.rejects(installArchive(options, deps), /SHA512/i); + assert.equal(state.calls.length, 0); +}); + +test('independently hashes streamed bytes instead of trusting digest claims', async t => { + const { options, deps, state } = await fixture(t); + const transport = deps.http; + deps.http = async (...args) => { + const response = await transport(...args); + if (args[1].destination) await fs.writeFile(args[1].destination, 'tampered bytes'); + return response; + }; + await assert.rejects(installArchive(options, deps), /SHA512|digest/i); + assert.equal(state.calls.length, 0); +}); + +for (const setting of ['missing PS parameter', 'missing Unix help flag', 'wrong helper path', 'changed cached script']) { + test(`fails closed: ${setting}`, async t => { + const unix = setting.includes('Unix'); + const { options, deps, state } = await fixture(t, { os: unix ? 'linux' : 'win' }); + if (setting.includes('PS parameter')) state.parameters = parameters.filter(p => p !== 'KeepZip'); + if (unix) state.help = help.replace('--keep-zip', '--keep-zip-unsupported'); + if (setting.includes('path')) state.wrongPath = path.join(options.cwd, 'other.ps1'); + if (setting.includes('script')) state.mutateScript = true; + const expected = setting.includes('path') ? /path/i : setting.includes('script') ? /changed|digest/i : /support.*KeepZip|support.*keep-zip/i; + await assert.rejects(installArchive(options, deps), expected); + assert.equal(state.feed, null); + }); +} + +test('loopback feed refuses all other versions, paths, queries and methods', async t => { + const { options, deps, state } = await fixture(t); + state.hook = async () => { + for (const url of [ + `${state.feed}/Sdk/latest.version`, `${state.feed}/Sdk/8.0.100/productVersion.txt`, + `${state.feed}/Sdk/${options.version}/other.zip`, `${state.archiveUrl}?redirect=https://example.com`, + `${state.feed}/../../secret`, `${state.feed}/Sdk/${options.version}/%2e%2e/secret`, + ]) assert.equal((await request(url)).status, 404, url); + assert.equal((await request(state.archiveUrl, 'POST')).status, 405); + await fs.copyFile(`${options.archivePath}.download`, options.archivePath); + return { exitCode: 0, stdout: '', stderr: '' }; + }; + await installArchive(options, deps); +}); + +for (const failure of ['exit', 'throw', 'retained digest', 'missing archive']) { + test(`closes feed on ${failure} failure`, async t => { + const { options, deps, state } = await fixture(t); + state.hook = async () => { + if (failure === 'exit') return { exitCode: 17, stdout: '', stderr: 'installer failed' }; + if (failure === 'throw') throw new Error('process timeout'); + if (failure === 'retained digest') await fs.writeFile(options.archivePath, 'corrupted retention'); + return { exitCode: 0, stdout: '', stderr: '' }; + }; + await assert.rejects(installArchive(options, deps), /17|timeout|digest|SHA512|ENOENT|archive/i); + await assert.rejects(request(state.archiveUrl), /ECONNREFUSED|ECONNRESET/); + }); +} + +for (const mutation of [ + o => { o.version = 'latest'; }, o => { o.product = 'invalid'; }, + o => { o.archive.sha512 = ''; }, o => { o.platform.architecture = 'x64; bad'; }, + o => { o.env = undefined; }, o => { o.archive.url = 'http://example.com/file.zip'; }, + o => { o.env.SystemRoot = 'relative'; }, +]) { + test(`rejects invalid input: ${mutation}`, async t => { + const { options, deps, state } = await fixture(t); + mutation(options); + await assert.rejects(installArchive(options, deps)); + assert.equal(state.calls.length, 0); + assert.equal(state.requests.length, 0); + }); +} + +test('unavailable archive is explicit, with no fallback or process execution', async t => { + const { options, deps, state } = await fixture(t); + deps.http = async url => ({ status: 404, url, redirects: [] }); + await assert.rejects(installArchive(options, deps), error => { + assert.match(error.message, /404/); + assert.equal(error.attempts[0].url, options.archive.url); + return true; + }); + assert.equal(state.calls.length, 0); +}); + +test('real repository helper returns seeded script and parses actual PS parameters', { skip: process.platform !== 'win32' }, async t => { + const { options, deps } = await fixture(t); + options.helperRoot = path.resolve(__dirname, '..', 'common'); + options.env.SystemRoot = process.env.SystemRoot; + const simulatedRun = deps.run; + let helperRan = false; + deps.run = async (executable, args, execution) => { + if (!args.some(arg => arg.includes('GetDotNetInstallScript'))) + return simulatedRun(executable, args, execution); + helperRan = true; + const { stdout, stderr } = await promisify(execFile)(executable, args, { + cwd: execution.cwd, env: execution.env, timeout: execution.timeoutMs, + windowsHide: true, maxBuffer: 1024 * 1024, + }); + return { stdout, stderr, exitCode: 0 }; + }; + await installArchive(options, deps); + assert.equal(helperRan, true); +}); diff --git a/eng/regression-bisect/toolchain-io.cjs b/eng/regression-bisect/toolchain-io.cjs new file mode 100644 index 00000000000..5481db6db47 --- /dev/null +++ b/eng/regression-bisect/toolchain-io.cjs @@ -0,0 +1,233 @@ +const fs = require("node:fs/promises"); +const path = require("node:path"); +const { createHash } = require("node:crypto"); +const { execFile } = require("node:child_process"); + +const hash = bytes => createHash("sha256").update(bytes).digest("hex"); +const requireValue = (condition, message) => { if (!condition) throw new Error(message); }; +const inside = (root, file) => { + const relative = path.relative(root, file); + return relative !== "" && !relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative); +}; + +function publicUrl(url, policy) { + const parsed = new URL(url); + requireValue(parsed.protocol === "https:" && !parsed.username && !parsed.password && !parsed.search && + !parsed.hash && !parsed.port && policy.toolchains.publicFeeds.some(feed => + parsed.href === feed || parsed.href.startsWith(`${feed}/`)), `unapproved public download URL: ${url}`); + return url; +} + +function createHttp(policy, fileSystem = fs, signal) { + return async (url, { destination, method = "GET" } = {}) => { + const redirects = []; + let current = publicUrl(url, policy); + for (let hop = 0; hop <= 5; hop++) { + const response = await fetch(current, { + method, redirect: "manual", + signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(300000)]) : AbortSignal.timeout(300000), + }); + if ([301, 302, 303, 307, 308].includes(response.status)) { + await response.body?.cancel(); + const next = publicUrl(new URL(response.headers.get("location"), current).href, policy); + redirects.push({ from: current, to: next, status: response.status }); + current = next; + continue; + } + const sha256 = createHash("sha256"); + const sha512 = createHash("sha512"); + const chunks = []; + let size = 0; + const file = destination && response.ok ? await fileSystem.open(destination, "wx") : null; + try { + for await (const chunk of response.body ?? []) { + size += chunk.length; + requireValue(size <= (file ? 1024 * 1024 * 1024 : 16 * 1024 * 1024), `download exceeds byte limit: ${current}`); + sha256.update(chunk); sha512.update(chunk); + if (file) await file.writeFile(chunk); + else chunks.push(chunk); + } + } finally { + await file?.close(); + } + return { + status: response.status, url: current, redirects, + ...(file ? {} : { bytes: Buffer.concat(chunks) }), + sha256: sha256.digest("hex"), sha512: sha512.digest("hex"), size, + }; + } + throw new Error(`too many redirects: ${url}`); + }; +} + +function run(executable, args, { cwd, env, timeoutMs = 120000, signal } = {}) { + return new Promise((resolve, reject) => { + execFile(executable, args, { cwd, env, timeout: timeoutMs, signal, maxBuffer: 4 * 1024 * 1024, windowsHide: true }, + (error, stdout, stderr) => { + if (error && (error.killed || typeof error.code !== "number")) reject(error); + else resolve({ stdout, stderr, exitCode: error?.code ?? 0 }); + }); + }); +} + +function isolatedEnvironment(root, platform) { + const home = path.join(root, "home"); + const dotnet = path.join(root, "dotnet"); + const systemRoot = process.env.SystemRoot || "C:\\Windows"; + const systemPath = platform.os === "windows" + ? [path.join(systemRoot, "System32"), systemRoot, path.join(systemRoot, "System32", "WindowsPowerShell", "v1.0")].join(path.delimiter) + : "/usr/bin:/bin:/usr/sbin:/sbin"; + return { + ...(platform.os === "windows" ? { SystemRoot: systemRoot, WINDIR: systemRoot, ComSpec: path.join(systemRoot, "System32", "cmd.exe") } : {}), + PATH: `${dotnet}${path.delimiter}${systemPath}`, + HOME: home, USERPROFILE: home, APPDATA: path.join(home, "AppData", "Roaming"), LOCALAPPDATA: path.join(home, "AppData", "Local"), + TMP: path.join(root, "tmp"), TEMP: path.join(root, "tmp"), TMPDIR: path.join(root, "tmp"), + DOTNET_ROOT: dotnet, [`DOTNET_ROOT_${platform.architecture.toUpperCase()}`]: dotnet, + DOTNET_INSTALL_DIR: dotnet, DOTNET_GLOBAL_INSTALL_DIR: dotnet, + DOTNET_HOST_PATH: path.join(dotnet, platform.os === "windows" ? "dotnet.exe" : "dotnet"), + DOTNET_MSBUILD_SDK_RESOLVER_CLI_DIR: dotnet, + DOTNET_CLI_HOME: home, DOTNET_MULTILEVEL_LOOKUP: "0", DOTNET_ROLL_FORWARD: "Disable", + DOTNET_CLI_TELEMETRY_OPTOUT: "1", DOTNET_SKIP_FIRST_TIME_EXPERIENCE: "1", DOTNET_NOLOGO: "1", + DOTNET_CLI_UI_LANGUAGE: "en-US", DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE: "true", + DOTNET_INSTALL_SKIP_TAR: "1", MSBUILDDISABLENODEREUSE: "1", + NUGET_PACKAGES: path.join(root, "nuget", "packages"), NUGET_HTTP_CACHE_PATH: path.join(root, "nuget", "http"), + NUGET_PLUGINS_CACHE_PATH: path.join(root, "nuget", "plugins"), + NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED: "false", LANG: "C.UTF-8", + }; +} + +async function fileIdentity(file, root, fileSystem = fs) { + const real = await fileSystem.realpath(file); + requireValue(inside(root, real), `identity path escapes isolated installation: ${real}`); + const bytes = await fileSystem.readFile(real); + return { path: real, sha256: hash(bytes) }; +} + +// No reproduction is executed: this fixed probe only inspects the acquired product. +const probe = ` +open System +open System.IO +open System.Reflection +open System.Diagnostics +open System.Runtime.InteropServices +open System.IO.Compression +open System.Xml.Linq +printfn "ARCH|%s" (RuntimeInformation.ProcessArchitecture.ToString().ToLowerInvariant()) +printfn "RUNTIME|%s" (Path.GetFileName(Path.GetDirectoryName(typeof.Assembly.Location))) +for file in fsi.CommandLineArgs |> Array.skip 1 do + if Path.GetExtension(file) = ".nupkg" then + use archive = ZipFile.OpenRead(file) + let entry = archive.Entries |> Seq.find (fun e -> e.FullName.EndsWith(".nuspec", StringComparison.Ordinal)) + use stream = entry.Open() + let xml = XDocument.Load(stream) + let value name = xml.Descendants() |> Seq.find (fun e -> e.Name.LocalName = name) |> fun e -> e.Value + printfn "PACKAGE|%s|%s|%s" (Path.GetFileName(file)) (value "id") (value "version") + else + let assembly = AssemblyName.GetAssemblyName(file) + let info = FileVersionInfo.GetVersionInfo(file) + printfn "FILE|%s|%s|%O|%s" (Path.GetFileName(file)) assembly.Name assembly.Version info.FileVersion +`; + +async function capture(request, deps) { + const { root, executable, cwd, env, toolchain } = request; + const command = async args => { + const result = await deps.run(executable, args, { cwd, env, timeoutMs: 120000 }); + requireValue(result.exitCode === 0, `${args.join(" ")} failed (${result.exitCode}): ${result.stderr || result.stdout}`); + return result.stdout.trim(); + }; + const info = await command(["--info"]); + const sdkList = await command(["--list-sdks"]); + const runtimeList = await command(["--list-runtimes"]); + const sdkVersion = await command(["--version"]); + const sdkBasePath = path.resolve(/^\s*Base Path:\s*(.+)/m.exec(info)?.[1]?.trim() ?? ""); + requireValue(sdkBasePath === path.join(root, "sdk", toolchain.sdkVersion), "SDK resolver selected a path outside the acquired SDK"); + const sdks = sdkList.split(/\r?\n/).filter(Boolean).map(line => { + const match = /^(\S+) \[(.+)\]$/.exec(line); + requireValue(match, `unrecognized SDK identity: ${line}`); + return { version: match[1], path: match[2] }; + }); + const runtimes = []; + for (const line of runtimeList.split(/\r?\n/).filter(Boolean)) { + const match = /^(\S+) (\S+) \[(.+)\]$/.exec(line); + requireValue(match, `unrecognized runtime identity: ${line}`); + const [name, version, location] = match.slice(1); + const marker = name === "Microsoft.NETCore.App" ? "System.Private.CoreLib.dll" + : name === "Microsoft.AspNetCore.App" ? "Microsoft.AspNetCore.dll" : "PresentationCore.dll"; + const file = await fileIdentity(path.join(location, version, marker), root, deps.fs); + runtimes.push({ name, version, path: await deps.fs.realpath(location), sha256: file.sha256 }); + } + const fsharp = path.join(root, "sdk", toolchain.sdkVersion, "FSharp"); + const files = {}; + for (const [name, file] of Object.entries({ compiler: "fsc.dll", fsi: "fsi.dll", core: "FSharp.Core.dll" })) { + files[name] = await fileIdentity(path.join(fsharp, file), root, deps.fs); + } + const config = JSON.parse(await deps.fs.readFile(path.join(fsharp, "fsi.runtimeconfig.json"), "utf8")); + const framework = config.runtimeOptions?.framework; + requireValue(framework?.name === "Microsoft.NETCore.App", "unsupported bundled FSI runtimeconfig"); + const runtime = runtimes.find(r => r.name === framework.name && r.version === toolchain.bundledRuntime.version); + requireValue(runtime, "bundled compiler runtime is missing"); + const probeFile = path.join(env.DOTNET_CLI_HOME, "identity.fsx"); + await deps.fs.writeFile(probeFile, probe, { flag: "wx" }); + let packageFile = null; + const libraryPacks = path.join(fsharp, "library-packs"); + try { + const names = (await deps.fs.readdir(libraryPacks)).filter(n => /^FSharp\.Core\..+\.nupkg$/i.test(n)); + requireValue(names.length === 1, "ambiguous bundled FSharp.Core package"); + packageFile = await fileIdentity(path.join(libraryPacks, names[0]), root, deps.fs); + } catch (error) { + if (error.code !== "ENOENT") throw error; + } + // Pin the runtime explicitly; SDKs often declare a lower minimum patch. + const probeOutput = await command(["exec", "--fx-version", runtime.version, files.fsi.path, + "--nologo", "--exec", probeFile, ...Object.values(files).map(f => f.path), ...(packageFile ? [packageFile.path] : [])]); + const architecture = /^ARCH\|(.+)$/m.exec(probeOutput)?.[1]?.trim(); + const compilerRuntime = { name: framework.name, version: /^RUNTIME\|(.+)$/m.exec(probeOutput)?.[1]?.trim() }; + const runtimeExecutions = []; + for (const required of toolchain.runtimeRequirements.filter(r => r.name === "Microsoft.NETCore.App" && ["compiler", "fsi"].includes(r.role))) { + requireValue(hash(await deps.fs.readFile(required.config.path)) === required.config.sha256, "runtimeconfig changed after acquisition request"); + requireValue(required.config.path.endsWith(".runtimeconfig.json"), "unsupported local compiler runtimeconfig name"); + const compiler = await deps.fs.realpath(required.config.path.replace(/\.runtimeconfig\.json$/, ".dll")); + requireValue(inside(toolchain.sourceDirectory, compiler), "local compiler path escapes historical source"); + const trace = path.join(env.DOTNET_CLI_HOME, `${required.role}-host-trace.txt`); + const result = await deps.run(executable, ["exec", "--fx-version", required.version, compiler, "--help"], { + cwd, env: { ...env, COREHOST_TRACE: "1", COREHOST_TRACEFILE: trace, DOTNET_HOST_TRACE: "1", DOTNET_HOST_TRACEFILE: trace }, + timeoutMs: 120000, + }); + requireValue(result.exitCode === 0, `local ${required.role} identity probe failed (${result.exitCode}): ${result.stderr}`); + const output = await deps.fs.readFile(trace, "utf8"); + const expected = path.join(root, "shared", required.name, required.version); + const selected = [...output.matchAll(/Chose FX version \[([^\]]+)\]/g)].map(m => path.normalize(m[1])); + requireValue(selected.includes(expected), `local ${required.role} host trace did not verify ${expected}`); + runtimeExecutions.push({ + role: required.role, configDigest: required.config.sha256, name: required.name, + version: required.version, architecture, hostTrace: { path: trace, sha256: hash(output) }, + compiler: { path: compiler, sha256: hash(await deps.fs.readFile(compiler)) }, + }); + } + for (const file of Object.values(files)) { + const line = probeOutput.split(/\r?\n/).find(l => l.startsWith(`FILE|${path.basename(file.path)}|`)); + requireValue(line, `assembly identity absent: ${file.path}`); + [, , file.assemblyName, file.assemblyVersion, file.fileVersion] = line.split("|"); + } + const depsFile = await fileIdentity(path.join(fsharp, "fsi.deps.json"), root, deps.fs); + const packages = JSON.parse(await deps.fs.readFile(depsFile.path, "utf8")).libraries; + const corePackage = Object.keys(packages).find(p => p.startsWith("FSharp.Core/")) ?? null; + files.core.packageIdentity = corePackage; + if (packageFile) { + const packageLine = probeOutput.split(/\r?\n/).find(l => l.startsWith(`PACKAGE|${path.basename(packageFile.path)}|`)); + requireValue(packageLine, "bundled FSharp.Core nuspec identity absent"); + const [, , id, version] = packageLine.split("|"); + requireValue(id === "FSharp.Core" && /^\d+\.\d+\.\d+(?:[-+].+)?$/.test(version), "invalid FSharp.Core package identity"); + files.core.packageIdentity = `${id}/${version}`; + files.core.package = { ...packageFile, id, version }; + } + const host = await fileIdentity(executable, root, deps.fs); + const rid = /^\s*RID:\s*(\S+)/m.exec(info)?.[1]; + return { + root, hostPath: host.path, hostSha256: host.sha256, sdkVersion, sdkBasePath, sdks, runtimes, + architecture, rid, compilerRuntime, runtimeExecutions, files, depsFile, + evidence: { info, sdkList, runtimeList, probeOutput }, + }; +} + +module.exports = { hash, requireValue, inside, publicUrl, createHttp, run, isolatedEnvironment, fileIdentity, capture }; diff --git a/eng/regression-bisect/toolchains.cjs b/eng/regression-bisect/toolchains.cjs new file mode 100644 index 00000000000..b46da225c0a --- /dev/null +++ b/eng/regression-bisect/toolchains.cjs @@ -0,0 +1,390 @@ +const fs = require("node:fs/promises"); +const path = require("node:path"); +const { validatePolicy, fingerprintInput } = require("./model.cjs"); +const { hash, requireValue, inside, publicUrl, createHttp, run, isolatedEnvironment, capture } = require("./toolchain-io.cjs"); + +const indexUrl = "https://builds.dotnet.microsoft.com/dotnet/release-metadata/releases-index.json"; +const exactVersion = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$/; +const stableVersion = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/; +const runtimeProducts = { "Microsoft.NETCore.App": "dotnet", "Microsoft.AspNetCore.App": "aspnetcore", "Microsoft.WindowsDesktop.App": "windowsdesktop" }; + +function dependencies(options, injected) { + validatePolicy(options.policy); + const fileSystem = injected.fs ?? fs; + return { + fs: fileSystem, http: createHttp(options.policy, fileSystem, options.signal), + run: (file, args, settings) => run(file, args, { ...settings, signal: options.signal }), + clock: () => new Date(), capture, ...injected, + }; +} +function failure(stage, error, attempts = []) { + return { + status: "untestable", + outcome: { + schemaVersion: 1, outcome: "untestable", + reason: { identity: "wrong-identity", sdk: "unavailable-sdk", runtime: "unavailable-runtime" }[stage] ?? "setup-failure", + }, + error: { stage, code: error.code ?? "toolchain-unavailable", message: error.message }, attempts, + }; +} +function validatePlatform(platform) { + requireValue(platform && ["x64", "arm64"].includes(platform.architecture), "unsupported architecture"); + const prefix = { windows: "win", linux: "linux", macos: "osx" }[platform.os]; + requireValue(prefix && (platform.rid === `${prefix}-${platform.architecture}` || + platform.os === "linux" && platform.rid === `linux-musl-${platform.architecture}`), "OS/RID/architecture mismatch"); +} +function validateResponse(url, response, policy) { + publicUrl(url, policy); + publicUrl(response.url, policy); + for (const hop of response.redirects ?? []) { + publicUrl(hop.from, policy); publicUrl(hop.to, policy); + } +} +async function document(url, options, deps, records, attempts) { + publicUrl(url, options.policy); + const attempt = { url, status: "pending" }; + attempts.push(attempt); + const response = await deps.http(url); + attempt.status = response.status; + validateResponse(url, response, options.policy); + requireValue(response.status === 200, `metadata HTTP ${response.status}: ${url}`); + const bytes = Buffer.from(response.bytes); + records.push({ url, redirectUrl: response.url, sha256: hash(bytes), bytesBase64: bytes.toString("base64") }); + return JSON.parse(bytes.toString("utf8")); +} +async function releases(options, deps, records, attempts, channel) { + const index = await document(indexUrl, options, deps, records, attempts); + requireValue(Array.isArray(index["releases-index"]) && index["releases-index"].length <= 32, "invalid release index"); + const result = []; + for (const item of index["releases-index"]) { + requireValue(/^\d+\.\d+$/.test(item["channel-version"]), "invalid release channel"); + if (channel && item["channel-version"] !== channel) continue; + const doc = await document(item["releases.json"], options, deps, records, attempts); + requireValue(doc["channel-version"] === item["channel-version"] && Array.isArray(doc.releases), "invalid channel metadata"); + for (const release of doc.releases) result.push({ channel: item["channel-version"], release }); + } + return result; +} +function selectArchive(product, platform, policy) { + const extension = platform.os === "windows" ? ".zip" : ".tar.gz"; + const files = product.files?.filter(file => file.rid === platform.rid && file.name?.endsWith(extension) && + file.url?.endsWith(extension)); + requireValue(files?.length === 1, `missing or ambiguous ${platform.rid} ${extension} archive for ${product.version}`); + const file = files[0]; + publicUrl(file.url, policy); + requireValue(/^[0-9a-f]{128}$/i.test(file.hash), `malformed SHA-512 for ${file.name}`); + return { name: file.name, url: file.url, sha512: file.hash.toLowerCase(), rid: file.rid, architecture: platform.architecture }; +} +function sdkEntries(release) { + return [release.sdk, ...(release.sdks ?? [])].filter(Boolean); +} +function numericVersion(a, b) { + const left = a.split(".").map(BigInt); + const right = b.split(".").map(BigInt); + for (let i = 0; i < 3; i++) if (left[i] !== right[i]) return left[i] > right[i] ? 1 : -1; + return 0; +} + +async function resolveLatestStableSdk(options, injected = {}) { + const attempts = []; + try { + const deps = dependencies(options, injected); + validatePlatform(options.platform); + const now = deps.clock(); + requireValue(Number.isFinite(now.getTime()), "invalid run clock"); + const metadata = []; + const candidates = new Map(); + for (const { channel, release } of await releases(options, deps, metadata, attempts)) { + const date = release["release-date"]; + requireValue(/^\d{4}-\d{2}-\d{2}$/.test(date) && Number.isFinite(Date.parse(date)) && + new Date(date).toISOString().slice(0, 10) === date, "invalid release date"); + if (Date.parse(date) > now.getTime()) continue; + for (const sdk of sdkEntries(release)) { + if (!stableVersion.test(sdk.version)) continue; + const candidate = { sdk, channel, releaseDate: date, bundledRuntime: { version: sdk["runtime-version"] ?? release.runtime?.version } }; + const entries = candidates.get(sdk.version) ?? []; + entries.push(candidate); + candidates.set(sdk.version, entries); + } + } + const version = [...candidates.keys()].sort(numericVersion).at(-1); + requireValue(version, "no released stable SDK in official metadata"); + const entries = candidates.get(version).sort((a, b) => a.releaseDate.localeCompare(b.releaseDate)); + const selected = entries[0]; + const archive = selectArchive(selected.sdk, options.platform, options.policy); + for (const duplicate of entries.slice(1)) { + requireValue(fingerprintInput(archive) === fingerprintInput(selectArchive(duplicate.sdk, options.platform, options.policy)) && + duplicate.bundledRuntime.version === selected.bundledRuntime.version, `conflicting SDK metadata: ${version}`); + } + requireValue(exactVersion.test(selected.bundledRuntime.version), "missing bundled runtime metadata"); + return { + status: "resolved", + toolchain: { + kind: "released", sdkVersion: version, toolsVersion: version, platform: options.platform, + channel: selected.channel, releaseDate: selected.releaseDate, frozenAt: now.toISOString(), + archive, + bundledRuntime: selected.bundledRuntime, metadata, runtimeRequirements: [], + }, attempts, + }; + } catch (error) { return failure("metadata", error, attempts); } +} + +async function snapshot(file, deps, optional = false) { + try { + const real = await deps.fs.realpath(file); + const bytes = await deps.fs.readFile(real); + return { path: real, sha256: hash(bytes), bytesBase64: bytes.toString("base64") }; + } catch (error) { + if (optional && error.code === "ENOENT") return null; + throw error; + } +} +async function readRevisionToolchain(options, injected = {}) { + try { + const deps = dependencies(options, injected); + validatePlatform(options.platform); + requireValue(/^[0-9a-f]{40}$/.test(options.sourceSha), "expected pinned source SHA"); + const sourceDirectory = await deps.fs.realpath(options.sourceDirectory); + const globalJson = await snapshot(path.join(sourceDirectory, "global.json"), deps); + requireValue(inside(sourceDirectory, globalJson.path), "global.json resolves outside historical source"); + const declaration = JSON.parse(Buffer.from(globalJson.bytesBase64, "base64").toString("utf8")); + const sdkVersion = declaration.sdk?.version; + const toolsVersion = declaration.tools?.dotnet; + requireValue(exactVersion.test(sdkVersion) && exactVersion.test(toolsVersion), "missing exact sdk.version or tools.dotnet"); + requireValue(sdkVersion === toolsVersion, `source sdk.version ${sdkVersion} disagrees with tools.dotnet ${toolsVersion}`); + const runtimeRequirements = []; + const declaredRuntimes = declaration.tools.runtimes ?? {}; + requireValue(declaredRuntimes && typeof declaredRuntimes === "object" && !Array.isArray(declaredRuntimes), "invalid tools.runtimes"); + for (const [key, versions] of Object.entries(declaredRuntimes)) { + const [product, architecture = options.platform.architecture, extra] = key.split("/"); + const name = Object.keys(runtimeProducts).find(name => runtimeProducts[name] === product); + requireValue(name && !extra && architecture === options.platform.architecture && Array.isArray(versions), + `unsupported historical runtime declaration: ${key}`); + for (const version of versions) { + requireValue(exactVersion.test(version), `historical runtime version must be exact: ${version}`); + runtimeRequirements.push({ name, version, role: "build", config: globalJson }); + } + } + for (const config of options.runtimeConfigs ?? []) { + requireValue(["compiler", "fsi", "sample"].includes(config.role), "invalid runtime requirement role"); + const record = await snapshot(config.path, deps); + requireValue(config.role === "sample" || inside(sourceDirectory, record.path), "compiler runtimeconfig resolves outside historical source"); + const settings = JSON.parse(Buffer.from(record.bytesBase64, "base64").toString("utf8")).runtimeOptions; + const frameworks = settings?.framework ? [settings.framework] : settings?.frameworks; + requireValue(Array.isArray(frameworks) && frameworks.length > 0, `missing framework requirement: ${config.path}`); + for (const framework of frameworks) { + requireValue(runtimeProducts[framework.name] && exactVersion.test(framework.version), `unsupported runtime requirement: ${config.path}`); + runtimeRequirements.push({ name: framework.name, version: framework.version, role: config.role, config: record }); + } + } + // Older sources need not have a centralized TFM file. Never invent or override it. + const declarations = { + targetFrameworks: await snapshot(path.join(sourceDirectory, "eng", "TargetFrameworks.props"), deps, true), + buildSh: await snapshot(path.join(sourceDirectory, "eng", "build.sh"), deps, true), + buildPs1: await snapshot(path.join(sourceDirectory, "eng", "Build.ps1"), deps, true), + }; + requireValue(Object.values(declarations).every(file => file === null || inside(sourceDirectory, file.path)), "build declaration resolves outside historical source"); + return { + status: "resolved", + toolchain: { + kind: "source", sourceSha: options.sourceSha, sourceDirectory, globalJson, declarations, + sdkVersion, toolsVersion, sdkSelection: declaration.sdk, platform: options.platform, runtimeRequirements, + }, + }; + } catch (error) { return failure("declarations", error); } +} + +async function exactArchive(toolchain, product, version, options, deps, attempts) { + const metadata = []; + const rows = await releases(options, deps, metadata, attempts, version.split(".").slice(0, 2).join(".")); + for (const row of rows) { + const entries = product === "sdk" ? sdkEntries(row.release) + : [row.release[{ dotnet: "runtime", aspnetcore: "aspnetcore-runtime", windowsdesktop: "windowsdesktop" }[product]]].filter(Boolean); + const found = entries.find(entry => entry.version === version); + if (found) return { + archive: selectArchive(found, toolchain.platform, options.policy), metadata, + bundledRuntime: { version: found["runtime-version"] ?? row.release.runtime?.version }, + }; + } + // Public unreleased build archives have no release record. Require their official + // sidecar checksum, never a hash computed from an otherwise unauthenticated archive. + const family = { sdk: "Sdk", dotnet: "Runtime", aspnetcore: "aspnetcore/Runtime", windowsdesktop: "WindowsDesktop" }[product]; + const prefix = { sdk: "dotnet-sdk", dotnet: "dotnet-runtime", aspnetcore: "aspnetcore-runtime", windowsdesktop: "windowsdesktop-runtime" }[product]; + const name = `${prefix}-${version}-${toolchain.platform.rid}${toolchain.platform.os === "windows" ? ".zip" : ".tar.gz"}`; + for (const feed of options.policy.toolchains.publicFeeds.filter(f => /(?:builds\.dotnet\.microsoft\.com\/dotnet|ci\.dot\.net\/public)$/.test(f))) { + const url = `${feed}/${family}/${version}/${name}`; + const attempt = { url: `${url}.sha512`, version, status: "pending" }; + attempts.push(attempt); + const response = await deps.http(attempt.url); + attempt.status = response.status; + validateResponse(attempt.url, response, options.policy); + if (response.status === 404) continue; + requireValue(response.status === 200, `checksum HTTP ${response.status}: ${attempt.url}`); + const checksum = Buffer.from(response.bytes).toString("utf8").trim().split(/\s+/)[0]; + requireValue(/^[a-f0-9]{128}$/i.test(checksum), `malformed historical checksum: ${attempt.url}`); + metadata.push({ url: attempt.url, redirectUrl: response.url, sha256: hash(response.bytes), bytesBase64: Buffer.from(response.bytes).toString("base64") }); + return { archive: { name, url, sha512: checksum.toLowerCase(), rid: toolchain.platform.rid, architecture: toolchain.platform.architecture }, metadata }; + } + throw new Error(`exact ${product} ${version} unavailable from permitted public feeds`); +} + +function verifyToolchainIdentity({ policy, toolchain, identity }) { + try { + validatePolicy(policy); + validatePlatform(toolchain.platform); + const root = identity.root; + requireValue(path.isAbsolute(root) && root === toolchain.installationRoot && + identity.hostPath === path.join(root, toolchain.platform.os === "windows" ? "dotnet.exe" : "dotnet"), + "wrong selected host path"); + requireValue(identity.sdkVersion === toolchain.sdkVersion && toolchain.toolsVersion === toolchain.sdkVersion, "resolved SDK version mismatch"); + requireValue(identity.sdkBasePath === path.join(root, "sdk", toolchain.sdkVersion), "wrong resolved SDK base path"); + requireValue(identity.architecture === toolchain.platform.architecture && identity.rid === toolchain.platform.rid, "resolved architecture/RID mismatch"); + requireValue(identity.sdks.length === 1 && identity.sdks[0].version === toolchain.sdkVersion && + identity.sdks[0].path === path.join(root, "sdk"), "unexpected installed SDK or SDK location"); + requireValue(identity.archive?.sha512 === toolchain.archive.sha512 && identity.archive.actualSha512 === toolchain.archive.sha512 && + identity.archive.url === toolchain.archive.url && /^[a-f0-9]{64}$/.test(identity.archive.sha256), "archive identity mismatch"); + publicUrl(identity.archive.redirectUrl, policy); + for (const [name, assemblyName] of Object.entries({ compiler: "fsc", fsi: "fsi", core: "FSharp.Core" })) { + const file = identity.files[name]; + requireValue(file && inside(path.join(root, "sdk", toolchain.sdkVersion, "FSharp"), file.path) && + path.basename(file.path) === `${assemblyName}.dll` && file.assemblyName === assemblyName && + /^[a-f0-9]{64}$/.test(file.sha256) && /^\d+\.\d+\.\d+\.\d+$/.test(file.assemblyVersion) && + file.fileVersion, `missing or invalid bundled F# ${name} identity`); + } + for (const runtime of identity.runtimes) { + requireValue(runtime.path === path.join(root, "shared", runtime.name) && + /^[a-f0-9]{64}$/.test(runtime.sha256), "wrong runtime installation identity"); + } + for (const required of [{ name: "Microsoft.NETCore.App", version: toolchain.bundledRuntime.version }, ...toolchain.runtimeRequirements]) { + requireValue(identity.runtimes.some(r => r.name === required.name && r.version === required.version), `missing runtime ${required.name} ${required.version}`); + } + for (const required of toolchain.runtimeRequirements.filter(r => r.name === "Microsoft.NETCore.App" && ["compiler", "fsi"].includes(r.role))) { + requireValue(identity.runtimeExecutions?.some(r => r.role === required.role && r.configDigest === required.config.sha256 && + r.name === required.name && r.version === required.version && r.architecture === toolchain.platform.architecture), + `unverified ${required.role} execution runtime ${required.version}`); + } + requireValue(identity.compilerRuntime.name === "Microsoft.NETCore.App" && + identity.compilerRuntime.version === toolchain.bundledRuntime.version, "wrong bundled compiler execution runtime"); + return { status: "verified" }; + } catch (error) { return failure("identity", error); } +} + +async function acquireToolchain(options, injected = {}) { + const attempts = []; + let stage = "acquisition"; + try { + const deps = dependencies(options, injected); + let toolchain = structuredClone(options.toolchain); + validatePlatform(toolchain.platform); + requireValue(exactVersion.test(toolchain.sdkVersion) && toolchain.sdkVersion === toolchain.toolsVersion, "inconsistent requested SDK"); + requireValue(["released", "source"].includes(toolchain.kind), "unsupported toolchain kind"); + const storage = await deps.fs.realpath(options.storageDirectory); + const work = await deps.fs.mkdtemp(path.join(storage, "toolchain-")); + const env = isolatedEnvironment(work, toolchain.platform); + for (const folder of [env.DOTNET_ROOT, env.HOME, env.TEMP, env.NUGET_PACKAGES, env.NUGET_HTTP_CACHE_PATH, env.APPDATA, env.LOCALAPPDATA]) { + await deps.fs.mkdir(folder, { recursive: true }); + } + const root = await deps.fs.realpath(env.DOTNET_ROOT); + toolchain.installationRoot = root; + const sampleDirectory = path.join(work, "sample"); + await deps.fs.mkdir(sampleDirectory); + await deps.fs.writeFile(path.join(sampleDirectory, "global.json"), JSON.stringify({ + sdk: { version: toolchain.sdkVersion, rollForward: "disable", allowPrerelease: toolchain.kind === "source" }, + }), { flag: "wx" }); + const nugetConfig = path.join(toolchain.platform.os === "windows" ? env.APPDATA : path.join(env.HOME, ".nuget"), "NuGet"); + await deps.fs.mkdir(nugetConfig, { recursive: true }); + await deps.fs.writeFile(path.join(nugetConfig, "NuGet.Config"), ""); + if (toolchain.kind === "source") { + requireValue(hash(await deps.fs.readFile(toolchain.globalJson.path)) === toolchain.globalJson.sha256, "historical global.json changed"); + for (const sdkPath of toolchain.sdkSelection.paths ?? []) { + if (sdkPath === "$host$") continue; + requireValue(typeof sdkPath === "string" && !path.isAbsolute(sdkPath) && + inside(toolchain.sourceDirectory, path.resolve(toolchain.sourceDirectory, sdkPath)), "unsupported historical SDK search path"); + try { + const actual = await deps.fs.realpath(path.resolve(toolchain.sourceDirectory, sdkPath)); + requireValue(actual === root, `historical SDK search path is not isolated: ${actual}`); + } catch (error) { + if (error.code !== "ENOENT") throw error; + } + } + stage = "sdk"; + const exact = await exactArchive(toolchain, "sdk", toolchain.sdkVersion, options, deps, attempts); + toolchain = { ...toolchain, ...exact }; + } + publicUrl(toolchain.archive.url, options.policy); + const installer = deps.installer ?? require("./toolchain-installer.cjs").installArchive; + const installed = []; + const install = async (product, version, archive) => { + const result = await installer({ + helperRoot: path.join(options.helperRoot ?? (toolchain.kind === "source" ? toolchain.sourceDirectory : path.resolve(__dirname, "..", "..")), + "eng", "common"), + root, archivePath: path.join(work, `${product}-${version}${toolchain.platform.os === "windows" ? ".zip" : ".tar.gz"}`), + product, version, platform: toolchain.platform, archive, env, cwd: sampleDirectory, + }, deps); + attempts.push(...result.attempts); + requireValue(result.archive.actualSha512 === archive.sha512 && result.archive.url === archive.url, "installer archive checksum or URL mismatch"); + validateResponse(archive.url, { url: result.archive.redirectUrl, redirects: result.archive.redirects }, options.policy); + installed.push({ product, version, ...result }); + }; + stage = "sdk"; + await install("sdk", toolchain.sdkVersion, toolchain.archive); + // An unreleased SDK's runtime version is obtained from its verified archive. + if (!toolchain.bundledRuntime?.version) { + const config = JSON.parse(await deps.fs.readFile(path.join(root, "sdk", toolchain.sdkVersion, "FSharp", "fsi.runtimeconfig.json"), "utf8")); + const framework = config.runtimeOptions?.framework; + requireValue(framework?.name === "Microsoft.NETCore.App" && exactVersion.test(framework.version), "unsupported historical bundled runtime"); + const versions = await deps.fs.readdir(path.join(root, "shared", framework.name)); + requireValue(versions.length === 1 && exactVersion.test(versions[0]), "ambiguous historical bundled runtime"); + toolchain.bundledRuntime = { version: versions[0] }; + } + stage = "runtime"; + for (const required of toolchain.runtimeRequirements) { + const location = path.join(root, "shared", required.name, required.version); + try { await deps.fs.access(location); } catch (error) { + if (error.code !== "ENOENT") throw error; + const product = runtimeProducts[required.name]; + requireValue(product && exactVersion.test(required.version), "unsupported explicit runtime requirement"); + const exact = await exactArchive(toolchain, product, required.version, options, deps, attempts); + await install(product, required.version, exact.archive); + } + } + stage = "identity"; + const executable = path.join(root, toolchain.platform.os === "windows" ? "dotnet.exe" : "dotnet"); + const cwd = toolchain.kind === "source" ? toolchain.sourceDirectory : sampleDirectory; + const identity = await deps.capture({ root, executable, cwd, env, toolchain }, deps); + identity.archive = installed[0].archive; + identity.installations = installed; + identity.requested = toolchain; + const verification = verifyToolchainIdentity({ policy: options.policy, toolchain, identity }); + if (verification.status !== "verified") return { ...verification, attempts }; + if (toolchain.kind === "source") { + requireValue(hash(await deps.fs.readFile(toolchain.globalJson.path)) === toolchain.globalJson.sha256, "historical global.json changed"); + } + const localCompilerRuntime = identity.runtimeExecutions.find(r => r.role === "compiler"); + if (toolchain.kind === "source" && localCompilerRuntime && identity.runtimeExecutions.some(r => r.role === "fsi")) { + identity.buildToolchain = { + sourceSha: toolchain.sourceSha, globalJsonDigest: toolchain.globalJson.sha256, + evidenceDigest: fingerprintInput(identity), requestedSdk: toolchain.sdkVersion, requestedTools: toolchain.toolsVersion, + requestedCompilerRuntime: toolchain.runtimeRequirements.find(r => r.role === "compiler" && r.name === "Microsoft.NETCore.App").version, + sdk: { version: identity.sdkVersion, sha256: identity.archive.sha256, verified: true }, + compilerRuntime: { + version: localCompilerRuntime.version, + sha256: identity.runtimes.find(r => r.name === localCompilerRuntime.name && r.version === localCompilerRuntime.version).sha256, + verified: true, + }, verified: true, supported: true, + }; + } + if (toolchain.kind === "released") { + identity.stableSdk = { + version: identity.sdkVersion, origin: "official-release-index", latest: true, verified: true, + releaseIndexUrl: indexUrl, releaseIndexDigest: toolchain.metadata.find(m => m.url === indexUrl).sha256, + archives: [{ os: toolchain.platform.os, architecture: toolchain.platform.architecture, sha256: identity.archive.sha256 }], + }; + } + const evidencePath = path.join(work, "identity.json"); + await deps.fs.writeFile(evidencePath, JSON.stringify({ identity, attempts }), { flag: "wx" }); + return { status: "acquired", identity, evidencePath, launch: { executable, cwd, env, sampleDirectory }, attempts }; + } catch (error) { return failure(stage, error, [...attempts, ...(error.attempts ?? [])]); } +} + +module.exports = { resolveLatestStableSdk, readRevisionToolchain, acquireToolchain, verifyToolchainIdentity }; diff --git a/eng/regression-bisect/toolchains.smoke.cjs b/eng/regression-bisect/toolchains.smoke.cjs new file mode 100644 index 00000000000..93034203d05 --- /dev/null +++ b/eng/regression-bisect/toolchains.smoke.cjs @@ -0,0 +1,54 @@ +const fs = require("node:fs/promises"); +const path = require("node:path"); +const { resolveLatestStableSdk, acquireToolchain } = require("./toolchains.cjs"); + +async function main() { + const args = process.argv.slice(2); + if (args.length !== 3 || args[0] !== "--run" || args[1] !== "--storage") { + throw new Error("Opt in with: node eng\\regression-bisect\\toolchains.smoke.cjs --run --storage "); + } + const storage = await fs.realpath(args[2]); + const temporary = await fs.mkdtemp(path.join(storage, "fsharp-sdk-smoke-")); + const os = { win32: "windows", linux: "linux", darwin: "macos" }[process.platform]; + const architecture = process.arch; + const platform = { os, architecture, rid: `${{ windows: "win", linux: "linux", macos: "osx" }[os]}-${architecture}` }; + const policy = JSON.parse(await fs.readFile(path.resolve(__dirname, "..", "..", ".github", "policies", "regression-bisect.json"))); + const options = { policy, platform, signal: AbortSignal.timeout(600000) }; + let report; + try { + const resolved = await resolveLatestStableSdk(options); + if (resolved.status !== "resolved") { + report = resolved; + } else { + const result = await acquireToolchain({ ...options, toolchain: resolved.toolchain, storageDirectory: temporary }); + if (result.status !== "acquired") { + report = { ...result, requestedSdk: resolved.toolchain.sdkVersion }; + } else { + const identity = result.identity; + report = { + status: result.status, requestedSdk: resolved.toolchain.sdkVersion, resolvedSdk: identity.sdkVersion, + platform, frozenAt: resolved.toolchain.frozenAt, + metadata: resolved.toolchain.metadata.map(({ bytesBase64, ...record }) => record), + archive: identity.archive, installers: identity.installations.map(i => i.installer), + hostPath: identity.hostPath, hostSha256: identity.hostSha256, + architecture: identity.architecture, rid: identity.rid, runtimes: identity.runtimes, + compilerRuntime: identity.compilerRuntime, files: identity.files, evidence: identity.evidence, + }; + } + } + } finally { + // Only the uniquely created child is ours, never the caller's storage root. + const real = await fs.realpath(temporary); + if (path.dirname(real) !== storage || !path.basename(real).startsWith("fsharp-sdk-smoke-")) { + throw new Error(`refusing cleanup of unexpected storage: ${real}`); + } + await fs.rm(real, { recursive: true, force: true }); + } + console.log(JSON.stringify({ ...report, cleaned: true }, null, 2)); + process.exitCode = report.status === "acquired" ? 0 : 2; +} + +main().catch(error => { + console.error(JSON.stringify({ status: "untestable", error: error.message })); + process.exitCode = 2; +}); diff --git a/eng/regression-bisect/toolchains.test.cjs b/eng/regression-bisect/toolchains.test.cjs new file mode 100644 index 00000000000..bfac8ab8ae4 --- /dev/null +++ b/eng/regression-bisect/toolchains.test.cjs @@ -0,0 +1,535 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs/promises"); +const os = require("node:os"); +const path = require("node:path"); +const { createHash } = require("node:crypto"); +const { resolveLatestStableSdk, readRevisionToolchain, acquireToolchain, verifyToolchainIdentity } = require("./toolchains.cjs"); +const { policy, admission, digest } = require("./fixtures/contracts.cjs"); +const { admitBisection, fingerprintInput } = require("./model.cjs"); +const { capture, publicUrl, isolatedEnvironment } = require("./toolchain-io.cjs"); + +const indexUrl = "https://builds.dotnet.microsoft.com/dotnet/release-metadata/releases-index.json"; +const feed = "https://builds.dotnet.microsoft.com/dotnet"; +const platform = { os: "windows", rid: "win-x64", architecture: "x64" }; +const hash512 = bytes => createHash("sha512").update(bytes).digest("hex"); +const archiveBytes = Buffer.from("verified SDK archive"); +const archive = (version, rid = "win-x64") => ({ + name: `dotnet-sdk-${rid}${rid.startsWith("win") ? ".zip" : ".tar.gz"}`, rid, + url: `${feed}/Sdk/${version}/dotnet-sdk-${version}-${rid}${rid.startsWith("win") ? ".zip" : ".tar.gz"}`, + hash: hash512(archiveBytes), +}); +const sdk = (version, rids = ["win-x64", "win-arm64", "linux-x64", "osx-arm64"]) => ({ + version, "runtime-version": `${version.split(".")[0]}.0.0`, files: rids.map(rid => archive(version, rid)), +}); +const release = (version, date = "2026-09-08", others = []) => ({ + "release-date": date, sdk: sdk(version), sdks: others.map(v => sdk(v)), + runtime: { version: `${version.split(".")[0]}.0.0`, files: [] }, +}); + +function metadata(channels = [ + ["11.0", "sts", [release("11.0.100-rc.1.26425.128")]], + ["10.0", "lts", [release("10.0.401", "2026-09-08", ["10.0.108", "10.0.401"]) ]], + ["9.0", "sts", [release("9.0.999")]], +]) { + const docs = new Map(); + docs.set(indexUrl, { + "releases-index": channels.map(([channel, type, releases]) => { + const url = `${feed}/release-metadata/${channel}/releases.json`; + docs.set(url, { "channel-version": channel, releases }); + return { "channel-version": channel, "release-type": type, "releases.json": url }; + }), + }); + return { + docs, + clock: () => new Date("2026-09-19T00:00:00Z"), + async http(url) { + assert.ok(docs.has(url), `unexpected HTTP ${url}`); + return { status: 200, url, redirects: [], bytes: Buffer.from(JSON.stringify(docs.get(url))) }; + }, + }; +} +const options = (p = platform) => ({ policy: policy(), platform: p }); + +for (const [name, channels, expected] of [ + ["preview first and multiple SDKs", undefined, "10.0.401"], + ["stable STS newer than LTS", [["10.0", "lts", [release("10.0.401")]], ["11.0", "sts", [release("11.0.100")]]], "11.0.100"], + ["numeric not lexical", [["9.0", "sts", [release("9.0.999")]], ["10.0", "lts", [release("10.0.99"), release("10.0.100")]]], "10.0.100"], + ["sdks exceeds singular SDK", [["10.0", "lts", [release("10.0.100", "2026-09-08", ["10.0.401"])]]], "10.0.401"], + ["future release excluded", [["10.0", "lts", [release("10.0.402", "2026-09-20"), release("10.0.401")]]], "10.0.401"], + ["reordered metadata", [["9.0", "sts", [release("9.0.999")]], ["10.0", "lts", [release("10.0.401")]], ["11.0", "sts", [release("11.0.100-rc.1")]]], "10.0.401"], +]) { + test(`stable resolution: ${name}`, async () => { + const deps = metadata(channels); + const result = await resolveLatestStableSdk(options(), deps); + assert.equal(result.status, "resolved", JSON.stringify(result)); + assert.equal(result.toolchain.sdkVersion, expected); + assert.equal(result.toolchain.archive.sha512, hash512(archiveBytes)); + for (const record of result.toolchain.metadata) { + assert.equal(digest(Buffer.from(record.bytesBase64, "base64")), record.sha256); + } + assert.equal(result.toolchain.frozenAt, "2026-09-19T00:00:00.000Z"); + }); +} + +for (const [p, suffix] of [ + [platform, "win-x64.zip"], + [{ os: "windows", rid: "win-arm64", architecture: "arm64" }, "win-arm64.zip"], + [{ os: "linux", rid: "linux-x64", architecture: "x64" }, "linux-x64.tar.gz"], + [{ os: "macos", rid: "osx-arm64", architecture: "arm64" }, "osx-arm64.tar.gz"], +]) { + test(`exact archive: ${p.rid}`, async () => { + const deps = metadata(); + const result = await resolveLatestStableSdk(options(p), deps); + assert.equal(result.status, "resolved", JSON.stringify(result)); + assert.ok(result.toolchain.archive.url.endsWith(suffix)); + }); +} + +for (const [name, mutate] of [ + ["missing archive", d => d.docs.get(`${feed}/release-metadata/10.0/releases.json`).releases[0].sdks = []], + ["malformed checksum", d => { + const r = d.docs.get(`${feed}/release-metadata/10.0/releases.json`).releases[0]; + r.sdk.files[0].hash = "not a hash"; r.sdks = []; + }], + ["unapproved archive host", d => { + const r = d.docs.get(`${feed}/release-metadata/10.0/releases.json`).releases[0]; + r.sdk.files[0].url = "https://unapproved.example/sdk.zip"; r.sdks = []; + }], + ["outage", d => d.http = async () => { throw new Error("HTTP 503"); }], + ["redirect violation", d => d.http = async url => ({ + status: 200, url: "https://unapproved.example/index", redirects: [], bytes: Buffer.from(JSON.stringify(d.docs.get(url))), + })], +]) { + test(`resolution fails closed: ${name}`, async () => { + const deps = metadata(); + mutate(deps); + if (name === "missing archive") deps.docs.get(`${feed}/release-metadata/10.0/releases.json`).releases[0].sdk.files = [ + { ...archive("10.0.401"), name: "dotnet-sdk-win-x64.exe", url: `${feed}/Sdk/10.0.401/installer.exe` }, + ]; + const result = await resolveLatestStableSdk(options(), deps); + assert.equal(result.status, "untestable"); + assert.equal(result.outcome.outcome, "untestable"); + assert.ok(result.error.message); + }); +} + +test("architecture and RID disagreements are never substituted", async () => { + for (const p of [{ ...platform, architecture: "x86" }, { ...platform, rid: "win-arm64" }, { ...platform, os: "linux" }]) { + assert.equal((await resolveLatestStableSdk(options(p), metadata())).status, "untestable"); + } +}); + +test("duplicate SDK entries compare product identity, not Visual Studio descriptions or file ordering", async () => { + const deps = metadata([["10.0", "lts", [release("10.0.401", "2026-09-08", ["10.0.401"])]]]); + const r = deps.docs.get(`${feed}/release-metadata/10.0/releases.json`).releases[0]; + r.sdk["vs-version"] = "10.0.0"; + r.sdks[0]["vs-version"] = "18.0.0"; + r.sdks[0].files.reverse(); + assert.equal((await resolveLatestStableSdk(options(), deps)).status, "resolved"); + r.sdks[0].files.find(f => f.rid === "win-x64").hash = "0".repeat(128); + assert.equal((await resolveLatestStableSdk(options(), deps)).status, "untestable"); +}); + +test("obsolete repeated SDK entries cannot make a newer stable release undiscoverable", async () => { + const older = release("1.1.14", "2018-04-10", ["1.1.14"]); + older.sdks[0]["runtime-version"] = "1.0.16"; + const result = await resolveLatestStableSdk(options(), + metadata([["10.0", "lts", [release("10.0.401")]], ["1.0", "lts", [older]]])); + assert.equal(result.status, "resolved", JSON.stringify(result)); + assert.equal(result.toolchain.sdkVersion, "10.0.401"); +}); + +async function directory(t) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "fsharp-toolchain-test-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + return fs.realpath(root); +} +async function source(t, version = "10.0.401", tools = version) { + const root = await directory(t); + const bytes = Buffer.from(JSON.stringify({ + sdk: { version, rollForward: "latestMinor", allowPrerelease: true, paths: [".dotnet", "$host$"] }, + tools: { dotnet: tools }, + }, null, 2)); + await fs.writeFile(path.join(root, "global.json"), bytes); + const result = await readRevisionToolchain({ + ...options(), sourceDirectory: root, sourceSha: "a".repeat(40), runtimeConfigs: [], + }); + return { root, bytes, result }; +} + +test("historical declarations need no modern TFM file and remain byte-for-byte intact", async t => { + const { root, bytes, result } = await source(t, "8.0.100-preview.1.23115.2"); + assert.equal(result.status, "resolved", JSON.stringify(result)); + assert.equal(result.toolchain.sdkVersion, "8.0.100-preview.1.23115.2"); + assert.equal(result.toolchain.toolsVersion, result.toolchain.sdkVersion); + assert.equal(result.toolchain.declarations.targetFrameworks, null); + assert.deepEqual(await fs.readFile(path.join(root, "global.json")), bytes); +}); + +test("different sdk and tools declarations are preserved, not silently reconciled", async t => { + const { result } = await source(t, "9.0.100", "8.0.100"); + assert.equal(result.status, "untestable"); + assert.match(result.error.message, /9\.0\.100.*8\.0\.100/); +}); + +test("runtimeconfig requirements retain exact identities and roles", async t => { + const { root } = await source(t); + const config = path.join(root, "fsc.runtimeconfig.json"); + await fs.writeFile(config, JSON.stringify({ runtimeOptions: { framework: { name: "Microsoft.NETCore.App", version: "9.0.8" } } })); + const result = await readRevisionToolchain({ + ...options(), sourceDirectory: root, sourceSha: "b".repeat(40), runtimeConfigs: [{ path: config, role: "compiler" }], + }); + assert.equal(result.status, "resolved", JSON.stringify(result)); + assert.deepEqual(result.toolchain.runtimeRequirements.map(({ name, version, role }) => ({ name, version, role })), + [{ name: "Microsoft.NETCore.App", version: "9.0.8", role: "compiler" }]); +}); + +for (const [declarations, expected] of [ + [{ dotnet: ["9.0.8"], "aspnetcore/x64": ["9.0.8"] }, "resolved"], + [{ "dotnet/arm64": ["9.0.8"] }, "untestable"], + [{ dotnet: ["$(RuntimeVersion)"] }, "untestable"], +]) { + test(`historical tools.runtimes: ${JSON.stringify(declarations)}`, async t => { + const { root } = await source(t); + const file = path.join(root, "global.json"); + const content = JSON.parse(await fs.readFile(file)); + content.tools.runtimes = declarations; + await fs.writeFile(file, JSON.stringify(content)); + const result = await readRevisionToolchain({ ...options(), sourceDirectory: root, sourceSha: "a".repeat(40) }); + assert.equal(result.status, expected, JSON.stringify(result)); + if (expected === "resolved") { + assert.deepEqual(result.toolchain.runtimeRequirements.map(r => [r.name, r.version, r.role]), + [["Microsoft.NETCore.App", "9.0.8", "build"], ["Microsoft.AspNetCore.App", "9.0.8", "build"]]); + } + }); +} + +function captured(toolchain, root) { + return { + sdkVersion: toolchain.sdkVersion, architecture: platform.architecture, rid: platform.rid, + hostPath: path.join(root, "dotnet.exe"), root, sdkBasePath: path.join(root, "sdk", toolchain.sdkVersion), + sdks: [{ version: toolchain.sdkVersion, path: path.join(root, "sdk") }], + runtimes: [{ name: "Microsoft.NETCore.App", version: "10.0.0", path: path.join(root, "shared", "Microsoft.NETCore.App"), sha256: digest("runtime") }], + compilerRuntime: { name: "Microsoft.NETCore.App", version: "10.0.0" }, + runtimeExecutions: [], + files: Object.fromEntries(Object.entries({ compiler: "fsc", fsi: "fsi", core: "FSharp.Core" }).map(([name, assemblyName]) => [name, { + path: path.join(root, "sdk", toolchain.sdkVersion, "FSharp", `${assemblyName}.dll`), + sha256: digest(name), assemblyName, assemblyVersion: "10.0.0.0", fileVersion: "10.0.0.0", + }])), + archive: { ...toolchain.archive, sha256: digest(archiveBytes), actualSha512: hash512(archiveBytes), redirectUrl: toolchain.archive.url }, + }; +} + +for (const [name, edit] of [ + ["wrong architecture", i => i.architecture = "arm64"], + ["wrong resolved SDK", i => i.sdkVersion = "11.0.100-rc.1"], + ["host SDK leakage", i => i.sdks.push({ version: "11.0.100", path: path.resolve("host-sdk") })], + ["wrong host", i => i.hostPath = path.resolve("machine", "dotnet.exe")], + ["self-asserted host root", i => { i.root = path.resolve("machine"); i.hostPath = path.join(i.root, "dotnet.exe"); }], + ["same-version SDK outside root", i => i.sdkBasePath = path.resolve("machine", "sdk", "10.0.401")], + ["archive mismatch", i => i.archive.actualSha512 = "0".repeat(128)], + ["wrong archive URL", i => i.archive.url = `${feed}/Sdk/wrong.zip`], + ["unapproved redirect destination", i => i.archive.redirectUrl = "https://unapproved.example/sdk.zip"], + ["missing runtime", i => i.runtimes = []], + ["wrong compiler runtime", i => i.compilerRuntime.version = "9.0.0"], + ["outside Core path", i => i.files.core.path = path.resolve("host-core.dll")], + ["wrong F# assembly", i => i.files.core.assemblyName = "Other.Assembly"], +]) { + test(`identity fails closed: ${name}`, async () => { + const { toolchain } = await resolveLatestStableSdk(options(), metadata()); + const identity = captured(toolchain, path.resolve("isolated")); + toolchain.installationRoot = identity.root; + edit(identity); + const result = verifyToolchainIdentity({ toolchain, identity, policy: policy() }); + assert.equal(result.status, "untestable", JSON.stringify(result)); + assert.equal(result.outcome.reason, "wrong-identity"); + }); +} + +test("exact acquisition sanitizes state and never calls a PATH-selected SDK", async t => { + const root = await directory(t); + await fs.writeFile(path.join(root, "global.json"), JSON.stringify({ sdk: { version: "11.0.100-rc.1.26425.128" } })); + for (const key of ["runtime_source_feed_key", "GITHUB_TOKEN", "NUGET_AUTH_TOKEN", "MSBuildSDKsPath"]) { + const previous = process.env[key]; + process.env[key] = "must-not-inherit"; + t.after(() => { if (previous === undefined) delete process.env[key]; else process.env[key] = previous; }); + } + const { toolchain } = await resolveLatestStableSdk(options(), metadata()); + const calls = []; + const result = await acquireToolchain({ + policy: policy(), toolchain, storageDirectory: root, helperRoot: path.resolve("."), + }, { + ...metadata(), + installer: async request => { + assert.equal(request.version, "10.0.401"); + for (const key of ["runtime_source_feed_key", "GITHUB_TOKEN", "NUGET_AUTH_TOKEN", "MSBuildSDKsPath"]) { + assert.equal(request.env[key], undefined); + } + assert.ok(request.env.DOTNET_ROOT.startsWith(root)); + assert.ok(request.env.NUGET_PACKAGES.startsWith(root)); + assert.ok(request.env.DOTNET_CLI_HOME.startsWith(root)); + assert.equal(request.env.DOTNET_MULTILEVEL_LOOKUP, "0"); + calls.push(request); + return { archive: captured(toolchain, root).archive, installer: { sha256: digest("script") }, attempts: [] }; + }, + capture: async request => { + assert.equal(request.executable, path.join(request.root, "dotnet.exe")); + const global = JSON.parse(await fs.readFile(path.join(request.cwd, "global.json"))); + assert.deepEqual(global.sdk, { version: "10.0.401", rollForward: "disable", allowPrerelease: false }); + return captured(toolchain, request.root); + }, + }); + assert.equal(result.status, "acquired", JSON.stringify(result)); + assert.equal(calls.length, 1); + assert.equal(result.identity.sdkVersion, "10.0.401"); +}); + +test("two acquisitions with the same versions never share CLI or NuGet state", async t => { + const root = await directory(t); + const { toolchain } = await resolveLatestStableSdk(options(), metadata()); + const environments = []; + const deps = { + installer: async request => { + environments.push(request.env); + return { archive: captured(toolchain, root).archive, installer: {}, attempts: [] }; + }, + capture: async request => captured(toolchain, request.root), + }; + for (let i = 0; i < 2; i++) { + assert.equal((await acquireToolchain({ policy: policy(), toolchain, storageDirectory: root }, deps)).status, "acquired"); + } + for (const key of ["DOTNET_ROOT", "NUGET_PACKAGES", "DOTNET_CLI_HOME", "HOME"]) { + assert.notEqual(environments[0][key], environments[1][key]); + } +}); + +test("unavailable historical SDK records public fallback attempts without running a host SDK", async t => { + const { result: { toolchain } } = await source(t, "8.0.100-preview.1.12345.6"); + const urls = []; + const result = await acquireToolchain({ policy: policy(), toolchain, storageDirectory: await directory(t) }, { + ...metadata(), + http: async url => { + urls.push(url); + if (url.endsWith(".sha512")) return { status: 404, url, redirects: [], bytes: Buffer.from("unavailable") }; + return metadata().http(url); + }, + installer: async () => { assert.fail("unverified archive must never reach installer"); }, + capture: async () => { assert.fail("no host SDK fallback"); }, + }); + assert.equal(result.status, "untestable"); + assert.ok(urls.some(url => url.startsWith("https://ci.dot.net/public/"))); + assert.ok(result.attempts.some(a => a.status === 404)); +}); + +test("different verified source declarations compose with invariant reproduction identities", () => { + const a = admission(); + for (const [o, version] of [[a.main, "11.0.100-rc.1.26425.128"], [a.good, "9.0.100"]]) { + o.cells[0].buildToolchain = { + sourceSha: o.target.sourceSha, globalJsonDigest: digest(version), evidenceDigest: digest(`evidence-${version}`), + requestedSdk: version, requestedTools: version, + requestedCompilerRuntime: `${version.split(".")[0]}.0.0`, + sdk: { version, sha256: digest(version), verified: true }, + compilerRuntime: { version: `${version.split(".")[0]}.0.0`, sha256: digest(`runtime-${version}`), verified: true }, + verified: true, supported: true, + }; + } + a.budget.authenticatedObservationDigests = [a.stable, a.main, a.good].map(fingerprintInput); + const result = admitBisection(a); + assert.equal(result.decision, "admit", JSON.stringify(result)); + assert.equal(result.toolchainTransitions.length, 1); + for (const patch of [ + b => b.requestedSdk = "8.0.100", + b => b.sourceSha = "0".repeat(40), + b => b.verified = false, + b => b.supported = false, + b => b.compilerRuntime.version = "7.0.0", + ]) { + const invalid = structuredClone(a); + patch(invalid.good.cells[0].buildToolchain); + invalid.budget.authenticatedObservationDigests = [invalid.stable, invalid.main, invalid.good].map(fingerprintInput); + assert.equal(admitBisection(invalid).decision, "blocked"); + } + const wrongRuntime = structuredClone(a); + wrongRuntime.good.cells[0].runtime.version = "7.0.0"; + wrongRuntime.budget.authenticatedObservationDigests = [wrongRuntime.stable, wrongRuntime.main, wrongRuntime.good].map(fingerprintInput); + assert.equal(admitBisection(wrongRuntime).decision, "blocked"); +}); + +for (const location of [ + "https://ci.dot.net/public/../../private/sdk.zip", + "https://ci.dot.net/public/%2e%2e/private/sdk.zip", + "https://builds.dotnet.microsoft.com/dotnet/sdk.zip?token=private", + "https://builds.dotnet.microsoft.com.evil.test/dotnet/sdk.zip", + "http://builds.dotnet.microsoft.com/dotnet/sdk.zip", +]) { + test(`public feed rejects ${location}`, () => assert.throws(() => publicUrl(location, policy()), /unapproved/)); +} + +test("bad archive aborts acquisition before any acquired executable runs", async t => { + const { toolchain } = await resolveLatestStableSdk(options(), metadata()); + const result = await acquireToolchain({ policy: policy(), toolchain, storageDirectory: await directory(t) }, { + installer: async () => ({ archive: { ...toolchain.archive, actualSha512: "0".repeat(128) }, attempts: [] }), + capture: () => assert.fail("acquired executable must not run before archive validation"), + }); + assert.equal(result.status, "untestable"); + assert.match(result.error.message, /checksum/); +}); + +test("existing historical SDK search directories cannot leak a same-version machine SDK", async t => { + const { root, result: { toolchain }, bytes } = await source(t); + await fs.mkdir(path.join(root, ".dotnet")); + const result = await acquireToolchain({ policy: policy(), toolchain, storageDirectory: await directory(t) }, { + installer: () => assert.fail("source search path must be checked before installation"), + }); + assert.equal(result.status, "untestable"); + assert.match(result.error.message, /not isolated/); + assert.deepEqual(await fs.readFile(path.join(root, "global.json")), bytes); +}); + +for (const available of [true, false]) { + test(`explicit supplemental runtime ${available ? "acquired and separated" : "unavailable is untestable"}`, async t => { + const { root, bytes, result: initial } = await source(t); + const sampleConfig = path.join(root, "sample.runtimeconfig.json"); + await fs.writeFile(sampleConfig, JSON.stringify({ + runtimeOptions: { framework: { name: "Microsoft.NETCore.App", version: "9.0.8" } }, + })); + const { toolchain } = await readRevisionToolchain({ + ...options(), sourceDirectory: root, sourceSha: initial.toolchain.sourceSha, runtimeConfigs: [{ path: sampleConfig, role: "sample" }], + }); + const deps = metadata(); + const runtimeRelease = deps.docs.get(`${feed}/release-metadata/9.0/releases.json`).releases[0]; + runtimeRelease.runtime = { + version: "9.0.8", files: [{ ...archive("9.0.8"), name: "dotnet-runtime-win-x64.zip", + url: `${feed}/Runtime/9.0.8/dotnet-runtime-9.0.8-win-x64.zip` }], + }; + const products = []; + const result = await acquireToolchain({ policy: policy(), toolchain, storageDirectory: await directory(t) }, { + ...deps, + installer: async request => { + products.push(request.product); + if (request.product === "dotnet" && !available) throw new Error("runtime archive HTTP 404"); + return { + archive: { ...request.archive, actualSha512: hash512(archiveBytes), sha256: digest(archiveBytes), redirectUrl: request.archive.url }, + installer: {}, attempts: [{ url: request.archive.url, version: request.version, status: 200 }], + }; + }, + capture: async request => { + const identity = captured(request.toolchain, request.root); + identity.runtimes.push({ + name: "Microsoft.NETCore.App", version: "9.0.8", + path: path.join(request.root, "shared", "Microsoft.NETCore.App"), sha256: digest("9.0.8"), + }); + return identity; + }, + }); + assert.deepEqual(products, ["sdk", "dotnet"]); + assert.deepEqual(await fs.readFile(path.join(root, "global.json")), bytes); + if (available) { + assert.equal(result.status, "acquired", JSON.stringify(result)); + assert.equal(result.identity.compilerRuntime.version, "10.0.0"); + assert.equal(result.identity.requested.runtimeRequirements[0].version, "9.0.8"); + assert.equal(result.identity.buildToolchain, undefined, "SDK-only acquisition cannot attest a not-yet-built local compiler"); + } else { + assert.equal(result.status, "untestable"); + assert.equal(result.outcome.reason, "unavailable-runtime"); + } + }); +} + +test("historical public preview fallback preserves checksum provenance and exact requested version", async t => { + const { result: { toolchain } } = await source(t, "8.0.100-preview.1.12345.6"); + const calls = []; + const result = await acquireToolchain({ policy: policy(), toolchain, storageDirectory: await directory(t) }, { + ...metadata(), + http: async url => { + if (!url.endsWith(".sha512")) return metadata().http(url); + return { status: url.startsWith("https://ci.dot.net/") ? 200 : 404, url, redirects: [], bytes: Buffer.from(hash512(archiveBytes)) }; + }, + installer: async request => { + calls.push(request); + const fsharp = path.join(request.root, "sdk", request.version, "FSharp"); + await fs.mkdir(fsharp, { recursive: true }); + await fs.writeFile(path.join(fsharp, "fsi.runtimeconfig.json"), JSON.stringify({ + runtimeOptions: { framework: { name: "Microsoft.NETCore.App", version: "8.0.0-preview.1.1" } }, + })); + await fs.mkdir(path.join(request.root, "shared", "Microsoft.NETCore.App", "8.0.0-preview.1.1"), { recursive: true }); + return { + archive: { ...request.archive, actualSha512: hash512(archiveBytes), sha256: digest(archiveBytes), redirectUrl: request.archive.url }, + installer: {}, attempts: [], + }; + }, + capture: async request => { + const identity = captured(request.toolchain, request.root); + identity.compilerRuntime.version = identity.runtimes[0].version = "8.0.0-preview.1.1"; + return identity; + }, + }); + assert.equal(result.status, "acquired", JSON.stringify(result)); + assert.equal(calls[0].version, toolchain.sdkVersion); + assert.ok(calls[0].archive.url.startsWith("https://ci.dot.net/public/")); + assert.deepEqual(result.attempts.filter(a => a.url.endsWith(".sha512")).map(a => a.status), [404, 200]); +}); + +for (const [name, packaged, localCompiler] of [ + ["a bundled nuspec", true, false], + ["no NuGet library in FSI deps", false, false], + ["local compiler runtime traces", true, true], +]) { + test(`captured identity with ${name}`, async t => { + const root = await directory(t); + const env = isolatedEnvironment(root, platform); + await fs.mkdir(env.HOME); + const fsharp = path.join(root, "sdk", "10.0.401", "FSharp"); + await fs.mkdir(fsharp, { recursive: true }); + for (const name of ["fsc.dll", "fsi.dll", "FSharp.Core.dll"]) await fs.writeFile(path.join(fsharp, name), name); + await fs.writeFile(path.join(fsharp, "fsi.deps.json"), JSON.stringify({ libraries: { "fsi/1.0.0": { type: "project" } } })); + const configBytes = JSON.stringify({ + runtimeOptions: { framework: { name: "Microsoft.NETCore.App", version: "10.0.0" } }, + }); + await fs.writeFile(path.join(fsharp, "fsi.runtimeconfig.json"), configBytes); + await fs.writeFile(path.join(fsharp, "fsc.runtimeconfig.json"), configBytes); + if (packaged) { + await fs.mkdir(path.join(fsharp, "library-packs")); + await fs.writeFile(path.join(fsharp, "library-packs", "FSharp.Core.10.1.401.nupkg"), "package"); + } + const runtime = path.join(root, "shared", "Microsoft.NETCore.App"); + await fs.mkdir(path.join(runtime, "10.0.12"), { recursive: true }); + await fs.writeFile(path.join(runtime, "10.0.12", "System.Private.CoreLib.dll"), "runtime"); + const executable = path.join(root, "dotnet.exe"); + await fs.writeFile(executable, "host"); + const runtimeRequirements = localCompiler ? [["compiler", "fsc"], ["fsi", "fsi"]].map(([role, file]) => ({ + name: "Microsoft.NETCore.App", version: "10.0.12", role, + config: { path: path.join(fsharp, `${file}.runtimeconfig.json`), sha256: digest(configBytes) }, + })) : []; + const identity = await capture({ + root, env, executable, cwd: root, + toolchain: { sourceDirectory: root, sdkVersion: "10.0.401", bundledRuntime: { version: "10.0.12" }, runtimeRequirements }, + }, { + fs, + run: async (file, args, settings) => { + assert.equal(file, executable); + if (args.at(-1) === "--help") { + assert.equal(args[2], "10.0.12"); + assert.equal(settings.env.DOTNET_ROLL_FORWARD, "Disable"); + await fs.writeFile(settings.env.DOTNET_HOST_TRACEFILE, `Chose FX version [${path.join(runtime, "10.0.12")}]`); + return { exitCode: 0, stderr: "", stdout: "F# compiler help" }; + } + const outputs = { + "--info": `Base Path: ${path.join(root, "sdk", "10.0.401")}\nRID: win-x64`, + "--version": "10.0.401", + "--list-sdks": `10.0.401 [${path.join(root, "sdk")}]`, + "--list-runtimes": `Microsoft.NETCore.App 10.0.12 [${runtime}]`, + exec: "ARCH|x64\nRUNTIME|10.0.12\nFILE|fsc.dll|fsc|15.2.401.0|15.204.126.42413\n" + + "FILE|fsi.dll|fsi|15.2.401.0|15.204.126.42413\nFILE|FSharp.Core.dll|FSharp.Core|10.1.0.0|10.104.126.42413\n" + + (packaged ? "PACKAGE|FSharp.Core.10.1.401.nupkg|FSharp.Core|10.1.401" : ""), + }; + return { exitCode: 0, stderr: "", stdout: outputs[args[0]] }; + }, + }); + assert.equal(identity.files.core.packageIdentity, packaged ? "FSharp.Core/10.1.401" : null); + assert.equal(identity.files.core.assemblyVersion, "10.1.0.0"); + assert.equal(identity.compilerRuntime.version, "10.0.12"); + assert.deepEqual(identity.runtimeExecutions.map(r => [r.role, r.version]), localCompiler ? [["compiler", "10.0.12"], ["fsi", "10.0.12"]] : []); + }); +} From 6db0b07c1152b3b76f80bfcc6d95dacd49353b41 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 02:37:39 +0200 Subject: [PATCH 11/99] Fix historical toolchain RID and execution evidence Preserve musl installer OS selection, verify versioned host RIDs against the acquired SDK graph, and isolate each runtime trace. Bind execution evidence to config paths as well as digests and exercise acquisition-generated admission records. Validation: 538 Node tests, scoped syntax and diff checks, and real Windows SDK 10.0.401 acquisition with runtime 10.0.12. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 33 ++- eng/regression-bisect/toolchain-installer.cjs | 5 +- .../toolchain-installer.test.cjs | 13 +- eng/regression-bisect/toolchain-io.cjs | 11 +- eng/regression-bisect/toolchains.cjs | 33 ++- eng/regression-bisect/toolchains.test.cjs | 271 ++++++++++++------ 6 files changed, 261 insertions(+), 105 deletions(-) diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index 5a6a27218d0..5e30ff7e1e3 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -192,6 +192,12 @@ validated maintainer `policy`. Resolution and acquisition also require `platform: { os, rid, architecture }`, either directly or in the frozen toolchain. Supported architectures are `x64` and `arm64`; RIDs are `win-*`, `linux-*`, `linux-musl-*`, or `osx-*`, matching that OS and architecture. +These are archive RIDs. A historical source SDK can report a versioned host RID +such as `win10-x64` or `ubuntu.22.04-x64`. Its acquired +`RuntimeIdentifierGraph.json` must prove compatibility with the requested +portable RID; the raw graph bytes/hash and actual host RID remain in the +evidence. Traversal stops at portable OS/ABI boundaries, so musl and glibc are +never interchangeable. Missing or inconsistent graph evidence is untestable. | Operation | Result | | --- | --- | @@ -224,7 +230,9 @@ the caller's secrets, private feeds, credential providers or user profile. `toolchain-installer.cjs` calls the existing Arcade `GetDotNetInstallScript` helper with a freshly verified official-script cache. It verifies actual script parameter support, then invokes the official installer with exact -version/architecture and archive-retention flags. A loopback-only, bounded +version/architecture and archive-retention flags. Linux-musl archive requests +use the official `--os linux-musl` option rather than dropping the ABI from the +RID. A loopback-only, bounded feed serves the already downloaded and SHA-512-verified archive to that installer. This transport boundary records the **actual public download's** redirects/digests without reimplementing extraction or trusting a separate HEAD @@ -248,6 +256,9 @@ archive verification. SDK-bundled runtime, supplemental sample requirements, and local compiler execution runtimes are separate records. Supplied local compiler/FSI runtimeconfigs are verified using the corresponding built DLL's `--help` and host trace with the exact runtime; sample code is never executed. +Each execution owns a unique trace file and binds both the runtimeconfig's +resolved path and digest, including when several configs have identical bytes +or the same compiler role. Only acquisition with both local compiler and FSI execution evidence emits the `buildToolchain` admission record. An SDK acquired before a source build does not attest the identity of a compiler that has not been built yet. @@ -264,7 +275,8 @@ its uniquely created child directory. It has a ten-minute cancellation deadline. Exit 0 means actual acquisition/identity success; exit 2 records the exact block, never mocked download success. This is not a source build or live issue run. -On 2026-09-19 at 00:02 UTC, that command completed successfully on Windows x64: +On 2026-09-19 at 00:35:31 UTC, that command resolved metadata and completed +successfully (exit 0) on Windows x64: requested/resolved SDK `10.0.401`, RID `win-x64`, bundled/executing runtime `10.0.12`, compiler and FSI assembly `15.2.401.0`, Core assembly `10.1.0.0`, and bundled package `FSharp.Core/10.1.401`. The archive's final public URL was @@ -278,11 +290,22 @@ The official PowerShell install script SHA-256 was Temporary acquisition storage was cleaned. These are recorded smoke results, not production version constants. -Validation also completed `node --check` for all seven changed/new `.cjs` +Validation also completed `node --check` for all twelve top-level `.cjs` files and the required `node --test eng\regression-bisect\model.test.cjs eng\regression-bisect\toolchains.test.cjs` -command (304 passed). The broader suite above passed all 516 tests, including -intake, authenticated state replay and installer-boundary coverage. The signed checkpoint fixture +command (322 passed), plus +`node --test eng\regression-bisect\toolchains.test.cjs` (72 passed). +The broader suite above passed all 538 tests, including intake, authenticated +state replay and installer-boundary coverage. The repair cases first reproduced +seven failures for musl installer arguments, historical host RIDs, and colliding +trace files. An additional failing case caught missing same-role execution +evidence hidden by identical config digests. The passing acquisition-to-admission +cases use captured evidence, preserve differing historical/preview build +toolchains and their coupled transitions, and reject invalid execution evidence +and incomparable reproduction runtimes. Cross-platform installer/identity cases +use injected boundaries; the actual download smoke above ran on Windows only. +`dotnet fantomas .` completed without changes or errors; working-tree and sprint +commit-range `git diff --check` checks passed. The signed checkpoint fixture was regenerated through its existing harness for the new pinned public-feed policy; an old-policy checkpoint is not silently accepted as a new-policy one. Root compiler Release builds/tests are not this Node adapter's acceptance gate. diff --git a/eng/regression-bisect/toolchain-installer.cjs b/eng/regression-bisect/toolchain-installer.cjs index 4645562a733..2248c140954 100644 --- a/eng/regression-bisect/toolchain-installer.cjs +++ b/eng/regression-bisect/toolchain-installer.cjs @@ -165,7 +165,8 @@ async function installArchive(options, deps) { const fs = deps.fs || require('node:fs/promises'); if (typeof deps.http !== 'function' || typeof deps.run !== 'function') throw new Error('Policy-constrained http and trusted run boundaries are required'); - const os = ({ win32: 'win', windows: 'win', darwin: 'osx', macos: 'osx' })[platform?.os] || platform?.os; + const os = platform?.os === 'linux' && platform.rid === `linux-musl-${platform.architecture}` ? 'linux-musl' + : ({ win32: 'win', windows: 'win', darwin: 'osx', macos: 'osx' })[platform?.os] || platform?.os; const windows = os === 'win'; if (!['win', 'linux', 'linux-musl', 'osx', 'freebsd', 'rhel.6'].includes(os)) throw new Error('Unsupported installer OS'); @@ -175,6 +176,8 @@ async function installArchive(options, deps) { throw new Error('An exact SDK/runtime version is required, not a channel or latest'); if (!['x64', 'x86', 'arm', 'arm64', 's390x', 'ppc64le', 'loongarch64', 'riscv64'].includes(platform.architecture)) throw new Error('Unsupported installer architecture'); + if (platform.rid !== `${os}-${platform.architecture}`) + throw new Error('Installer OS/RID/architecture mismatch'); if (!archive || !/^[0-9a-f]{128}$/i.test(archive.sha512)) throw new Error('Official archive SHA512 is required'); const archiveName = decodeURIComponent(publicUrl(archive.url).pathname.split('/').pop()); diff --git a/eng/regression-bisect/toolchain-installer.test.cjs b/eng/regression-bisect/toolchain-installer.test.cjs index d14b0d25ae5..eba4030a1ba 100644 --- a/eng/regression-bisect/toolchain-installer.test.cjs +++ b/eng/regression-bisect/toolchain-installer.test.cjs @@ -27,11 +27,10 @@ function request(url, method = 'GET') { }); } -async function fixture(t, { product = 'sdk', os = 'win', architecture = 'x64', version = '11.0.100-preview.3.26159.112' } = {}) { +async function fixture(t, { product = 'sdk', os = 'win', architecture = 'x64', rid = `${os}-${architecture}`, version = '11.0.100-preview.3.26159.112' } = {}) { const directory = await fs.mkdtemp(path.join(__dirname, '.installer-test-')); t.after(() => fs.rm(directory, { recursive: true, force: true })); const windows = os === 'win'; - const rid = `${os}-${architecture}`; const filename = `${product === 'sdk' ? 'dotnet-sdk' : product === 'dotnet' ? 'dotnet-runtime' : `${product}-runtime`}-${version}-${rid}.${windows ? 'zip' : 'tar.gz'}`; const helperRoot = path.join(directory, 'eng', 'common'); await fs.mkdir(helperRoot, { recursive: true }); @@ -111,13 +110,14 @@ async function fixture(t, { product = 'sdk', os = 'win', architecture = 'x64', v } else { assert.equal(executable, '/bin/bash'); assert.deepEqual(args.slice(0, 2), ['--noprofile', '--norc']); - assert.equal(flag('Os'), os); + assert.equal(flag('Os'), rid.slice(0, -(architecture.length + 1))); } state.feed = flag('AzureFeed'); assert.equal(new URL(state.feed).hostname, '127.0.0.1'); const productDir = product === 'sdk' ? 'Sdk' : product === 'aspnetcore' ? 'aspnetcore/Runtime' : product === 'windowsdesktop' && Number(version.split('.')[0]) >= 5 ? 'WindowsDesktop' : 'Runtime'; - state.archiveUrl = `${state.feed}/${productDir}/${version}/${filename}`; + const requestedName = filename.replace(rid, windows ? `win-${flag('Architecture')}` : `${flag('Os')}-${flag('Architecture')}`); + state.archiveUrl = `${state.feed}/${productDir}/${version}/${requestedName}`; const productVersion = await request(`${state.feed}/${productDir}/${version}/productVersion.txt`); assert.equal(productVersion.status, 200); assert.equal(productVersion.bytes.toString().trim(), version); @@ -135,9 +135,11 @@ for (const settings of [ { product: 'sdk' }, { product: 'dotnet' }, { product: 'aspnetcore' }, { product: 'windowsdesktop' }, { product: 'windowsdesktop', version: '3.1.32' }, { product: 'sdk', os: 'linux' }, { product: 'dotnet', os: 'linux-musl' }, + { product: 'sdk', os: 'linux', rid: 'linux-musl-x64' }, + { product: 'dotnet', os: 'linux', architecture: 'arm64', rid: 'linux-musl-arm64' }, { product: 'aspnetcore', os: 'osx' }, { product: 'sdk', architecture: 'arm64' }, ]) { - test(`installs exact ${settings.product} ${settings.os || 'win'} ${settings.architecture || 'x64'} ${settings.version || 'preview'}`, async t => { + test(`installs exact ${settings.product} ${settings.rid || settings.os || 'win'} ${settings.architecture || 'x64'} ${settings.version || 'preview'}`, async t => { const { options, deps, state, script, helperPath } = await fixture(t, settings); const result = await installArchive(options, deps); assert.equal(result.archive.sha256, digest(payload, 'sha256')); @@ -240,6 +242,7 @@ for (const failure of ['exit', 'throw', 'retained digest', 'missing archive']) { for (const mutation of [ o => { o.version = 'latest'; }, o => { o.product = 'invalid'; }, o => { o.archive.sha512 = ''; }, o => { o.platform.architecture = 'x64; bad'; }, + o => { o.platform.rid = 'win-arm64'; }, o => { o.platform.rid = 'linux-x64'; }, o => { o.env = undefined; }, o => { o.archive.url = 'http://example.com/file.zip'; }, o => { o.env.SystemRoot = 'relative'; }, ]) { diff --git a/eng/regression-bisect/toolchain-io.cjs b/eng/regression-bisect/toolchain-io.cjs index 5481db6db47..f04da0ef6d7 100644 --- a/eng/regression-bisect/toolchain-io.cjs +++ b/eng/regression-bisect/toolchain-io.cjs @@ -188,7 +188,8 @@ async function capture(request, deps) { requireValue(required.config.path.endsWith(".runtimeconfig.json"), "unsupported local compiler runtimeconfig name"); const compiler = await deps.fs.realpath(required.config.path.replace(/\.runtimeconfig\.json$/, ".dll")); requireValue(inside(toolchain.sourceDirectory, compiler), "local compiler path escapes historical source"); - const trace = path.join(env.DOTNET_CLI_HOME, `${required.role}-host-trace.txt`); + const traceDirectory = await deps.fs.mkdtemp(path.join(env.DOTNET_CLI_HOME, `${required.role}-`)); + const trace = path.join(traceDirectory, "host-trace.txt"); const result = await deps.run(executable, ["exec", "--fx-version", required.version, compiler, "--help"], { cwd, env: { ...env, COREHOST_TRACE: "1", COREHOST_TRACEFILE: trace, DOTNET_HOST_TRACE: "1", DOTNET_HOST_TRACEFILE: trace }, timeoutMs: 120000, @@ -199,7 +200,7 @@ async function capture(request, deps) { const selected = [...output.matchAll(/Chose FX version \[([^\]]+)\]/g)].map(m => path.normalize(m[1])); requireValue(selected.includes(expected), `local ${required.role} host trace did not verify ${expected}`); runtimeExecutions.push({ - role: required.role, configDigest: required.config.sha256, name: required.name, + role: required.role, configPath: required.config.path, configDigest: required.config.sha256, name: required.name, version: required.version, architecture, hostTrace: { path: trace, sha256: hash(output) }, compiler: { path: compiler, sha256: hash(await deps.fs.readFile(compiler)) }, }); @@ -223,9 +224,15 @@ async function capture(request, deps) { } const host = await fileIdentity(executable, root, deps.fs); const rid = /^\s*RID:\s*(\S+)/m.exec(info)?.[1]; + let runtimeGraph; + if (rid !== toolchain.platform.rid) { + runtimeGraph = await fileIdentity(path.join(sdkBasePath, "RuntimeIdentifierGraph.json"), root, deps.fs); + runtimeGraph.bytesBase64 = (await deps.fs.readFile(runtimeGraph.path)).toString("base64"); + } return { root, hostPath: host.path, hostSha256: host.sha256, sdkVersion, sdkBasePath, sdks, runtimes, architecture, rid, compilerRuntime, runtimeExecutions, files, depsFile, + ...(runtimeGraph ? { runtimeGraph } : {}), evidence: { info, sdkList, runtimeList, probeOutput }, }; } diff --git a/eng/regression-bisect/toolchains.cjs b/eng/regression-bisect/toolchains.cjs index b46da225c0a..17923b75217 100644 --- a/eng/regression-bisect/toolchains.cjs +++ b/eng/regression-bisect/toolchains.cjs @@ -228,6 +228,34 @@ async function exactArchive(toolchain, product, version, options, deps, attempts throw new Error(`exact ${product} ${version} unavailable from permitted public feeds`); } +function verifyHostRid(toolchain, identity) { + requireValue(identity.architecture === toolchain.platform.architecture, "resolved architecture/RID mismatch"); + if (identity.rid === toolchain.platform.rid) return; + const record = identity.runtimeGraph; + requireValue(toolchain.kind === "source" && record?.path === path.join(identity.sdkBasePath, "RuntimeIdentifierGraph.json"), + "historical RID requires the acquired SDK's runtime graph"); + const bytes = Buffer.from(record.bytesBase64, "base64"); + requireValue(hash(bytes) === record.sha256, "historical RID graph digest mismatch"); + const graph = JSON.parse(bytes).runtimes; + const pending = [identity.rid]; + const seen = new Set(); + const portable = new Set(); + for (const rid of pending) { + if (seen.has(rid)) continue; + seen.add(rid); + requireValue(typeof rid === "string", "invalid historical RID"); + if (/^(win|linux(?:-musl)?|osx)-(x64|arm64)$/.test(rid)) { + // Stop at the ABI boundary: linux-musl imports linux, but is not glibc. + portable.add(rid); + } else { + const imports = graph?.[rid]?.["#import"]; + requireValue(Array.isArray(imports), `unsupported historical RID: ${rid}`); + pending.push(...imports.filter(r => typeof r === "string" && r.endsWith(`-${toolchain.platform.architecture}`))); + } + } + requireValue(portable.size === 1 && portable.has(toolchain.platform.rid), "resolved architecture/RID mismatch"); +} + function verifyToolchainIdentity({ policy, toolchain, identity }) { try { validatePolicy(policy); @@ -238,7 +266,7 @@ function verifyToolchainIdentity({ policy, toolchain, identity }) { "wrong selected host path"); requireValue(identity.sdkVersion === toolchain.sdkVersion && toolchain.toolsVersion === toolchain.sdkVersion, "resolved SDK version mismatch"); requireValue(identity.sdkBasePath === path.join(root, "sdk", toolchain.sdkVersion), "wrong resolved SDK base path"); - requireValue(identity.architecture === toolchain.platform.architecture && identity.rid === toolchain.platform.rid, "resolved architecture/RID mismatch"); + verifyHostRid(toolchain, identity); requireValue(identity.sdks.length === 1 && identity.sdks[0].version === toolchain.sdkVersion && identity.sdks[0].path === path.join(root, "sdk"), "unexpected installed SDK or SDK location"); requireValue(identity.archive?.sha512 === toolchain.archive.sha512 && identity.archive.actualSha512 === toolchain.archive.sha512 && @@ -259,7 +287,8 @@ function verifyToolchainIdentity({ policy, toolchain, identity }) { requireValue(identity.runtimes.some(r => r.name === required.name && r.version === required.version), `missing runtime ${required.name} ${required.version}`); } for (const required of toolchain.runtimeRequirements.filter(r => r.name === "Microsoft.NETCore.App" && ["compiler", "fsi"].includes(r.role))) { - requireValue(identity.runtimeExecutions?.some(r => r.role === required.role && r.configDigest === required.config.sha256 && + requireValue(identity.runtimeExecutions?.some(r => r.role === required.role && + r.configPath === required.config.path && r.configDigest === required.config.sha256 && r.name === required.name && r.version === required.version && r.architecture === toolchain.platform.architecture), `unverified ${required.role} execution runtime ${required.version}`); } diff --git a/eng/regression-bisect/toolchains.test.cjs b/eng/regression-bisect/toolchains.test.cjs index bfac8ab8ae4..0cbfd141a03 100644 --- a/eng/regression-bisect/toolchains.test.cjs +++ b/eng/regression-bisect/toolchains.test.cjs @@ -5,7 +5,7 @@ const os = require("node:os"); const path = require("node:path"); const { createHash } = require("node:crypto"); const { resolveLatestStableSdk, readRevisionToolchain, acquireToolchain, verifyToolchainIdentity } = require("./toolchains.cjs"); -const { policy, admission, digest } = require("./fixtures/contracts.cjs"); +const { policy, admission, observation, seal, digest } = require("./fixtures/contracts.cjs"); const { admitBisection, fingerprintInput } = require("./model.cjs"); const { capture, publicUrl, isolatedEnvironment } = require("./toolchain-io.cjs"); @@ -19,7 +19,7 @@ const archive = (version, rid = "win-x64") => ({ url: `${feed}/Sdk/${version}/dotnet-sdk-${version}-${rid}${rid.startsWith("win") ? ".zip" : ".tar.gz"}`, hash: hash512(archiveBytes), }); -const sdk = (version, rids = ["win-x64", "win-arm64", "linux-x64", "osx-arm64"]) => ({ +const sdk = (version, rids = ["win-x64", "win-arm64", "linux-x64", "linux-musl-x64", "osx-arm64"]) => ({ version, "runtime-version": `${version.split(".")[0]}.0.0`, files: rids.map(rid => archive(version, rid)), }); const release = (version, date = "2026-09-08", others = []) => ({ @@ -76,6 +76,7 @@ for (const [p, suffix] of [ [platform, "win-x64.zip"], [{ os: "windows", rid: "win-arm64", architecture: "arm64" }, "win-arm64.zip"], [{ os: "linux", rid: "linux-x64", architecture: "x64" }, "linux-x64.tar.gz"], + [{ os: "linux", rid: "linux-musl-x64", architecture: "x64" }, "linux-musl-x64.tar.gz"], [{ os: "macos", rid: "osx-arm64", architecture: "arm64" }, "osx-arm64.tar.gz"], ]) { test(`exact archive: ${p.rid}`, async () => { @@ -322,40 +323,6 @@ test("unavailable historical SDK records public fallback attempts without runnin assert.ok(result.attempts.some(a => a.status === 404)); }); -test("different verified source declarations compose with invariant reproduction identities", () => { - const a = admission(); - for (const [o, version] of [[a.main, "11.0.100-rc.1.26425.128"], [a.good, "9.0.100"]]) { - o.cells[0].buildToolchain = { - sourceSha: o.target.sourceSha, globalJsonDigest: digest(version), evidenceDigest: digest(`evidence-${version}`), - requestedSdk: version, requestedTools: version, - requestedCompilerRuntime: `${version.split(".")[0]}.0.0`, - sdk: { version, sha256: digest(version), verified: true }, - compilerRuntime: { version: `${version.split(".")[0]}.0.0`, sha256: digest(`runtime-${version}`), verified: true }, - verified: true, supported: true, - }; - } - a.budget.authenticatedObservationDigests = [a.stable, a.main, a.good].map(fingerprintInput); - const result = admitBisection(a); - assert.equal(result.decision, "admit", JSON.stringify(result)); - assert.equal(result.toolchainTransitions.length, 1); - for (const patch of [ - b => b.requestedSdk = "8.0.100", - b => b.sourceSha = "0".repeat(40), - b => b.verified = false, - b => b.supported = false, - b => b.compilerRuntime.version = "7.0.0", - ]) { - const invalid = structuredClone(a); - patch(invalid.good.cells[0].buildToolchain); - invalid.budget.authenticatedObservationDigests = [invalid.stable, invalid.main, invalid.good].map(fingerprintInput); - assert.equal(admitBisection(invalid).decision, "blocked"); - } - const wrongRuntime = structuredClone(a); - wrongRuntime.good.cells[0].runtime.version = "7.0.0"; - wrongRuntime.budget.authenticatedObservationDigests = [wrongRuntime.stable, wrongRuntime.main, wrongRuntime.good].map(fingerprintInput); - assert.equal(admitBisection(wrongRuntime).decision, "blocked"); -}); - for (const location of [ "https://ci.dot.net/public/../../private/sdk.zip", "https://ci.dot.net/public/%2e%2e/private/sdk.zip", @@ -471,65 +438,189 @@ test("historical public preview fallback preserves checksum provenance and exact assert.deepEqual(result.attempts.filter(a => a.url.endsWith(".sha512")).map(a => a.status), [404, 200]); }); -for (const [name, packaged, localCompiler] of [ - ["a bundled nuspec", true, false], - ["no NuGet library in FSI deps", false, false], - ["local compiler runtime traces", true, true], -]) { - test(`captured identity with ${name}`, async t => { +async function captureFixture(request, { packaged = true, rid = request.toolchain.platform.rid } = {}) { + const { root, executable, toolchain } = request; + const version = toolchain.sdkVersion; + const runtimeVersion = toolchain.bundledRuntime.version; + const fsharp = path.join(root, "sdk", version, "FSharp"); + await fs.mkdir(fsharp, { recursive: true }); + for (const name of ["fsc.dll", "fsi.dll", "FSharp.Core.dll"]) await fs.writeFile(path.join(fsharp, name), name); + await fs.writeFile(path.join(fsharp, "fsi.deps.json"), JSON.stringify({ libraries: { "fsi/1.0.0": { type: "project" } } })); + await fs.writeFile(path.join(fsharp, "fsi.runtimeconfig.json"), JSON.stringify({ + runtimeOptions: { framework: { name: "Microsoft.NETCore.App", version: runtimeVersion } }, + })); + if (packaged) { + await fs.mkdir(path.join(fsharp, "library-packs")); + await fs.writeFile(path.join(fsharp, "library-packs", "FSharp.Core.10.1.401.nupkg"), "package"); + } + const runtime = path.join(root, "shared", "Microsoft.NETCore.App"); + const runtimeVersions = [...new Set([runtimeVersion, ...toolchain.runtimeRequirements.map(r => r.version)])]; + for (const version of runtimeVersions) { + await fs.mkdir(path.join(runtime, version), { recursive: true }); + await fs.writeFile(path.join(runtime, version, "System.Private.CoreLib.dll"), `runtime-${version}`); + } + await fs.writeFile(executable, "host"); + const graph = { runtimes: { + "win10-x64": { "#import": ["win10", "win7-x64"] }, "win7-x64": { "#import": ["win-x64"] }, + "ubuntu.22.04-x64": { "#import": ["ubuntu-x64"] }, "ubuntu-x64": { "#import": ["linux-x64"] }, + "osx.12-arm64": { "#import": ["osx-arm64"] }, + "alpine.3.18-x64": { "#import": ["linux-musl-x64"] }, "linux-musl-x64": { "#import": ["linux-x64"] }, + } }; + await fs.writeFile(path.join(root, "sdk", version, "RuntimeIdentifierGraph.json"), JSON.stringify(graph)); + return { + fs, + run: async (file, args, settings) => { + assert.equal(file, executable); + if (args.at(-1) === "--help") { + assert.equal(settings.env.DOTNET_ROLL_FORWARD, "Disable"); + await fs.appendFile(settings.env.DOTNET_HOST_TRACEFILE, + `Probing ${args[3]}\nChose FX version [${path.join(runtime, args[2])}]\n`); + return { exitCode: 0, stderr: "", stdout: "F# compiler help" }; + } + const outputs = { + "--info": `Base Path: ${path.join(root, "sdk", version)}\nRID: ${rid}`, + "--version": version, + "--list-sdks": `${version} [${path.join(root, "sdk")}]`, + "--list-runtimes": runtimeVersions.map(v => `Microsoft.NETCore.App ${v} [${runtime}]`).join("\n"), + exec: `ARCH|${toolchain.platform.architecture}\nRUNTIME|${runtimeVersion}\nFILE|fsc.dll|fsc|15.2.401.0|15.204.126.42413\n` + + "FILE|fsi.dll|fsi|15.2.401.0|15.204.126.42413\nFILE|FSharp.Core.dll|FSharp.Core|10.1.0.0|10.104.126.42413\n" + + (packaged ? "PACKAGE|FSharp.Core.10.1.401.nupkg|FSharp.Core|10.1.401" : ""), + }; + return { exitCode: 0, stderr: "", stdout: outputs[args[0]] }; + }, + }; +} + +for (const packaged of [true, false]) { + test(`captured identity with bundled package: ${packaged}`, async t => { const root = await directory(t); const env = isolatedEnvironment(root, platform); await fs.mkdir(env.HOME); - const fsharp = path.join(root, "sdk", "10.0.401", "FSharp"); - await fs.mkdir(fsharp, { recursive: true }); - for (const name of ["fsc.dll", "fsi.dll", "FSharp.Core.dll"]) await fs.writeFile(path.join(fsharp, name), name); - await fs.writeFile(path.join(fsharp, "fsi.deps.json"), JSON.stringify({ libraries: { "fsi/1.0.0": { type: "project" } } })); - const configBytes = JSON.stringify({ - runtimeOptions: { framework: { name: "Microsoft.NETCore.App", version: "10.0.0" } }, - }); - await fs.writeFile(path.join(fsharp, "fsi.runtimeconfig.json"), configBytes); - await fs.writeFile(path.join(fsharp, "fsc.runtimeconfig.json"), configBytes); - if (packaged) { - await fs.mkdir(path.join(fsharp, "library-packs")); - await fs.writeFile(path.join(fsharp, "library-packs", "FSharp.Core.10.1.401.nupkg"), "package"); - } - const runtime = path.join(root, "shared", "Microsoft.NETCore.App"); - await fs.mkdir(path.join(runtime, "10.0.12"), { recursive: true }); - await fs.writeFile(path.join(runtime, "10.0.12", "System.Private.CoreLib.dll"), "runtime"); - const executable = path.join(root, "dotnet.exe"); - await fs.writeFile(executable, "host"); - const runtimeRequirements = localCompiler ? [["compiler", "fsc"], ["fsi", "fsi"]].map(([role, file]) => ({ - name: "Microsoft.NETCore.App", version: "10.0.12", role, - config: { path: path.join(fsharp, `${file}.runtimeconfig.json`), sha256: digest(configBytes) }, - })) : []; - const identity = await capture({ - root, env, executable, cwd: root, - toolchain: { sourceDirectory: root, sdkVersion: "10.0.401", bundledRuntime: { version: "10.0.12" }, runtimeRequirements }, - }, { - fs, - run: async (file, args, settings) => { - assert.equal(file, executable); - if (args.at(-1) === "--help") { - assert.equal(args[2], "10.0.12"); - assert.equal(settings.env.DOTNET_ROLL_FORWARD, "Disable"); - await fs.writeFile(settings.env.DOTNET_HOST_TRACEFILE, `Chose FX version [${path.join(runtime, "10.0.12")}]`); - return { exitCode: 0, stderr: "", stdout: "F# compiler help" }; - } - const outputs = { - "--info": `Base Path: ${path.join(root, "sdk", "10.0.401")}\nRID: win-x64`, - "--version": "10.0.401", - "--list-sdks": `10.0.401 [${path.join(root, "sdk")}]`, - "--list-runtimes": `Microsoft.NETCore.App 10.0.12 [${runtime}]`, - exec: "ARCH|x64\nRUNTIME|10.0.12\nFILE|fsc.dll|fsc|15.2.401.0|15.204.126.42413\n" + - "FILE|fsi.dll|fsi|15.2.401.0|15.204.126.42413\nFILE|FSharp.Core.dll|FSharp.Core|10.1.0.0|10.104.126.42413\n" + - (packaged ? "PACKAGE|FSharp.Core.10.1.401.nupkg|FSharp.Core|10.1.401" : ""), - }; - return { exitCode: 0, stderr: "", stdout: outputs[args[0]] }; - }, - }); + const request = { + root, env, executable: path.join(root, "dotnet.exe"), cwd: root, + toolchain: { platform, sdkVersion: "10.0.401", bundledRuntime: { version: "10.0.12" }, runtimeRequirements: [] }, + }; + const identity = await capture(request, await captureFixture(request, { packaged })); assert.equal(identity.files.core.packageIdentity, packaged ? "FSharp.Core/10.1.401" : null); assert.equal(identity.files.core.assemblyVersion, "10.1.0.0"); assert.equal(identity.compilerRuntime.version, "10.0.12"); - assert.deepEqual(identity.runtimeExecutions.map(r => [r.role, r.version]), localCompiler ? [["compiler", "10.0.12"], ["fsi", "10.0.12"]] : []); + assert.deepEqual(identity.runtimeExecutions, []); + }); +} + +async function acquireSource(t, version, sourceSha, p, { rid = p.rid, edit = () => {} } = {}) { + const { root, bytes } = await source(t, version); + const runtimeVersion = `${version.split(".")[0]}.0.0`; + const runtimeConfigs = []; + // Equal config bytes and repeated roles must still retain independent traces. + for (const [name, role] of [["fsc", "compiler"], ["fsi", "fsi"], ["other-fsc", "compiler"]]) { + const file = path.join(root, `${name}.runtimeconfig.json`); + await fs.writeFile(file, JSON.stringify({ runtimeOptions: { framework: { name: "Microsoft.NETCore.App", version: runtimeVersion } } })); + await fs.writeFile(path.join(root, `${name}.dll`), name); + runtimeConfigs.push({ path: file, role }); + } + const { toolchain } = await readRevisionToolchain({ ...options(p), sourceDirectory: root, sourceSha, runtimeConfigs }); + const result = await acquireToolchain({ policy: policy(), toolchain, storageDirectory: await directory(t) }, { + ...metadata([[version.split(".").slice(0, 2).join("."), "lts", [release(version)]]]), + installer: async request => { + await fs.mkdir(path.join(request.root, "shared", "Microsoft.NETCore.App", runtimeVersion), { recursive: true }); + return { + archive: { ...request.archive, actualSha512: hash512(archiveBytes), sha256: digest(archiveBytes), redirectUrl: request.archive.url }, + installer: {}, attempts: [], + }; + }, + capture: async request => { + assert.equal(request.cwd, root); + const identity = await capture(request, await captureFixture(request, { rid })); + edit(identity); + return identity; + }, + }); + assert.deepEqual(await fs.readFile(path.join(root, "global.json")), bytes); + return result; +} + +for (const [p, hostRid] of [ + [platform, "win10-x64"], + [{ os: "linux", rid: "linux-x64", architecture: "x64" }, "ubuntu.22.04-x64"], + [{ os: "macos", rid: "osx-arm64", architecture: "arm64" }, "osx.12-arm64"], + [{ os: "linux", rid: "linux-musl-x64", architecture: "x64" }, "alpine.3.18-x64"], + [platform, "win-x64"], +]) { + test(`acquisition to admission preserves ${hostRid} source toolchains and execution evidence`, async t => { + const a = admission(); + a.manifest.matrix[0].os = p.os; + a.manifest.matrix[0].architecture = p.architecture; + seal(a.manifest); + a.budget.authorization.manifestDigest = fingerprintInput(a.manifest); + a.budget.authorization.inputDigest = a.manifest.approvedInputDigest; + a.stable = observation("released", "product-failure", a.manifest); + a.budget.stableSdk = a.stable.target.sdk; + for (const [key, version, outcome, rid] of [ + ["good", "6.0.428", "pass", hostRid], ["main", "11.0.100-rc.1.26425.128", "product-failure", p.rid], + ]) { + const sourceSha = a[key].target.sourceSha; + a[key] = observation("source", outcome, a.manifest, sourceSha); + const result = await acquireSource(t, version, sourceSha, p, { rid }); + assert.equal(result.status, "acquired", JSON.stringify(result)); + const { buildToolchain, ...evidence } = result.identity; + assert.equal(buildToolchain.evidenceDigest, fingerprintInput(evidence)); + assert.equal(buildToolchain.sourceSha, sourceSha); + assert.equal(buildToolchain.globalJsonDigest, evidence.requested.globalJson.sha256); + assert.equal(buildToolchain.sdk.version, version); + assert.equal(buildToolchain.compilerRuntime.version, `${version.split(".")[0]}.0.0`); + assert.equal(evidence.rid, rid); + const traces = evidence.runtimeExecutions.map(r => r.hostTrace.path); + assert.equal(new Set(traces).size, 3, "each execution owns its trace"); + for (const execution of evidence.runtimeExecutions) { + assert.equal(digest(await fs.readFile(execution.hostTrace.path)), execution.hostTrace.sha256); + } + assert.deepEqual(JSON.parse(await fs.readFile(result.evidencePath)).identity, result.identity); + a[key].cells[0].buildToolchain = buildToolchain; + } + a.budget.authenticatedObservationDigests = [a.stable, a.main, a.good].map(fingerprintInput); + const admitted = admitBisection(a); + assert.equal(admitted.decision, "admit", JSON.stringify(admitted)); + assert.deepEqual(admitted.toolchainTransitions, [{ + cellId: a.main.cells[0].id, good: a.good.cells[0].buildToolchain, main: a.main.cells[0].buildToolchain, + }]); + for (const edit of [ + c => c.buildToolchain.requestedSdk = "8.0.100", + c => c.buildToolchain.sourceSha = "0".repeat(40), + c => c.buildToolchain.verified = false, + c => c.buildToolchain.supported = false, + c => c.buildToolchain.compilerRuntime.version = "7.0.0", + c => c.runtime.version = "7.0.0", + ]) { + const invalid = structuredClone(a); + edit(invalid.good.cells[0]); + invalid.budget.authenticatedObservationDigests = [invalid.stable, invalid.main, invalid.good].map(fingerprintInput); + assert.equal(admitBisection(invalid).decision, "blocked"); + } + }); +} + +for (const [name, edit, message, rid = platform.rid, p = platform] of [ + ["missing compiler execution", i => i.runtimeExecutions = i.runtimeExecutions.filter(r => r.role !== "compiler"), /unverified compiler/], + ["missing FSI execution", i => i.runtimeExecutions = i.runtimeExecutions.filter(r => r.role !== "fsi"), /unverified fsi/], + ["missing same-role same-digest config execution", i => i.runtimeExecutions.pop(), /unverified compiler/], + ["wrong config digest", i => i.runtimeExecutions.forEach(r => r.configDigest = digest("other")), /unverified compiler/], + ["wrong local runtime", i => i.runtimeExecutions.forEach(r => r.version = "7.0.0"), /unverified compiler/], + ["wrong local architecture", i => i.runtimeExecutions.forEach(r => r.architecture = "arm64"), /unverified compiler/], + ["wrong host OS", i => i.rid = "linux-x64", /RID/], + ["wrong host architecture", i => i.rid = "win-arm64", /RID/], + ["unproven historical RID", i => delete i.runtimeGraph, /RID/, "win10-x64"], + ["changed RID graph", i => { if (i.runtimeGraph) i.runtimeGraph.sha256 = digest("other"); }, /RID/, "win10-x64"], + ["foreign RID graph", i => { if (i.runtimeGraph) i.runtimeGraph.path = path.resolve("machine", "RuntimeIdentifierGraph.json"); }, /RID/, "win10-x64"], + ["unknown historical RID", () => {}, /RID/, "unknown-x64"], + ["musl host for glibc archive", () => {}, /RID/, "alpine.3.18-x64", { os: "linux", rid: "linux-x64", architecture: "x64" }], + ["glibc host for musl archive", () => {}, /RID/, "ubuntu.22.04-x64", { os: "linux", rid: "linux-musl-x64", architecture: "x64" }], +]) { + test(`acquisition emits no admission record for ${name}`, async t => { + const result = await acquireSource(t, "6.0.428", "a".repeat(40), p, { edit, rid }); + assert.equal(result.status, "untestable", JSON.stringify(result)); + assert.equal(result.outcome.reason, "wrong-identity"); + assert.match(result.error.message, message); + assert.equal(result.identity, undefined); }); } From 4abb7392d420bf06a17cbdd9507c4544f6bba845 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 02:57:35 +0200 Subject: [PATCH 12/99] Cover toolchain trace and redirect failure paths Reject redirects without a usable Location before another request. Exercise malformed local runtime traces and pre-request public-feed checks through production acquisition boundaries. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 20 ++-- eng/regression-bisect/toolchain-io.cjs | 4 +- eng/regression-bisect/toolchains.test.cjs | 112 ++++++++++++++++++++-- 3 files changed, 121 insertions(+), 15 deletions(-) diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index 5e30ff7e1e3..266da288f2a 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -224,7 +224,8 @@ The shared boundaries are injectable: `fs` (`node:fs/promises`), `clock` final URL, redirect hops and raw bytes, or streams a large download to `destination`. Every production redirect is checked against the policy's credential-free public-feed allowlist; downloads, processes and smoke runs -are bounded. Process execution receives a newly constructed environment, not +are bounded. Redirects without a nonempty `Location` fail before another +request. Process execution receives a newly constructed environment, not the caller's secrets, private feeds, credential providers or user profile. `toolchain-installer.cjs` calls the existing Arcade `GetDotNetInstallScript` @@ -275,7 +276,7 @@ its uniquely created child directory. It has a ten-minute cancellation deadline. Exit 0 means actual acquisition/identity success; exit 2 records the exact block, never mocked download success. This is not a source build or live issue run. -On 2026-09-19 at 00:35:31 UTC, that command resolved metadata and completed +On 2026-09-19 at 00:55:06 UTC, that command resolved metadata and completed successfully (exit 0) on Windows x64: requested/resolved SDK `10.0.401`, RID `win-x64`, bundled/executing runtime `10.0.12`, compiler and FSI assembly `15.2.401.0`, Core assembly `10.1.0.0`, @@ -293,10 +294,10 @@ not production version constants. Validation also completed `node --check` for all twelve top-level `.cjs` files and the required `node --test eng\regression-bisect\model.test.cjs eng\regression-bisect\toolchains.test.cjs` -command (322 passed), plus -`node --test eng\regression-bisect\toolchains.test.cjs` (72 passed). -The broader suite above passed all 538 tests, including intake, authenticated -state replay and installer-boundary coverage. The repair cases first reproduced +command (416 passed), plus +`node --test eng\regression-bisect\toolchains.test.cjs` (166 passed). +The broader suite above passed all 632 tests without skips, including intake, +authenticated state replay and installer-boundary coverage. Earlier repair cases reproduced seven failures for musl installer arguments, historical host RIDs, and colliding trace files. An additional failing case caught missing same-role execution evidence hidden by identical config digests. The passing acquisition-to-admission @@ -304,6 +305,13 @@ cases use captured evidence, preserve differing historical/preview build toolchains and their coupled transitions, and reject invalid execution evidence and incomparable reproduction runtimes. Cross-platform installer/identity cases use injected boundaries; the actual download smoke above ran on Windows only. +The additional failure-path cases supply malformed trace bytes to production +capture for compiler, FSI and repeated compiler roles, and exercise production +HTTP redirects with only `fetch` mocked. Removing trace validation caused 18 +failures; removing redirect allowlist validation caused 45 failures. Both guards +were restored. Fifteen missing/empty `Location` cases failed before immediate +rejection was added; permitted relative/public redirects still preserve +GET/HEAD behavior, provenance and hashes. `dotnet fantomas .` completed without changes or errors; working-tree and sprint commit-range `git diff --check` checks passed. The signed checkpoint fixture was regenerated through its existing harness for the new pinned public-feed diff --git a/eng/regression-bisect/toolchain-io.cjs b/eng/regression-bisect/toolchain-io.cjs index f04da0ef6d7..e23e2a868f8 100644 --- a/eng/regression-bisect/toolchain-io.cjs +++ b/eng/regression-bisect/toolchain-io.cjs @@ -29,7 +29,9 @@ function createHttp(policy, fileSystem = fs, signal) { }); if ([301, 302, 303, 307, 308].includes(response.status)) { await response.body?.cancel(); - const next = publicUrl(new URL(response.headers.get("location"), current).href, policy); + const location = response.headers.get("location"); + requireValue(location?.trim(), "redirect response is missing Location"); + const next = publicUrl(new URL(location, current).href, policy); redirects.push({ from: current, to: next, status: response.status }); current = next; continue; diff --git a/eng/regression-bisect/toolchains.test.cjs b/eng/regression-bisect/toolchains.test.cjs index 0cbfd141a03..4d8f59b558d 100644 --- a/eng/regression-bisect/toolchains.test.cjs +++ b/eng/regression-bisect/toolchains.test.cjs @@ -7,7 +7,7 @@ const { createHash } = require("node:crypto"); const { resolveLatestStableSdk, readRevisionToolchain, acquireToolchain, verifyToolchainIdentity } = require("./toolchains.cjs"); const { policy, admission, observation, seal, digest } = require("./fixtures/contracts.cjs"); const { admitBisection, fingerprintInput } = require("./model.cjs"); -const { capture, publicUrl, isolatedEnvironment } = require("./toolchain-io.cjs"); +const { capture, publicUrl, createHttp, isolatedEnvironment } = require("./toolchain-io.cjs"); const indexUrl = "https://builds.dotnet.microsoft.com/dotnet/release-metadata/releases-index.json"; const feed = "https://builds.dotnet.microsoft.com/dotnet"; @@ -323,16 +323,81 @@ test("unavailable historical SDK records public fallback attempts without runnin assert.ok(result.attempts.some(a => a.status === 404)); }); -for (const location of [ +const forbiddenDownloads = [ "https://ci.dot.net/public/../../private/sdk.zip", "https://ci.dot.net/public/%2e%2e/private/sdk.zip", "https://builds.dotnet.microsoft.com/dotnet/sdk.zip?token=private", "https://builds.dotnet.microsoft.com.evil.test/dotnet/sdk.zip", "http://builds.dotnet.microsoft.com/dotnet/sdk.zip", -]) { + "https://user:secret@builds.dotnet.microsoft.com/dotnet/sdk.zip", + "https://builds.dotnet.microsoft.com:8443/dotnet/sdk.zip", + "https://builds.dotnet.microsoft.com/dotnet/sdk.zip#fragment", + "//unapproved.example/sdk.zip", +]; +for (const location of forbiddenDownloads.filter(url => url.startsWith("https:") || url.startsWith("http:"))) { test(`public feed rejects ${location}`, () => assert.throws(() => publicUrl(location, policy()), /unapproved/)); } +for (const status of [301, 302, 303, 307, 308]) { + for (const location of forbiddenDownloads) { + test(`HTTP ${status} rejects redirect before requesting ${location}`, async t => { + const allowed = `${feed}/redirected-sdk.zip`; + const responses = [ + new Response("first hop", { status, headers: { location: "/dotnet/redirected-sdk.zip" } }), + new Response("forbidden hop", { status, headers: { location } }), + ]; + const cancellations = responses.map(response => t.mock.method(response.body, "cancel")); + const requests = []; + t.mock.method(globalThis, "fetch", async (url, settings) => { + requests.push(url); + assert.equal(settings.redirect, "manual"); + assert.equal(settings.method, "GET"); + return responses[requests.length - 1] ?? new Response("must not download"); + }); + const transport = createHttp(policy(), { open: () => assert.fail("must not write forbidden download") }); + await assert.rejects(transport(indexUrl, { destination: "unused.zip" }), /unapproved public download URL/); + assert.deepEqual(requests, [indexUrl, allowed]); + assert.ok(cancellations.every(cancel => cancel.mock.callCount() === 1)); + }); + } + for (const location of [null, "", " "]) { + test(`HTTP ${status} rejects missing/empty Location ${JSON.stringify(location)}`, async t => { + const response = new Response(null, { status, headers: location === null ? {} : { location } }); + const fetch = t.mock.method(globalThis, "fetch", async () => response); + await assert.rejects(createHttp(policy())(indexUrl), /redirect response is missing Location/); + assert.equal(fetch.mock.callCount(), 1); + }); + } + for (const method of ["GET", "HEAD"]) { + test(`HTTP ${status} preserves allowed relative/public redirects for ${method}`, async t => { + const relative = `${feed}/release-metadata/channel.json`; + const final = "https://ci.dot.net/public/sdk.zip"; + const responses = [ + new Response(null, { status, headers: { location: "channel.json" } }), + new Response(null, { status, headers: { location: final } }), + new Response(method === "GET" ? archiveBytes : null), + ]; + const requests = []; + t.mock.method(globalThis, "fetch", async (url, settings) => { + requests.push(url); + assert.equal(settings.redirect, "manual"); + assert.equal(settings.method, method); + return responses[requests.length - 1]; + }); + const result = await createHttp(policy())(indexUrl, { method }); + const bytes = method === "GET" ? archiveBytes : Buffer.alloc(0); + assert.deepEqual(requests, [indexUrl, relative, final]); + assert.equal(result.status, 200); + assert.equal(result.url, final); + assert.deepEqual(result.redirects, [{ from: indexUrl, to: relative, status }, { from: relative, to: final, status }]); + assert.deepEqual(result.bytes, bytes); + assert.equal(result.size, bytes.length); + assert.equal(result.sha256, digest(bytes)); + assert.equal(result.sha512, hash512(bytes)); + }); + } +} + test("bad archive aborts acquisition before any acquired executable runs", async t => { const { toolchain } = await resolveLatestStableSdk(options(), metadata()); const result = await acquireToolchain({ policy: policy(), toolchain, storageDirectory: await directory(t) }, { @@ -438,7 +503,9 @@ test("historical public preview fallback preserves checksum provenance and exact assert.deepEqual(result.attempts.filter(a => a.url.endsWith(".sha512")).map(a => a.status), [404, 200]); }); -async function captureFixture(request, { packaged = true, rid = request.toolchain.platform.rid } = {}) { +async function captureFixture(request, { + packaged = true, rid = request.toolchain.platform.rid, trace = expected => `Chose FX version [${expected}]\n`, +} = {}) { const { root, executable, toolchain } = request; const version = toolchain.sdkVersion; const runtimeVersion = toolchain.bundledRuntime.version; @@ -473,8 +540,8 @@ async function captureFixture(request, { packaged = true, rid = request.toolchai assert.equal(file, executable); if (args.at(-1) === "--help") { assert.equal(settings.env.DOTNET_ROLL_FORWARD, "Disable"); - await fs.appendFile(settings.env.DOTNET_HOST_TRACEFILE, - `Probing ${args[3]}\nChose FX version [${path.join(runtime, args[2])}]\n`); + const output = trace(path.join(runtime, args[2]), args[3]); + if (output !== null) await fs.appendFile(settings.env.DOTNET_HOST_TRACEFILE, output); return { exitCode: 0, stderr: "", stdout: "F# compiler help" }; } const outputs = { @@ -508,7 +575,7 @@ for (const packaged of [true, false]) { }); } -async function acquireSource(t, version, sourceSha, p, { rid = p.rid, edit = () => {} } = {}) { +async function acquireSource(t, version, sourceSha, p, { rid = p.rid, edit = () => {}, trace } = {}) { const { root, bytes } = await source(t, version); const runtimeVersion = `${version.split(".")[0]}.0.0`; const runtimeConfigs = []; @@ -531,7 +598,7 @@ async function acquireSource(t, version, sourceSha, p, { rid = p.rid, edit = () }, capture: async request => { assert.equal(request.cwd, root); - const identity = await capture(request, await captureFixture(request, { rid })); + const identity = await capture(request, await captureFixture(request, { rid, trace })); edit(identity); return identity; }, @@ -540,6 +607,35 @@ async function acquireSource(t, version, sourceSha, p, { rid = p.rid, edit = () return result; } +for (const compiler of ["fsc", "fsi", "other-fsc"]) { + for (const [name, trace] of [ + ["missing", () => null], + ["empty", () => ""], + ["no selection", expected => `Considering FX version [${expected}]\n`], + ["unterminated selection", expected => `Chose FX version [${expected}\n`], + ["wrong version", expected => `Chose FX version [${path.join(path.dirname(expected), "99.0.0")}]\n`], + ["wrong framework", expected => `Chose FX version [${path.join(path.dirname(path.dirname(expected)), "Other.App", path.basename(expected))}]\n`], + ["outside installation", expected => `Chose FX version [${path.join(path.parse(expected).root, "machine", "shared", "Microsoft.NETCore.App", path.basename(expected))}]\n`], + ]) { + test(`capture rejects ${name} runtime trace from ${compiler} before admission`, async t => { + let malformed = 0; + const result = await acquireSource(t, "6.0.428", "a".repeat(40), platform, { + trace: (expected, file) => { + if (path.basename(file, ".dll") !== compiler) return `Chose FX version [${expected}]\n`; + malformed++; + return trace(expected); + }, + }); + assert.equal(malformed, 1); + assert.equal(result.status, "untestable", JSON.stringify(result)); + assert.equal(result.outcome.reason, "wrong-identity"); + assert.equal(result.error.stage, "identity"); + assert.match(result.error.message, name === "missing" ? /ENOENT.*host-trace/ : /host trace did not verify/); + for (const field of ["identity", "evidencePath", "launch"]) assert.equal(result[field], undefined); + }); + } +} + for (const [p, hostRid] of [ [platform, "win10-x64"], [{ os: "linux", rid: "linux-x64", architecture: "x64" }, "ubuntu.22.04-x64"], From 553101960fcb8551d305fc5c0dabad9f2a3e70f3 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 03:15:25 +0200 Subject: [PATCH 13/99] Await regression toolchain archive delivery before verification Synchronize loopback delivery and file closure with installer verification and cleanup. Cover controlled completion ordering, late delivery errors, and missing archive GETs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 17 +++- eng/regression-bisect/toolchain-installer.cjs | 25 ++++-- .../toolchain-installer.test.cjs | 79 +++++++++++++++++++ 3 files changed, 109 insertions(+), 12 deletions(-) diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index 266da288f2a..c9789b0e1e2 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -237,7 +237,10 @@ RID. A loopback-only, bounded feed serves the already downloaded and SHA-512-verified archive to that installer. This transport boundary records the **actual public download's** redirects/digests without reimplementing extraction or trusting a separate HEAD -request. The official retained archive is hashed again. Supplemental runtime +request. Verification waits for pending server deliveries and file closure, +not just installer exit; cleanup also drains deliveries before removing their +archive. Late delivery errors still fail acquisition. The official retained +archive is hashed again. Supplemental runtime installation uses the official skip-non-versioned-files flag to preserve the SDK host. Arcade-owned files are neither edited nor copied. @@ -276,7 +279,7 @@ its uniquely created child directory. It has a ten-minute cancellation deadline. Exit 0 means actual acquisition/identity success; exit 2 records the exact block, never mocked download success. This is not a source build or live issue run. -On 2026-09-19 at 00:55:06 UTC, that command resolved metadata and completed +On 2026-09-19 at 01:13:39 UTC, that command resolved metadata and completed successfully (exit 0) on Windows x64: requested/resolved SDK `10.0.401`, RID `win-x64`, bundled/executing runtime `10.0.12`, compiler and FSI assembly `15.2.401.0`, Core assembly `10.1.0.0`, @@ -296,8 +299,14 @@ files and the required `node --test eng\regression-bisect\model.test.cjs eng\regression-bisect\toolchains.test.cjs` command (416 passed), plus `node --test eng\regression-bisect\toolchains.test.cjs` (166 passed). -The broader suite above passed all 632 tests without skips, including intake, -authenticated state replay and installer-boundary coverage. Earlier repair cases reproduced +The broader suite above passed all 642 tests without skips in three consecutive +runs, including intake, authenticated state replay and installer-boundary coverage. +`node --test eng\regression-bisect\toolchain-installer.test.cjs` passed all 47 +tests. Controlled Windows/Linux delivery ordering reproduced six failures before +synchronization was added. The regression cases now verify pending delivery, +stream/close errors, cleanup after installer failure, and rejection of success +without an archive GET (including HEAD-only requests). +Earlier repair cases reproduced seven failures for musl installer arguments, historical host RIDs, and colliding trace files. An additional failing case caught missing same-role execution evidence hidden by identical config digests. The passing acquisition-to-admission diff --git a/eng/regression-bisect/toolchain-installer.cjs b/eng/regression-bisect/toolchain-installer.cjs index 2248c140954..516812a9911 100644 --- a/eng/regression-bisect/toolchain-installer.cjs +++ b/eng/regression-bisect/toolchain-installer.cjs @@ -102,12 +102,16 @@ async function startFeed(fs, download, archiveName, productDirectory, version, p let active = 0; let downloads = 0; let error; - const server = http.createServer(async (req, res) => { + const deliveries = []; + const server = http.createServer((req, res) => { res.setHeader('Connection', 'close'); if (++requests > 64 || active >= 4) { res.writeHead(429).end(); return; } if (!['GET', 'HEAD'].includes(req.method)) { res.writeHead(405).end(); return; } if (req.url !== archiveRoute && !versionRoutes.has(req.url)) { res.writeHead(404).end(); return; } active++; + deliveries.push(deliver(req, res)); + }); + async function deliver(req, res) { try { if (versionRoutes.has(req.url)) { res.setHeader('Content-Type', 'text/plain'); @@ -132,7 +136,7 @@ async function startFeed(fs, download, archiveName, productDirectory, version, p } finally { active--; } - }); + } server.requestTimeout = server.headersTimeout = 30_000; server.timeout = 60_000; server.maxHeadersCount = 32; @@ -142,14 +146,19 @@ async function startFeed(fs, download, archiveName, productDirectory, version, p }); return { url: `http://127.0.0.1:${server.address().port}${prefix}`, - verify() { + async verify() { + // Installer exit and client receipt can precede server-side stream completion. + await Promise.all(deliveries); if (error) throw new Error(`Loopback archive delivery failed: ${error.message}`, { cause: error }); if (!downloads) throw new Error('Official installer did not download the verified archive'); }, - close: () => new Promise((resolve, reject) => { - server.close(cause => cause ? reject(cause) : resolve()); - server.closeAllConnections(); - }), + async close() { + await new Promise((resolve, reject) => { + server.close(cause => cause ? reject(cause) : resolve()); + server.closeAllConnections(); + }); + await Promise.all(deliveries); + }, }; } @@ -279,7 +288,7 @@ async function installArchive(options, deps) { '--no-path', '--keep-zip', '--zip-path', archivePath, '--azure-feed', feed.url, '--os', os]; if (product !== 'sdk') args.push(windows ? '-Runtime' : '--runtime', product, windows ? '-SkipNonVersionedFiles' : '--skip-non-versioned-files'); await run(args, 1_200_000); - feed.verify(); + await feed.verify(); const retained = await hashFile(fs, archivePath); if (retained.sha512 !== digests.sha512 || retained.sha256 !== digests.sha256) throw new Error('Retained archive digest does not match verified download'); diff --git a/eng/regression-bisect/toolchain-installer.test.cjs b/eng/regression-bisect/toolchain-installer.test.cjs index eba4030a1ba..7ea55e1eff9 100644 --- a/eng/regression-bisect/toolchain-installer.test.cjs +++ b/eng/regression-bisect/toolchain-installer.test.cjs @@ -163,6 +163,69 @@ for (const settings of [ } for (const os of ['win', 'linux']) { + for (const completion of ['success', 'stream error', 'close error', 'installer error']) { + test(`awaits archive delivery after installer exit: ${os}, ${completion}`, async t => { + const { options, deps, state } = await fixture(t, { os }); + let reads = 0; + let releaseDelivery; + let deliveryFinished = false; + let signalDeliveryReady; + const deliveryReady = new Promise(resolve => { signalDeliveryReady = resolve; }); + const deliveryAllowed = new Promise(resolve => { releaseDelivery = resolve; }); + let received; + deps.fs = { + ...fs, + async open(filename, ...args) { + if (filename === options.archivePath) assert.equal(deliveryFinished, true); + const file = await fs.open(filename, ...args); + if (filename !== `${options.archivePath}.download` || ++reads !== 2) return file; + signalDeliveryReady(); + await deliveryAllowed; + return { + createReadStream(settings) { + if (completion === 'stream error') throw new Error('controlled stream failure'); + return file.createReadStream(settings); + }, + async close() { + await file.close(); + deliveryFinished = true; + if (completion === 'close error') throw new Error('controlled close failure'); + }, + }; + }, + }; + const run = deps.run; + deps.run = async (executable, args, execution) => { + const feedIndex = args.indexOf(os === 'win' ? '-AzureFeed' : '--azure-feed'); + if (feedIndex < 0) return run(executable, args, execution); + state.archiveUrl = `${args[feedIndex + 1]}/Sdk/${options.version}/${path.basename(options.archivePath)}`; + received = request(state.archiveUrl).then(value => value, error => error); + await deliveryReady; + await fs.copyFile(`${options.archivePath}.download`, options.archivePath); + assert.equal(deliveryFinished, false); + // Force installer completion before the server's pending delivery settles. + setImmediate(releaseDelivery); + return { exitCode: completion === 'installer error' ? 17 : 0, stdout: '', stderr: '' }; + }; + if (completion === 'success') { + const result = await installArchive(options, deps); + assert.equal(result.archive.actualSha512, options.archive.sha512); + assert.deepEqual(await fs.readFile(options.archivePath), payload); + } else { + await assert.rejects(installArchive(options, deps), + completion === 'stream error' ? /Loopback archive delivery failed: controlled stream failure/ + : completion === 'close error' ? /Loopback archive delivery failed: controlled close failure/ + : /Official installer process exited 17/); + } + assert.equal(deliveryFinished, true); + if (completion === 'stream error' || completion === 'installer error') + assert.match((await received).message, /ECONNRESET|socket hang up/); + else assert.deepEqual((await received).bytes, payload); + await assert.rejects(request(state.archiveUrl), /ECONNREFUSED|ECONNRESET/); + await assert.rejects(fs.stat(`${options.archivePath}.download`), { code: 'ENOENT' }); + }); + } + for (const product of ['sdk', 'dotnet']) { test(`missing nonversioned-file protection is ${product === 'sdk' ? 'allowed' : 'rejected'} for ${product} on ${os}`, async t => { const { options, deps, state } = await fixture(t, { os, product }); @@ -177,6 +240,22 @@ for (const os of ['win', 'linux']) { } } +for (const method of [null, 'HEAD']) { + test(`rejects installer success without archive GET: ${method || 'no request'}`, async t => { + const { options, deps, state } = await fixture(t); + const run = deps.run; + deps.run = async (executable, args, execution) => { + if (!args.includes('-AzureFeed')) return run(executable, args, execution); + state.archiveUrl = `${args[args.indexOf('-AzureFeed') + 1]}/Sdk/${options.version}/${path.basename(options.archivePath)}`; + if (method) assert.equal((await request(state.archiveUrl, method)).status, 200); + await fs.copyFile(`${options.archivePath}.download`, options.archivePath); + return { exitCode: 0, stdout: 'already installed', stderr: '' }; + }; + await assert.rejects(installArchive(options, deps), /Official installer did not download the verified archive/); + await assert.rejects(request(state.archiveUrl), /ECONNREFUSED|ECONNRESET/); + }); +} + test('rejects wrong SHA512 before any process runs', async t => { const { options, deps, state } = await fixture(t); state.payload = Buffer.from('wrong archive'); From 41bf362d1a491f8f349155a86c133dd94d3119f5 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 04:22:43 +0200 Subject: [PATCH 14/99] Add faithful regression reproduction execution Verify FSI, executed MSBuild compiler/task/target/Core identities, runtime assertions, and immutable producer metadata. Add bounded process handling, structured evidence tooling, and real released/local-product smoke fixtures. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 152 +++++ eng/regression-bisect/ReproEvidence.targets | 20 + .../evidence/AssetEvidence.cs | 76 +++ .../evidence/BinlogEvidence.cs | 286 +++++++++ .../evidence/Evidence.csproj | 22 + eng/regression-bisect/evidence/Program.cs | 27 + .../evidence/StartupHook/StartupHook.cs | 92 +++ .../evidence/StartupHook/StartupHook.csproj | 17 + .../fixtures/reproduction.cjs | 75 +++ eng/regression-bisect/fixtures/smoke.cjs | 111 ++++ eng/regression-bisect/repro-process.cjs | 49 ++ eng/regression-bisect/repro.cjs | 572 ++++++++++++++++++ eng/regression-bisect/repro.smoke.cjs | 91 +++ eng/regression-bisect/repro.test.cjs | 324 ++++++++++ 14 files changed, 1914 insertions(+) create mode 100644 eng/regression-bisect/ReproEvidence.targets create mode 100644 eng/regression-bisect/evidence/AssetEvidence.cs create mode 100644 eng/regression-bisect/evidence/BinlogEvidence.cs create mode 100644 eng/regression-bisect/evidence/Evidence.csproj create mode 100644 eng/regression-bisect/evidence/Program.cs create mode 100644 eng/regression-bisect/evidence/StartupHook/StartupHook.cs create mode 100644 eng/regression-bisect/evidence/StartupHook/StartupHook.csproj create mode 100644 eng/regression-bisect/fixtures/reproduction.cjs create mode 100644 eng/regression-bisect/fixtures/smoke.cjs create mode 100644 eng/regression-bisect/repro-process.cjs create mode 100644 eng/regression-bisect/repro.cjs create mode 100644 eng/regression-bisect/repro.smoke.cjs create mode 100644 eng/regression-bisect/repro.test.cjs diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index c9789b0e1e2..2f0419eaaf3 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -327,6 +327,158 @@ was regenerated through its existing harness for the new pinned public-feed policy; an old-policy checkpoint is not silently accepted as a new-policy one. Root compiler Release builds/tests are not this Node adapter's acceptance gate. +## Reproduction execution + +`repro.cjs` is an execution adapter, not an authorization boundary. Its caller +must first admit the frozen manifest and place execution in a disposable, +credential-free worker. It does not fetch issue code, choose revisions, bisect, +or publish results. A manifest digest alone is not approval. + +The exports are `runReproduction`, `selectFsiCommand`, +`verifyProjectIdentities`, and `evaluateAssertion`. Outcomes and the two-run +consistency rule come from `model.cjs`. A successful build never substitutes +for the runtime assertion. + +`runReproduction` takes: + +* `manifest`, `sourceDirectory` containing exactly the reviewed source assets, + and `workDirectory` for fresh per-observation/per-attempt workspaces; +* `toolchain`: the verified `identity` returned by `acquireToolchain`; +* `product`: `{ kind, root, sourceSha, buildInstanceId, configuration, + runtime: { name, version }, corePackageVersion, files }`. `kind` is `source` + or `released`; local `root` is the clean build checkout, not this adapter's + checkout. Each inventoried file has `role`, absolute `path`, and `sha256`; + assemblies also have `assemblyVersion`. Roles are `fsi`, `compiler`, + `tasks`, `props`, `targets`, `sdkTargets`, `overrides`, `core`, and + `corePackage`. Managed FSI/compiler records carry + `runtimeConfig: { path, sha256 }`. There must be exactly one selected + artifact per role: this adapter does not search for the newest build; +* `helper`: `{ executable, runtimeVersion, dll, hook }`, identifying the + separately built evidence reader and Core-load startup hook; +* optional `sampleRuntimes`, mapping each TFM to an explicitly acquired + `{ name, version }`. Omission uses the product's explicit runtime, never + automatic major-version roll-forward. + +The adapter checks inventory bytes and assembly versions, then rechecks frozen +inputs and product bytes after execution. Local FSI uses the selected `fsi.dll` +with an absolute acquired host and `exec --fx-version`; it never invokes +`dotnet fsi`. Windows executable layouts use a separate fixed Core-load probe. +Managed executions capture the actual loaded Core through a startup hook, +including its path, digest, assembly version, runtime version and architecture. +Unsupported host/runtime/helper combinations are untestable. + +Project builds reuse **both** `UseLocalCompiler.Directory.Build` imports. +The local compiler TFM comes from the inventoried runtimeconfig, not this +checkout's current product TFM. Each repetition has a distinct `NUGET_PACKAGES`; +same-version packages from different builds cannot share an extraction cache. +Core package bytes and the selected compile/runtime DLL entries are checked, +not just their version strings. Other packages must match the manifest's +frozen versions and package digests. + +MSBuild's compiler child uses an explicit `LatestPatch` policy only when there +is exactly one compatible, acquired runtime in that host. PID-separated startup +hook records must prove that the actual `fsc.dll` process loaded that runtime's +CoreLib bytes and the inventoried compiler Core. This permits a pinned compatible +prerelease runtime without editing the built runtimeconfig or silently selecting +another installed runtime. FSI and application execution use `--fx-version`. + +`ReproEvidence.targets` contributes build-context evidence; that evidence alone +is insufficient. The pinned structured binlog reader must find the executed +Fsc task and correlate its assembly, command, references and flags with imports, +assets and the expected project/TFM/configuration cell. Missing inner builds, +skipped Fsc tasks, fallbacks and late flag overrides are untestable. + +Assertions compare complete, bounded streams and exit codes. Line endings are +normalized to LF. Compile-time assertions use a frozen diagnostic of the form +`Program.fs(line,column[,endLine,endColumn]): error FSnnnn: message`, including +the relevant message and span. The reader supplies structured diagnostics with +workspace-relative paths; unrelated diagnostics or restore errors cannot +satisfy the assertion. All other manifests execute the produced application. +Timeouts terminate the process tree; truncated output, unexpected crashes, +missing identity evidence and mixed repetitions are untestable. + +An immutable producer is declared in a reviewed `*.producer.json` source asset: +`{ role: "immutable-producer", compiler: { version, sha256 }, flags, tfm, +packages, dll: { path, sha256, resources }, package: { path, id, version, +sha256 } }`. The DLL and optional package are themselves reviewed source +assets; `manifest.producer.sha256` identifies the DLL. Resource records contain +the F# metadata resource names and digests. Consumers must reference those bytes, +not rebuild a producer ProjectReference. An explicit absence of producers is +`{ name: "no-producer", version: "0.0.0", sha256: SHA256(empty bytes) }`. +Producer dependency provenance is retained separately from the candidate's Core; +the adapter never rewrites or repacks a frozen producer. + +`result.json` retains outcomes, attempts, command arrays and evidence paths. +`identities.json` retains the exact product/host/SDK/runtime/package identities +and the acquisition digest. Binlogs and loaded-Core evidence remain alongside +each attempt. These are execution records, not authenticated controller +observations or publication-ready issue text. + +### Bounded reproduction smoke + +The opt-in `repro.smoke.cjs ` +entrypoint generates only small synthetic fixtures. The configuration has +`released` and `source` objects containing `{ product, toolchain, tfm, +sampleRuntimes? }`, plus the shared `helper` above. Supply an acquired released +SDK and products from a separate clean checkout built with: + +```powershell +eng\Build.ps1 -restore -build -c Release -pack -bootstrap -noVisualStudio -msbuildEngine dotnet +``` + +Use an explicit credential-free environment for that build. Do not reuse this +worktree's product artifacts or run competing builds in the temporary checkout. +Windows requires static `OS=Windows_NT` and `PATHEXT=.COM;.EXE;.BAT;.CMD`; +NuGet also needs private `ProgramFiles`/`ProgramFiles(x86)` paths. Do not copy +the user's credentials or machine-wide NuGet configuration to satisfy these. +Build the reader with the explicit +`eng\regression-bisect\evidence\Evidence.csproj` project. Its pinned +`MSBuild.StructuredLogger` dependency reads binlogs, and its dependency-free +`netstandard2.0` hook can run on older Core runtimes independently of the +reader's .NET 8+ host. + +The smoke covers FSI, Debug/Release projects, successful compilation followed by +an intended runtime failure, intended diagnostics, late compiler flag overrides, +explicit/central Core package overrides, two inner TFMs on Windows, and an immutable released +producer. Its source-evaluation control illustrates how rebuilding a producer +without its old flags would hide the failure; candidate consumers only receive +the frozen DLL/package. Results are written to `smoke-results.json`. +An explicit final `released` or `source` argument selects just that phase while +preparing the other product; a released-only pass is not completion of the +clean-local-product acceptance gate. + +Scoped checks (a full compiler test run does not prove injection): + +```powershell +node --test eng\regression-bisect\model.test.cjs eng\regression-bisect\toolchains.test.cjs eng\regression-bisect\repro.test.cjs +node eng\regression-bisect\repro.smoke.cjs +git diff --check +``` + +Syntax-check changed `.cjs` files with `node --check` as well. Do not report +the real smoke as passing based on mocked process output or Node tests. + +Validation recorded on 2026-09-19: all **466** scoped Node tests passed. The +prescribed clean Release/bootstrap/pack build completed with exit 0 at source +`553101960fcb8551d305fc5c0dabad9f2a3e70f3`. All **21** bounded fixture scenarios +matched their expected outcomes, including both deliberate late-override +rejections; normal assertions were repeated twice and both declared inner TFMs +were verified. The reproduction SDK was **10.0.401**, with runtime **10.0.12** +for released FSI/applications and explicitly acquired runtime +**11.0.0-rc.1.26425.128** for the clean local FSI/compiler. Local Core package +**11.0.101** and the released producer's DLL, package and compressed F# metadata +were preserved and checked by digest. + +The clean build itself used the repository entrypoint's pinned SDK +`11.0.100-rc.1.26420.103`. Its unavailable archive checksum sidecar was recorded +as a separate acquisition failure, not converted into an acquisition success: +the source reproduction instead used the independently acquired SDK/runtime +above, both of which passed the existing acquisition identity guard. +Complete execution evidence is retained at `Q:\rb912-s4-validated-fixtures`; +`smoke-results.json` SHA-256 is +`b182f1a3c504a9d4da94b0cfda10cde5e5280c0e20deb89338160038e3991ff1`. +These checks do not imply a full compiler-test-suite pass. + ## Complete intake and bounded discovery `intake.cjs` exports `readIssueSnapshot(options)`, diff --git a/eng/regression-bisect/ReproEvidence.targets b/eng/regression-bisect/ReproEvidence.targets new file mode 100644 index 00000000000..fc0be5a1a0b --- /dev/null +++ b/eng/regression-bisect/ReproEvidence.targets @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + diff --git a/eng/regression-bisect/evidence/AssetEvidence.cs b/eng/regression-bisect/evidence/AssetEvidence.cs new file mode 100644 index 00000000000..842d9ef9c0a --- /dev/null +++ b/eng/regression-bisect/evidence/AssetEvidence.cs @@ -0,0 +1,76 @@ +using System.Diagnostics; +using System.IO.Compression; +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; +using System.Security.Cryptography; +using System.Xml.Linq; + +namespace ReproEvidence; + +internal static class AssetEvidence +{ + internal static string Hash(Stream stream) => Convert.ToHexString(SHA256.HashData(stream)).ToLowerInvariant(); + + internal static object Inspect(string file) + { + string path = Path.GetFullPath(file); + using var stream = File.OpenRead(path); + string sha256 = Hash(stream); + stream.Position = 0; + if (Path.GetExtension(path).Equals(".nupkg", StringComparison.OrdinalIgnoreCase)) + { + using var zip = new ZipArchive(stream, ZipArchiveMode.Read); + var manifest = zip.Entries.Single(e => !e.FullName.Contains('/') && e.FullName.EndsWith(".nuspec", StringComparison.OrdinalIgnoreCase)); + using var manifestStream = manifest.Open(); + var metadata = XDocument.Load(manifestStream).Root!.Elements().Single(e => e.Name.LocalName == "metadata"); + string Field(string name) => metadata.Elements().Single(e => e.Name.LocalName == name).Value; + var entries = zip.Entries.Where(e => !e.FullName.EndsWith('/')).Select(e => + { + using var contents = e.Open(); + return new { path = e.FullName, sha256 = Hash(contents) }; + }).ToArray(); + return new { path, sha256, id = Field("id"), version = Field("version"), entries }; + } + + string? assemblyName = null, assemblyVersion = null; + var resources = new List(); + // Imports and runtime configuration files need byte identity too, but are not PE files. + bool isPe = stream.ReadByte() == 'M' && stream.ReadByte() == 'Z'; + stream.Position = 0; + using var pe = isPe ? new PEReader(stream) : null; + if (pe?.HasMetadata == true) + { + var reader = pe.GetMetadataReader(); + if (reader.IsAssembly) + { + var definition = reader.GetAssemblyDefinition(); + assemblyName = reader.GetString(definition.Name); + assemblyVersion = definition.Version.ToString(); + } + foreach (var handle in reader.ManifestResources) + { + var resource = reader.GetManifestResource(handle); + string name = reader.GetString(resource.Name); + if (!resource.Implementation.IsNil) + { + resources.Add(new { name, sha256 = (string?)null, embedded = false }); + continue; + } + var block = pe.GetSectionData(pe.PEHeaders.CorHeader!.ResourcesDirectory.RelativeVirtualAddress); + var content = block.GetReader(checked((int)resource.Offset), block.Length - checked((int)resource.Offset)); + int length = content.ReadInt32(); + string hash = Convert.ToHexString(SHA256.HashData(content.ReadBytes(length))).ToLowerInvariant(); + resources.Add(new { name, sha256 = hash, embedded = true }); + } + } + return new + { + path, + sha256, + assemblyName, + assemblyVersion, + fileVersion = FileVersionInfo.GetVersionInfo(path).FileVersion, + resources + }; + } +} diff --git a/eng/regression-bisect/evidence/BinlogEvidence.cs b/eng/regression-bisect/evidence/BinlogEvidence.cs new file mode 100644 index 00000000000..75be3400957 --- /dev/null +++ b/eng/regression-bisect/evidence/BinlogEvidence.cs @@ -0,0 +1,286 @@ +using System.Text.RegularExpressions; +using Microsoft.Build.Logging.StructuredLogger; +using BuildTask = Microsoft.Build.Logging.StructuredLogger.Task; + +namespace ReproEvidence; + +internal static class BinlogEvidence +{ + private const string SnapshotPrefix = "REPRO_EVIDENCE_V1|"; + + internal static object Read(string file) + { + var build = BinaryLog.ReadBuild(file); + var cells = new List(); + build.VisitAllChildren(project => + { + if (!project.ProjectFile.EndsWith(".fsproj", StringComparison.OrdinalIgnoreCase) || project.IsOuterProject) + return; + var tasks = Descendants(project).Where(t => t.Name == "Fsc" && Owner(t) == project).ToArray(); + var snapshots = Descendants(project) + .Where(m => Owner(m) == project && m.Text.StartsWith(SnapshotPrefix, StringComparison.Ordinal)) + .Select(m => m.Text[SnapshotPrefix.Length..].Split('|', 2)) + .Where(parts => parts.Length == 2).ToArray(); + if (tasks.Length == 0 && snapshots.Length == 0 && !project.Children.OfType().Any(t => t.Name == "CoreCompile")) + return; + + var evaluation = build.FindEvaluation(project.EvaluationId); + var properties = evaluation?.GetProperties() ?? new Dictionary(); + var snapshot = snapshots.Where(p => p[0] != "ProjectReference") + .GroupBy(p => p[0]).ToDictionary(g => g.Key, g => g.Last()[1]); + string? Value(string name) => snapshot.GetValueOrDefault(name) ?? properties.GetValueOrDefault(name); + var imports = evaluation == null ? [] : Descendants(evaluation) + .Select(i => i.ImportedProjectFilePath).Where(p => !string.IsNullOrEmpty(p)) + .Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); + var fscTasks = tasks.Select(t => ReadTask(t, project.ProjectDirectory)).ToArray(); + var diagnostics = new List(); + if (evaluation == null) diagnostics.Add("Matching project evaluation is missing."); + if (!imports.Any(p => Path.GetFileName(p).Contains("FSharp", StringComparison.OrdinalIgnoreCase) + && p.EndsWith(".props", StringComparison.OrdinalIgnoreCase)) + || !imports.Any(p => Path.GetFileName(p).Contains("FSharp", StringComparison.OrdinalIgnoreCase) + && p.EndsWith(".targets", StringComparison.OrdinalIgnoreCase))) + diagnostics.Add("Evaluated F# props/targets imports are missing."); + if (fscTasks.Length == 0) diagnostics.Add("No Fsc task executed (possibly an incremental or design-time build)."); + if (fscTasks.Any(t => !t.Complete)) diagnostics.Add("At least one Fsc task has incomplete execution evidence."); + if (string.Equals(Value("DesignTimeBuild"), "true", StringComparison.OrdinalIgnoreCase) + || string.Equals(Value("BuildingInsideVisualStudio"), "true", StringComparison.OrdinalIgnoreCase)) + diagnostics.Add("IDE/host-object compilation cannot prove external compiler execution."); + if (!snapshot.ContainsKey("OtherFlags")) + diagnostics.Add("The pre-compilation OtherFlags snapshot is missing."); + var expected = ReadFlags( + SplitArguments(snapshot.GetValueOrDefault("OtherFlags") ?? "", diagnostics), + string.IsNullOrWhiteSpace(snapshot.GetValueOrDefault("LangVersion")) ? "default" : snapshot["LangVersion"], + bool.TryParse(snapshot.GetValueOrDefault("Optimize"), out bool optimize) ? optimize : null, + snapshot.GetValueOrDefault("DefineConstants")?.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); + + var cell = new Cell + { + Project = project.ProjectFile, + EvaluationId = project.EvaluationId, + ProjectInstanceId = project.ProjectInstanceId, + TargetFramework = Value("TargetFramework") ?? project.TargetFramework, + Configuration = Value("Configuration") ?? project.Configuration, + LanguageVersion = fscTasks.LastOrDefault()?.LanguageVersion, + Optimize = fscTasks.LastOrDefault()?.Optimize, + Defines = fscTasks.LastOrDefault()?.Defines ?? [], + ExpectedLanguageVersion = expected.LanguageVersion, + ExpectedOptimize = expected.Optimize, + ExpectedDefines = expected.Defines, + SdkVersion = Value("NETCoreSdkVersion"), + HostPath = fscTasks.LastOrDefault()?.Executable, + AssetsFile = Value("ProjectAssetsFile"), + TargetPath = Value("TargetPath"), + Imports = imports, + ProjectReferences = snapshots.Where(p => p[0] == "ProjectReference") + .Select(p => p[1]).Distinct(StringComparer.OrdinalIgnoreCase).ToArray(), + Snapshot = snapshot, + FscTasks = fscTasks, + Diagnostics = diagnostics + }; + if (string.IsNullOrEmpty(cell.TargetFramework) || string.IsNullOrEmpty(cell.Configuration) + || string.IsNullOrEmpty(cell.SdkVersion) || string.IsNullOrEmpty(cell.AssetsFile) + || string.IsNullOrEmpty(cell.TargetPath) || string.IsNullOrEmpty(cell.HostPath)) + diagnostics.Add("Project/TFM/configuration/SDK/host/assets/output context is incomplete."); + cell.Complete = diagnostics.Count == 0; + cells.Add(cell); + }); + return new + { + schemaVersion = 1, + binlog = Path.GetFullPath(file), + buildSucceeded = build.Succeeded, + complete = cells.Count > 0 && cells.All(c => c.Complete), + diagnostics = Descendants(build).Select(Diagnostic).ToArray(), + cells + }; + } + + private static FscRecord ReadTask(BuildTask task, string directory) + { + var parameters = new Dictionary(StringComparer.OrdinalIgnoreCase); + var folder = task.Children.OfType().FirstOrDefault(f => f.Name == "Parameters"); + if (folder != null) + foreach (var node in folder.Children) + if (node is Property property) + parameters[property.Name] = [property.Value]; + else if (node is Parameter parameter) + parameters[parameter.ParameterName ?? parameter.Name] = parameter.Children.OfType().Select(i => i.Text).ToArray(); + + var record = new FscRecord + { + TaskAssembly = task.FromAssembly, + TargetsPath = task.SourceFilePath, + CommandLine = task.CommandLineArguments, + Parameters = parameters, + Diagnostics = Descendants(task).Select(Diagnostic).ToArray() + }; + string? toolPath = record.Parameter("ToolPath"), toolExe = record.Parameter("ToolExe"); + string? expectedTool = toolPath != null && toolExe != null ? Path.Combine(toolPath.Trim('"'), toolExe.Trim('"')) : null; + // Fsc overrides ToolTask.LogToolCommand with a regular Message event rather + // than TaskCommandLineEventArgs. Accept only a message naming the actual tool. + if (string.IsNullOrEmpty(record.CommandLine) && expectedTool != null) + record.CommandLine = Descendants(task).Select(m => m.Text) + .FirstOrDefault(m => m.StartsWith(expectedTool + " ", StringComparison.OrdinalIgnoreCase) + || m.StartsWith('"' + expectedTool + "\" ", StringComparison.OrdinalIgnoreCase)); + string commandPattern = expectedTool != null + ? "^\\s*(\"?" + Regex.Escape(expectedTool) + "\"?)\\s+([\\s\\S]*)$" + : "^\\s*(\"[^\"]+\"|\\S+)\\s+([\\s\\S]*)$"; + var command = Regex.Match(record.CommandLine ?? "", commandPattern, RegexOptions.IgnoreCase); + if (command.Success) + { + record.Executable = command.Groups[1].Value.Trim('"'); + record.Arguments = command.Groups[2].Value; + } + record.CompilerPath = record.Parameter("DotnetFscCompilerPath")?.Trim('"'); + if (string.IsNullOrEmpty(record.CompilerPath) && Path.GetFileNameWithoutExtension(record.Executable) == "fsc") + record.CompilerPath = record.Executable; + string arguments = record.Arguments ?? ""; + if (record.CompilerPath != null && record.CompilerPath != record.Executable) + { + var compiler = Regex.Match(arguments, "^\"?" + Regex.Escape(record.CompilerPath) + "\"?\\s+", RegexOptions.IgnoreCase); + if (compiler.Success) arguments = arguments[compiler.Length..]; + else record.MissingEvidence.Add("Compiler task parameter does not match the executed command."); + } + // Fsc logs its expanded response file one argument per line, including spaces + // in unquoted paths. Older single-line commands instead need quote-aware splitting. + string[] tokens; + if (arguments.Contains('\n')) + tokens = arguments.Split('\n').Select(t => t.Trim()).Where(t => t.Length > 0).ToArray(); + else + tokens = SplitArguments(arguments, record.MissingEvidence); + (record.LanguageVersion, record.Optimize, record.Defines) = ReadFlags(tokens); + var references = new List(); + foreach (string token in tokens) + { + if (token.StartsWith('@') || Regex.IsMatch(token, "^(?:--lib|-I|/lib|/I)(:|$)")) + record.MissingEvidence.Add("A response file or library-search override prevents reference resolution."); + var reference = Regex.Match(token, "^(?:--reference|-r|/reference|/r)(?::(.*))?$"); + if (!reference.Success) continue; + string value = reference.Groups[1].Value; + try + { + if (value.StartsWith('"') && value.EndsWith('"')) value = value[1..^1]; + if (string.IsNullOrWhiteSpace(value) || value.Contains('"')) + throw new ArgumentException("Reference path is empty or has unsupported quoting."); + string path = Path.GetFullPath(value, directory); + if (!Path.IsPathFullyQualified(value) && !File.Exists(path)) + throw new ArgumentException("Relative reference does not resolve in the project directory."); + references.Add(path); + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException) + { + record.MissingEvidence.Add($"Cannot resolve compiler reference '{value}': {ex.Message}"); + } + } + record.References = references.ToArray(); + foreach (string reference in parameters.GetValueOrDefault("References", [])) + if (!references.Contains(Path.GetFullPath(reference.Trim('"'), directory), StringComparer.OrdinalIgnoreCase)) + record.MissingEvidence.Add($"Typed task reference is absent from the compiler command: {reference}"); + if (!Path.IsPathFullyQualified(record.TaskAssembly ?? "")) record.MissingEvidence.Add("Actual Fsc task assembly path is missing."); + if (!Path.IsPathFullyQualified(record.TargetsPath ?? "")) record.MissingEvidence.Add("Executed Fsc task definition path is missing."); + if (!Path.IsPathFullyQualified(record.CompilerPath ?? "")) record.MissingEvidence.Add("Actual compiler path is missing."); + if (string.IsNullOrEmpty(record.CommandLine) || record.Executable == null) + record.MissingEvidence.Add("Actual Fsc tool command message is missing."); + if (parameters.Count == 0) record.MissingEvidence.Add("Fsc task inputs are missing."); + if (!record.References.Any(p => Path.GetFileName(p).Equals("FSharp.Core.dll", StringComparison.OrdinalIgnoreCase))) + record.MissingEvidence.Add("FSharp.Core compile reference is missing."); + if (string.Equals(record.Parameter("SkipCompilerExecution"), "true", StringComparison.OrdinalIgnoreCase)) + record.MissingEvidence.Add("Fsc explicitly skipped compiler execution."); + if (task.EndTime == default) record.MissingEvidence.Add("Fsc task completion is missing."); + record.Complete = record.MissingEvidence.Count == 0; + return record; + } + + private static string[] SplitArguments(string arguments, List diagnostics) + { + var matches = Regex.Matches(arguments, "\\G\\s*((?:[^\\s\"]+|\"[^\"]*\")+)"); + int consumed = matches.LastOrDefault() is { } last ? last.Index + last.Length : 0; + if (!string.IsNullOrWhiteSpace(arguments[consumed..]) || arguments.Contains("\\\"", StringComparison.Ordinal)) + diagnostics.Add("Compiler arguments contain unsupported or unterminated quoting."); + return matches.Select(m => m.Groups[1].Value.Replace("\"", "")).ToArray(); + } + + private static (string LanguageVersion, bool? Optimize, string[] Defines) ReadFlags( + string[] tokens, string languageVersion = "default", bool? optimize = null, string[]? defines = null) + { + var languageFlag = tokens.Select(t => Regex.Match(t, "^(?:--|/)langversion:(.*)$")).LastOrDefault(m => m.Success); + var optimizeFlag = tokens.Select(t => Regex.Match(t, "^(?:--optimize|/optimize|-O|/O)([+-]?)$")).LastOrDefault(m => m.Success); + var defineFlags = tokens.Select(t => Regex.Match(t, "^(?:--define|-d|/define|/d):(.*)$")) + .Where(m => m.Success).Select(m => m.Groups[1].Value); + return ( + languageFlag?.Groups[1].Value.Trim('"') ?? languageVersion, + optimizeFlag == null ? optimize : optimizeFlag.Groups[1].Value != "-", + (defines ?? []).Concat(defineFlags).Distinct(StringComparer.Ordinal).ToArray()); + } + + private static Project? Owner(BaseNode node) + { + for (var parent = node.Parent; parent != null; parent = parent.Parent) + if (parent is Project project) return project; + return null; + } + + private static List Descendants(TreeNode node) where T : BaseNode + { + var result = new List(); + node.VisitAllChildren(result.Add); + return result; + } + + private static object Diagnostic(AbstractDiagnostic diagnostic) => new + { + severity = diagnostic is Error ? "error" : "warning", + code = diagnostic.Code, + message = diagnostic.Text, + file = diagnostic.File, + line = diagnostic.LineNumber, + column = diagnostic.ColumnNumber, + endLine = diagnostic.EndLineNumber, + endColumn = diagnostic.EndColumnNumber + }; + + private sealed class Cell + { + public required string Project { get; init; } + public int EvaluationId { get; init; } + public int ProjectInstanceId { get; init; } + public string? TargetFramework { get; init; } + public string? Configuration { get; init; } + public string? LanguageVersion { get; init; } + public bool? Optimize { get; init; } + public string[] Defines { get; init; } = []; + public string? ExpectedLanguageVersion { get; init; } + public bool? ExpectedOptimize { get; init; } + public string[] ExpectedDefines { get; init; } = []; + public string? SdkVersion { get; init; } + public string? HostPath { get; init; } + public string? AssetsFile { get; init; } + public string? TargetPath { get; init; } + public string[] Imports { get; init; } = []; + public string[] ProjectReferences { get; init; } = []; + public required Dictionary Snapshot { get; init; } + public required FscRecord[] FscTasks { get; init; } + public required List Diagnostics { get; init; } + public bool Complete { get; set; } + } + + private sealed class FscRecord + { + public string? TaskAssembly { get; init; } + public string? TargetsPath { get; init; } + public string? CompilerPath { get; set; } + public string? CommandLine { get; set; } + public string? Executable { get; set; } + public string? HostPath => Executable; + public string? Arguments { get; set; } + public string? LanguageVersion { get; set; } + public bool? Optimize { get; set; } + public string[] Defines { get; set; } = []; + public required Dictionary Parameters { get; init; } + public string[] References { get; set; } = []; + public object[] Diagnostics { get; init; } = []; + public List MissingEvidence { get; } = []; + public bool Complete { get; set; } + internal string? Parameter(string name) => Parameters.GetValueOrDefault(name)?.LastOrDefault(); + } +} diff --git a/eng/regression-bisect/evidence/Evidence.csproj b/eng/regression-bisect/evidence/Evidence.csproj new file mode 100644 index 00000000000..09c61e83ebb --- /dev/null +++ b/eng/regression-bisect/evidence/Evidence.csproj @@ -0,0 +1,22 @@ + + + false + false + false + + + + Exe + net8.0 + enable + enable + false + false + + + + + + + + diff --git a/eng/regression-bisect/evidence/Program.cs b/eng/regression-bisect/evidence/Program.cs new file mode 100644 index 00000000000..ce6c9e23179 --- /dev/null +++ b/eng/regression-bisect/evidence/Program.cs @@ -0,0 +1,27 @@ +using System.Text.Json; +using ReproEvidence; + +try +{ + object result = args.Length >= 2 ? args[0] switch + { + "binlog" when args.Length == 2 => BinlogEvidence.Read(args[1]), + "inspect" => args.Skip(1).Select(AssetEvidence.Inspect).ToArray(), + _ => throw new ArgumentException("Usage: Evidence binlog | inspect ") + } : throw new ArgumentException("Usage: Evidence binlog | inspect "); + Console.WriteLine(JsonSerializer.Serialize(result, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + })); + return 0; +} +catch (Exception ex) +{ + Console.WriteLine(JsonSerializer.Serialize(new + { + schemaVersion = 1, + complete = false, + diagnostics = new[] { $"{ex.GetType().Name}: {ex.Message}" } + })); + return 1; +} diff --git a/eng/regression-bisect/evidence/StartupHook/StartupHook.cs b/eng/regression-bisect/evidence/StartupHook/StartupHook.cs new file mode 100644 index 00000000000..856fd661d76 --- /dev/null +++ b/eng/regression-bisect/evidence/StartupHook/StartupHook.cs @@ -0,0 +1,92 @@ +using System.Diagnostics; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Runtime.Serialization.Json; +using System.Security.Cryptography; +using System.Text; + +// Startup hooks require this name in the global namespace. A runtime without startup +// hook support or a missing output is untestable, never evidence of a Core load. +public static class StartupHook +{ + private static readonly object Gate = new(); + private static readonly HashSet Seen = new(); + private static string? destination; + + public static void Initialize() + { + // The runner supplies a fresh path; JSONL preserves distinct Core load contexts. + destination = Environment.GetEnvironmentVariable("REPRO_CORE_EVIDENCE"); + if (string.IsNullOrEmpty(destination)) + return; + using var process = Process.GetCurrentProcess(); + destination = destination!.Replace("{pid}", process.Id.ToString(System.Globalization.CultureInfo.InvariantCulture)); + AppDomain.CurrentDomain.AssemblyLoad += (_, args) => Record(args.LoadedAssembly); + foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies()) + Record(assembly); + } + + private static void Record(Assembly assembly) + { + if (assembly.GetName().Name != "FSharp.Core") + return; + lock (Gate) + { + if (!Seen.Add(assembly)) + return; + try + { + string path = assembly.Location; + var runtimeDirectory = Directory.GetParent(typeof(object).Assembly.Location); + if (runtimeDirectory?.Parent?.Name != "Microsoft.NETCore.App") + throw new InvalidOperationException("The loaded CoreLib has no shared-framework version directory."); + using var contents = File.OpenRead(path); + using var hash = SHA256.Create(); + using var process = Process.GetCurrentProcess(); + File.AppendAllText(destination!, Serialize(new Dictionary + { + ["schemaVersion"] = 1, + ["complete"] = true, + ["path"] = path, + ["sha256"] = BitConverter.ToString(hash.ComputeHash(contents)).Replace("-", "").ToLowerInvariant(), + ["assemblyName"] = assembly.GetName().Name, + ["assemblyVersion"] = assembly.GetName().Version?.ToString(), + ["fileVersion"] = FileVersionInfo.GetVersionInfo(path).FileVersion, + ["entryAssembly"] = Assembly.GetEntryAssembly()?.Location, + ["processId"] = process.Id, + ["architecture"] = RuntimeInformation.ProcessArchitecture.ToString().ToLowerInvariant(), + ["runtimeVersion"] = runtimeDirectory.Name, + ["runtimePath"] = runtimeDirectory.FullName + }) + Environment.NewLine); + } + catch (Exception ex) + { + // Observation must not change application behavior. A missing/incomplete + // report is rejected by the runner, including pathless single-file loads. + try + { + File.AppendAllText(destination!, Serialize(new Dictionary + { + ["schemaVersion"] = 1, + ["complete"] = false, + ["diagnostics"] = new[] { $"{ex.GetType().Name}: {ex.Message}" } + }) + Environment.NewLine); + } + catch (Exception writeError) + { + var message = $"{writeError.GetType().Name}: {writeError.Message}".Replace('\r', ' ').Replace('\n', ' '); + Console.Error.WriteLine("REPRO_CORE_EVIDENCE_ERROR: " + message.Substring(0, Math.Min(512, message.Length))); + } + } + } + } + + private static string Serialize(Dictionary record) + { + var serializer = new DataContractJsonSerializer(typeof(Dictionary), + new DataContractJsonSerializerSettings { UseSimpleDictionaryFormat = true, KnownTypes = new[] { typeof(string[]) } }); + using var stream = new MemoryStream(); + serializer.WriteObject(stream, record); + return Encoding.UTF8.GetString(stream.ToArray()); + } +} diff --git a/eng/regression-bisect/evidence/StartupHook/StartupHook.csproj b/eng/regression-bisect/evidence/StartupHook/StartupHook.csproj new file mode 100644 index 00000000000..33fe69db51b --- /dev/null +++ b/eng/regression-bisect/evidence/StartupHook/StartupHook.csproj @@ -0,0 +1,17 @@ + + + false + false + false + + + + netstandard2.0 + 12.0 + Repro.StartupHook + enable + enable + false + + + diff --git a/eng/regression-bisect/fixtures/reproduction.cjs b/eng/regression-bisect/fixtures/reproduction.cjs new file mode 100644 index 00000000000..0692c5a6977 --- /dev/null +++ b/eng/regression-bisect/fixtures/reproduction.cjs @@ -0,0 +1,75 @@ +const fs = require("node:fs/promises"); +const os = require("node:os"); +const path = require("node:path"); +const { hash } = require("../toolchain-io.cjs"); +const { manifest, seal } = require("./contracts.cjs"); + +async function fixture(t, { kind = "source", executable = false, tfms = ["net10.0"], configuration = "Release" } = {}) { + const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "repro-contract-"))); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const file = async (relative, contents = relative) => { + const location = path.join(root, relative); + await fs.mkdir(path.dirname(location), { recursive: true }); + await fs.writeFile(location, contents); + return { path: await fs.realpath(location), sha256: hash(contents) }; + }; + const sdkRoot = path.join(root, "dotnet"); + const host = await file("dotnet/dotnet.exe"); + const sdkBasePath = path.join(sdkRoot, "sdk", "10.0.100"); + const runtime = await file("dotnet/shared/Microsoft.NETCore.App/10.0.0/System.Private.CoreLib.dll"); + const productRoot = kind === "source" ? path.join(root, "local") : path.join(sdkBasePath, "FSharp"); + const prefix = path.relative(root, productRoot); + const files = []; + for (const [role, name] of Object.entries({ + fsi: executable ? "fsi.exe" : "fsi.dll", compiler: "fsc.dll", tasks: "FSharp.Build.dll", + props: "Microsoft.FSharp.NetSdk.props", targets: "Microsoft.FSharp.Targets", + sdkTargets: "Microsoft.FSharp.NetSdk.targets", overrides: "Microsoft.FSharp.Overrides.NetSdk.targets", + core: "FSharp.Core.dll", corePackage: "FSharp.Core.10.0.100.nupkg", + })) { + files.push({ role, ...await file(path.join(prefix, name)), ...(/\.(dll|exe)$/.test(name) ? { assemblyVersion: "10.0.0.0" } : {}) }); + } + const config = await file(path.join(prefix, "fsi.runtimeconfig.json"), JSON.stringify({ + runtimeOptions: { tfm: "net10.0", framework: { name: "Microsoft.NETCore.App", version: "10.0.0" } }, + })); + files.find(f => f.role === "fsi").runtimeConfig = config; + const m = manifest(); + m.sources = [{ path: "repro.fsx", ...(await file("source/repro.fsx", 'printfn "ok"')) }]; + m.sources = m.sources.map(s => ({ path: "repro.fsx", sha256: s.sha256 })); + m.packages = []; + m.producer = { name: "no-producer", version: "0.0.0", sha256: hash("") }; + m.matrix = tfms.map((tfm, i) => ({ + id: `cell-${i}`, os: "windows", architecture: "x64", project: null, tfm, configuration, + languageVersion: "10.0", command: ["fsi", "repro.fsx"], + })); + seal(m); + const product = { + kind, root: productRoot, sourceSha: "a".repeat(40), buildInstanceId: "clean-build", + runtime: { name: "Microsoft.NETCore.App", version: "10.0.0" }, files, + corePackageVersion: "10.0.100", + }; + const toolchain = { + root: sdkRoot, hostPath: host.path, hostSha256: host.sha256, sdkVersion: "10.0.100", sdkBasePath, + architecture: "x64", rid: "win-x64", + runtimes: [{ name: "Microsoft.NETCore.App", version: "10.0.0", + path: path.dirname(path.dirname(runtime.path)), sha256: runtime.sha256 }], + }; + const url = "https://builds.dotnet.microsoft.com/dotnet/Sdk/10.0.100/dotnet-sdk-10.0.100-win-x64.zip"; + toolchain.archive = { url, redirectUrl: url, sha512: "a".repeat(128), actualSha512: "a".repeat(128), sha256: hash("archive") }; + toolchain.requested = { + kind: "released", platform: { os: "windows", architecture: "x64", rid: "win-x64" }, + installationRoot: sdkRoot, sdkVersion: "10.0.100", toolsVersion: "10.0.100", + bundledRuntime: { version: "10.0.0" }, runtimeRequirements: [], archive: toolchain.archive, + }; + toolchain.sdks = [{ version: "10.0.100", path: path.join(sdkRoot, "sdk") }]; + toolchain.compilerRuntime = { name: "Microsoft.NETCore.App", version: "10.0.0" }; + toolchain.files = {}; + for (const [name, assemblyName] of Object.entries({ compiler: "fsc", fsi: "fsi", core: "FSharp.Core" })) { + toolchain.files[name] = { + ...await file(path.join("dotnet", "sdk", "10.0.100", "FSharp", `${assemblyName}.dll`)), + assemblyName, assemblyVersion: "10.0.0.0", fileVersion: "10.0.0", + }; + } + return { root, file, product, toolchain, manifest: m, sourceDirectory: path.join(root, "source") }; +} + +module.exports = { fixture }; diff --git a/eng/regression-bisect/fixtures/smoke.cjs b/eng/regression-bisect/fixtures/smoke.cjs new file mode 100644 index 00000000000..13c428eaf1d --- /dev/null +++ b/eng/regression-bisect/fixtures/smoke.cjs @@ -0,0 +1,111 @@ +const fs = require("node:fs/promises"); +const path = require("node:path"); +const { hash } = require("../toolchain-io.cjs"); +const { fingerprintInput } = require("../model.cjs"); + +async function createFixture(directory, { kind, tfm, configuration = "Release", scenario = "pass", producer, coreVersion }) { + await fs.mkdir(directory, { recursive: true }); + const contents = new Map(); + const failing = scenario === "runtime-failure"; + const producerSource = [ + "#if OLD_PRODUCER", + 'let inline value () = "OLD_METADATA"', + "#else", + 'let inline value () = "NEW_METADATA"', + "#endif", + ].join("\n"); + const body = producer + ? 'if FrozenProducer.value () = "OLD_METADATA" then\n eprintfn "FROZEN_RUNTIME_FAILURE"\n 1\nelse\n printfn "PASS"\n 0' + : failing ? 'eprintfn "FROZEN_RUNTIME_FAILURE"\n1' : 'printfn "PASS"\n0'; + if (kind === "fsi") contents.set("repro.fsx", scenario === "producer-control" + ? `${producerSource}\nprintfn "%s" (value ())\n` + : `let result =\n${body.split("\n").map(l => ` ${l}`).join("\n")}\nexit result\n`); + else { + contents.set("Program.fs", `module ${scenario === "producer" ? "FrozenProducer" : "Repro"} +${scenario === "producer" ? `${producerSource}\n` : ""}[] +let main _ = +${body.split("\n").map(l => ` ${l}`).join("\n")} +`); + if (scenario === "diagnostic") contents.set("Program.fs", [ + "module Repro", + '[]', + "let value = 1", + "[]", + "let main _ =", + " value", + "", + ].join("\n")); + contents.set("Repro.fsproj", ` + + Exe + ${scenario === "multi-tfm" ? `${tfm};${tfm}-windows7.0` : `${tfm}`} + false + ${scenario.startsWith("core-") ? "true" : ""} + ${scenario === "producer" ? "FrozenProducerFrozenProducer1.0.0$(DefineConstants);OLD_PRODUCERtrue" : ""} + + + + ${scenario === "core-explicit" ? `` : ""} + ${scenario === "core-central" ? '' : ""} + ${producer ? 'producer/FrozenProducer.dll' : ""} + + ${scenario === "late-flags" ? '$(OtherFlags) --optimize-' : ""} + +`); + if (scenario === "core-central") contents.set("Directory.Packages.props", ` + true + + +`); + } + const sources = []; + for (const [name, text] of contents) { + await fs.writeFile(path.join(directory, name), text); + sources.push({ path: name, sha256: hash(text) }); + } + if (producer) { + await fs.mkdir(path.join(directory, "producer")); + for (const record of [producer.dll, producer.package]) { + await fs.copyFile(record.originalPath, path.join(directory, record.path)); + sources.push({ path: record.path, sha256: record.sha256 }); + } + const descriptor = { ...producer }; + delete descriptor.identity; + descriptor.dll = { ...producer.dll }; delete descriptor.dll.originalPath; + descriptor.package = { ...producer.package }; delete descriptor.package.originalPath; + const bytes = JSON.stringify(descriptor); + await fs.writeFile(path.join(directory, "frozen.producer.json"), bytes); + sources.push({ path: "frozen.producer.json", sha256: hash(bytes) }); + } + const project = kind === "project" ? "Repro.fsproj" : null; + const manifest = { + schemaVersion: 1, repository: "dotnet/fsharp", issueNumber: 1, + discussionFingerprint: hash("synthetic adapter smoke; not an upstream issue"), + kind, sources, + assertions: { + pass: { exitCode: 0, stdout: scenario === "producer-control" ? "NEW_METADATA\n" : "PASS\n", stderr: "" }, + failure: { exitCode: 1, stdout: "", stderr: "FROZEN_RUNTIME_FAILURE\n" }, + }, + matrix: [{ + id: `${kind}-${tfm}-${configuration}`, os: { win32: "windows", linux: "linux", darwin: "macos" }[process.platform], + architecture: process.arch, project, tfm, configuration, languageVersion: "8.0", + command: [kind === "fsi" ? "fsi" : "run-project", project ?? "repro.fsx"], + }], + timeoutSeconds: 120, + producer: producer?.identity ?? { name: "no-producer", version: "0.0.0", sha256: hash("") }, + packages: producer ? [{ id: producer.package.id, version: producer.package.version, sha256: producer.package.sha256 }] : [], + searchHints: { sdkVersions: [], sourceShas: [], urls: [] }, + }; + if (scenario === "diagnostic") manifest.assertions = { + pass: { exitCode: 0, stdout: "", stderr: "" }, + failure: { exitCode: 1, stdout: "", stderr: "Program.fs(6,5): error FS0101: This construct is deprecated. FROZEN_DIAGNOSTIC\n" }, + }; + if (scenario === "multi-tfm") manifest.matrix.push({ + ...manifest.matrix[0], id: `${kind}-${tfm}-windows7.0-${configuration}`, tfm: `${tfm}-windows7.0`, + }); + manifest.approvedInputDigest = fingerprintInput(manifest); + await fs.writeFile(path.join(directory, "manifest.json"), JSON.stringify(manifest, null, 2)); + return manifest; +} + +module.exports = { createFixture }; diff --git a/eng/regression-bisect/repro-process.cjs b/eng/regression-bisect/repro-process.cjs new file mode 100644 index 00000000000..6f0e5166048 --- /dev/null +++ b/eng/regression-bisect/repro-process.cjs @@ -0,0 +1,49 @@ +const path = require("node:path"); +const { spawn } = require("node:child_process"); +const { constants } = require("node:os"); +const policy = require("../../.github/policies/regression-bisect.json"); + +function execute(executable, args, { cwd, env, timeoutMs, maxBytes = policy.evidenceBytesPerStream }) { + return new Promise(resolve => { + const child = spawn(executable, args, { cwd, env, shell: false, windowsHide: true, detached: process.platform !== "win32" }); + const streams = { stdout: [], stderr: [] }; + const lengths = { stdout: 0, stderr: 0 }; + let timedOut = false, truncated = false, launchError = null, killing = Promise.resolve(); + for (const name of Object.keys(streams)) child[name].on("data", bytes => { + const remaining = Math.max(0, maxBytes - lengths[name]); + if (bytes.length > remaining) truncated = true; + if (remaining > 0) streams[name].push(bytes.subarray(0, remaining)); + lengths[name] += bytes.length; + }); + child.on("error", error => { launchError = error.message; }); + const timer = setTimeout(() => { + timedOut = true; + if (!child.pid) return; + if (process.platform === "win32") { + killing = new Promise(done => { + const killer = spawn(path.join(env.SystemRoot, "System32", "taskkill.exe"), ["/PID", String(child.pid), "/T", "/F"], + { windowsHide: true, stdio: "ignore", env }); + killer.on("error", error => { launchError = `process-tree termination failed: ${error.message}`; child.kill(); done(); }); + killer.on("close", code => { if (code !== 0) launchError = `process-tree termination failed (${code})`; done(); }); + }); + } else { + try { process.kill(-child.pid, "SIGKILL"); } catch (error) { launchError = `process-tree termination failed: ${error.message}`; } + } + }, timeoutMs); + child.on("close", async (code, signal) => { + clearTimeout(timer); + await killing; + const stdout = Buffer.concat(streams.stdout).toString("utf8").replace(/\r\n/g, "\n"); + const stderr = Buffer.concat(streams.stderr).toString("utf8").replace(/\r\n/g, "\n"); + const managedException = signal === "SIGABRT" && /^Unhandled exception\./.test(stderr); + resolve({ + status: timedOut ? "timeout" : launchError || signal && !managedException ? "exception" : "completed", + complete: !truncated && !launchError, exitCode: managedException ? 128 + constants.signals[signal] : code, + stdout, stderr, + ...(launchError ? { error: launchError } : {}), + }); + }); + }); +} + +module.exports = { execute }; diff --git a/eng/regression-bisect/repro.cjs b/eng/regression-bisect/repro.cjs new file mode 100644 index 00000000000..70829e61867 --- /dev/null +++ b/eng/regression-bisect/repro.cjs @@ -0,0 +1,572 @@ +const fs = require("node:fs/promises"); +const path = require("node:path"); +const { execute } = require("./repro-process.cjs"); +const { validateManifest, classifyAttempts, fingerprintInput } = require("./model.cjs"); +const { verifyToolchainIdentity } = require("./toolchains.cjs"); +const { hash, inside, requireValue, isolatedEnvironment } = require("./toolchain-io.cjs"); +const policy = require("../../.github/policies/regression-bisect.json"); + +const untestable = reason => ({ schemaVersion: 1, outcome: "untestable", reason }); +const role = (product, name) => { + const files = product.files.filter(f => f.role === name); + requireValue(files.length === 1, `missing or ambiguous ${name} artifact`); + return files[0]; +}; +const samePath = (a, b) => process.platform === "win32" + ? path.resolve(a).toLowerCase() === path.resolve(b).toLowerCase() : path.resolve(a) === path.resolve(b); + +async function checkedFile(record, root) { + requireValue(record && path.isAbsolute(record.path), "expected absolute artifact path"); + const real = await fs.realpath(record.path); + requireValue(inside(await fs.realpath(root), real), `artifact escapes expected root: ${record.path}`); + requireValue(hash(await fs.readFile(real)) === record.sha256, `artifact bytes changed: ${record.path}`); + return { ...record, path: real }; +} + +async function checkedRuntime(toolchain, required) { + const matches = toolchain.runtimes.filter(r => r.name === required.name && r.version === required.version); + requireValue(matches.length === 1, `missing or ambiguous runtime ${required.name} ${required.version}`); + const runtime = matches[0]; + requireValue(samePath(runtime.path, path.join(toolchain.root, "shared", runtime.name)), "wrong runtime root"); + await checkedFile({ path: path.join(runtime.path, runtime.version, "System.Private.CoreLib.dll"), sha256: runtime.sha256 }, toolchain.root); + return runtime; +} + +async function selectFsiCommand({ product, toolchain, cell }) { + requireValue(["source", "released"].includes(product.kind), "unknown product kind"); + requireValue(toolchain.architecture === cell.architecture, "wrong host architecture"); + requireValue(toolchain.rid.startsWith({ windows: "win-", linux: "linux-", macos: "osx-" }[cell.os]), "wrong host OS"); + const host = await checkedFile({ path: toolchain.hostPath, sha256: toolchain.hostSha256 }, toolchain.root); + requireValue(samePath(host.path, path.join(toolchain.root, cell.os === "windows" ? "dotnet.exe" : "dotnet")), "wrong selected host"); + const fsi = await checkedFile(role(product, "fsi"), product.root); + const core = await checkedFile(role(product, "core"), product.root); + requireValue(fsi.assemblyVersion && core.assemblyVersion, "missing FSI/Core assembly versions"); + if (product.kind === "released") { + requireValue(samePath(product.root, path.join(toolchain.sdkBasePath, "FSharp")), "released FSI must be bundled with the acquired SDK"); + } else { + requireValue(!inside(toolchain.root, fsi.path) && /^[a-f0-9]{40}$/.test(product.sourceSha) && + product.buildInstanceId, "local FSI must belong to the tested source build"); + } + if (path.basename(fsi.path).toLowerCase() === "fsi.exe") { + requireValue(cell.os === "windows", "framework FSI executable requires Windows"); + let runtime; + if (product.runtime.name === "Microsoft.NETCore.App") runtime = await checkedRuntime(toolchain, product.runtime); + else { + requireValue(product.runtime.name === ".NETFramework", "unsupported executable runtime"); + runtime = await checkedFile(product.runtime, product.runtime.root); + } + return { executable: fsi.path, args: [], identity: { fsi, core, runtime, architecture: cell.architecture } }; + } + requireValue(path.basename(fsi.path) === "fsi.dll", "local FSI cannot be host dotnet fsi"); + const config = await checkedFile(fsi.runtimeConfig, product.root); + const options = JSON.parse(await fs.readFile(config.path, "utf8")).runtimeOptions; + const frameworks = options?.framework ? [options.framework] : options?.frameworks; + requireValue(frameworks?.length === 1 && frameworks[0].name === product.runtime.name, "unsupported FSI runtimeconfig"); + const required = frameworks[0].version; + const selected = product.runtime.version; + requireValue(required.split(".").slice(0, 2).join(".") === selected.split(".").slice(0, 2).join(".") && + Number(selected.split(".")[2].split("-")[0]) >= Number(required.split(".")[2].split("-")[0]), + "explicit runtime is incompatible with FSI runtimeconfig"); + const runtime = await checkedRuntime(toolchain, product.runtime); + return { + executable: host.path, args: ["exec", "--fx-version", runtime.version, fsi.path], + identity: { host, fsi, core, runtime, runtimeConfig: config, architecture: cell.architecture }, + }; +} + +const compileAssertion = manifest => /(?:^|\n)[^\n]+\(\d+,\d+(?:,\d+,\d+)?\): error FS\d{4}: [^\n]+/.test(manifest.assertions.failure.stderr); +const attemptFor = ({ status, complete, exitCode, stdout, stderr }) => ({ + schemaVersion: 1, checks: { restore: true, build: true, setup: true, identity: true, sdk: true, runtime: true }, + execution: { status, complete, exitCode, stdout, stderr }, +}); + +function evaluateAssertion(manifest, execution, { phase = "runtime", checks = {} } = {}) { + validateManifest(manifest); + if (phase !== (compileAssertion(manifest) ? "compile" : "runtime")) return untestable("assertion-mismatch"); + const attempt = attemptFor(execution); + Object.assign(attempt.checks, checks); + if (/You must install or update .NET|Failed to (?:load|initialize) (?:CoreCLR|the runtime)|hostfxr.*(?:not found|failed)/i.test(execution.stderr)) { + attempt.checks.runtime = false; + } + return classifyAttempts(Array.from({ length: policy.limits.assertionRepetitions }, () => attempt), manifest); +} + +async function verifyProjectIdentities({ manifest, product, toolchain, sourceDirectory, packageRoot, evidence, producerRecords = [] }) { + try { + validateManifest(manifest); + requireValue(evidence.length === manifest.matrix.length, "missing or extra project/TFM/configuration evidence"); + const expected = Object.fromEntries(await Promise.all( + ["compiler", "tasks", "props", "targets", "sdkTargets", "overrides", "core", "corePackage"] + .map(async name => [name, await checkedFile(role(product, name), product.root)]))); + requireValue(product.kind === "source" + ? /^[a-f0-9]{40}$/.test(product.sourceSha) && product.buildInstanceId && + Object.values(expected).every(f => !inside(toolchain.root, f.path)) + : product.kind === "released" && samePath(product.root, path.join(toolchain.sdkBasePath, "FSharp")), + "project artifact origin mismatch"); + const verified = []; + const seen = new Set(); + for (const cell of manifest.matrix) { + const project = await fs.realpath(path.join(sourceDirectory, cell.project)); + const rows = evidence.filter(e => samePath(e.project, project) && + e.targetFramework === cell.tfm && e.configuration === cell.configuration); + requireValue(rows.length === 1 && !seen.has(rows[0]), `missing or ambiguous inner build: ${cell.id}`); + const row = rows[0]; + seen.add(row); + requireValue(row.complete && row.fsc.length === 1, `no unique executed Fsc task: ${cell.id}`); + requireValue(row.sdkVersion === toolchain.sdkVersion && samePath(row.hostPath, toolchain.hostPath), "SDK/host fallback"); + requireValue(row.languageVersion === cell.languageVersion, "language version changed"); + requireValue(row.projectReferences.every(p => manifest.matrix.some(c => c.project && + samePath(path.resolve(path.dirname(project), p), path.join(sourceDirectory, c.project)))), + "undeclared project reference (immutable producers must not be rebuilt)"); + const imports = await Promise.all(row.imports.map(f => fs.realpath(f))); + for (const name of ["Sdk.props", "Sdk.targets"]) { + const sdkImport = await fs.realpath(path.join(toolchain.sdkBasePath, "Sdks", "Microsoft.NET.Sdk", "Sdk", name)); + requireValue(imports.some(f => samePath(f, sdkImport)), "SDK imports fallback"); + } + for (const name of ["props", "targets", "sdkTargets", "overrides"]) { + requireValue(imports.some(f => samePath(f, expected[name].path)), `${name} fallback`); + } + const fsc = row.fsc[0]; + for (const [field, file] of [["compilerPath", expected.compiler], ["taskAssembly", expected.tasks], ["targetsPath", expected.targets]]) { + requireValue(samePath(await fs.realpath(fsc[field]), file.path), `${field} fallback or late override`); + } + requireValue(samePath(fsc.hostPath, toolchain.hostPath) && fsc.commandLine, "wrong Fsc host or missing invocation"); + requireValue(fsc.languageVersion === cell.languageVersion && fsc.optimize === row.optimize && + fingerprintInput([...fsc.defines].sort()) === fingerprintInput([...row.defines].sort()), "late compiler flag override"); + const core = row.core; + requireValue(core.package.id === "FSharp.Core" && core.package.version === product.corePackageVersion, "Core package version fallback"); + requireValue(samePath(await fs.realpath(core.package.root), await fs.realpath(packageRoot)), "shared package cache"); + await checkedFile({ ...core.package, sha256: expected.corePackage.sha256 }, packageRoot); + for (const kind of ["compile", "runtime"]) { + const asset = core[kind]; + const packageDirectory = path.join(packageRoot, "fsharp.core", product.corePackageVersion.toLowerCase()); + const relative = path.relative(packageDirectory, asset.path).split(path.sep).join("/"); + const digest = product.coreAssets + ? product.coreAssets.find(e => e.path === relative)?.sha256 : expected.core.sha256; + requireValue(digest, `Core ${kind} asset absent from approved package`); + await checkedFile({ ...asset, sha256: digest }, packageDirectory); + } + const references = await Promise.all(fsc.references.filter(f => path.basename(f).toLowerCase() === "fsharp.core.dll").map(f => fs.realpath(f))); + requireValue(references.length === 1 && samePath(references[0], await fs.realpath(core.compile.path)), "Fsc did not reference selected Core"); + verified.push(row); + } + for (const producer of producerRecords) { + requireValue(evidence.every(e => !e.targetPath || path.basename(e.targetPath) !== path.basename(producer.dll.path)), + "immutable producer was rebuilt"); + const references = [...new Set(evidence.flatMap(e => e.fsc.flatMap(f => f.references)) + .filter(f => path.basename(f) === path.basename(producer.dll.path)))]; + requireValue(references.length > 0, "consumer did not reference immutable producer"); + for (const reference of references) { + const real = await fs.realpath(reference); + requireValue((inside(await fs.realpath(sourceDirectory), real) || inside(await fs.realpath(packageRoot), real)) && + hash(await fs.readFile(real)) === producer.dll.sha256, "consumer did not reference immutable producer bytes"); + } + } + return { status: "verified", cells: verified }; + } catch (error) { + return { status: "untestable", outcome: untestable("wrong-identity"), error: error.message }; + } +} + +async function verifySources(manifest, sourceDirectory) { + for (const source of manifest.sources) await checkedFile({ ...source, path: path.join(sourceDirectory, source.path) }, sourceDirectory); +} + +async function prepareEnvironment(root, toolchain, platform) { + const env = isolatedEnvironment(root, platform); + if (platform.os === "windows") Object.assign(env, { + PATHEXT: ".COM;.EXE;.BAT;.CMD", OS: "Windows_NT", + ProgramFiles: path.join(root, "program-files"), + "ProgramFiles(x86)": path.join(root, "program-files-x86"), + }); + for (const folder of [env.HOME, env.TEMP, env.APPDATA, env.LOCALAPPDATA, env.NUGET_PACKAGES]) await fs.mkdir(folder, { recursive: true }); + const isolatedDotnet = env.DOTNET_ROOT; + for (const [name, value] of Object.entries(env)) { + if (value === isolatedDotnet) env[name] = toolchain.root; + } + env.PATH = `${toolchain.root}${path.delimiter}${env.PATH.split(path.delimiter).slice(1).join(path.delimiter)}`; + env.DOTNET_HOST_PATH = toolchain.hostPath; + return env; +} + +async function invokeHelper(helper, args, settings, run) { + requireValue(helper?.executable && helper.dll && helper.runtimeVersion, "evidence helper is required"); + const result = await run(helper.executable, ["exec", "--fx-version", helper.runtimeVersion, helper.dll, ...args], + { ...settings, maxBytes: 4 * 1024 * 1024 }); + requireValue(result.status === "completed" && result.complete && result.exitCode === 0, + `evidence helper failed: ${result.error ?? result.stderr}`); + return JSON.parse(result.stdout); +} + +async function inspectProduct(product, inspect) { + for (const file of product.files) await checkedFile(file, product.root); + const files = product.files.filter(f => /\.(dll|exe|nupkg)$/i.test(f.path)); + const inspected = await inspect(files.map(f => f.path)); + for (const file of files.filter(f => f.assemblyVersion)) { + const actual = inspected.find(f => samePath(f.path, file.path)); + requireValue(actual?.sha256 === file.sha256 && actual.assemblyVersion === file.assemblyVersion, "artifact assembly identity mismatch"); + } + const pkg = inspected.find(f => samePath(f.path, role(product, "corePackage").path)); + requireValue(pkg?.id === "FSharp.Core" && pkg.version === product.corePackageVersion, "package identity mismatch"); + return { ...product, coreAssets: pkg.entries.filter(e => /^lib\/[^/]+\/FSharp\.Core\.dll$/i.test(e.path)) }; +} + +async function producers(manifest, sourceDirectory, inspect) { + const records = []; + for (const source of manifest.sources.filter(s => s.path.endsWith(".producer.json"))) { + const record = JSON.parse(await fs.readFile(path.join(sourceDirectory, source.path), "utf8")); + requireValue(record.role === "immutable-producer" && record.compiler?.sha256 && record.compiler.version && + Array.isArray(record.flags) && record.tfm && Array.isArray(record.packages), "incomplete immutable producer provenance"); + requireValue(record.dll.sha256 === manifest.producer.sha256, "producer does not match frozen identity"); + for (const item of [record.dll, ...(record.package ? [record.package] : [])]) { + requireValue(manifest.sources.some(s => s.path === item.path && s.sha256 === item.sha256), "producer asset must be a reviewed source"); + await checkedFile({ ...item, path: path.join(sourceDirectory, item.path) }, sourceDirectory); + } + if (record.package) requireValue(manifest.packages.some(p => p.id === record.package.id && + p.version === record.package.version && p.sha256 === record.package.sha256), "producer package not frozen"); + const [dll] = await inspect([path.join(sourceDirectory, record.dll.path)]); + requireValue(fingerprintInput(dll.resources) === fingerprintInput(record.dll.resources) && + dll.resources.some(r => /^FSharp(Signature|Optimization)(Compressed)?DataB?\./.test(r.name)), "producer metadata resource changed or missing"); + records.push(record); + } + requireValue(records.length <= 1, "ambiguous immutable producer role"); + requireValue(records.length === 1 || manifest.producer.name === "no-producer" && + manifest.producer.version === "0.0.0" && manifest.producer.sha256 === hash(""), "missing immutable producer declaration"); + return records; +} + +async function coreAssets(row, packageRoot, product, manifest) { + const assets = JSON.parse(await fs.readFile(row.assetsFile, "utf8")); + const keys = Object.keys(assets.targets).filter(k => k === row.targetFramework); + requireValue(keys.length === 1, "missing or ambiguous assets TFM"); + const target = assets.targets[keys[0]]; + const identity = `FSharp.Core/${product.corePackageVersion}`; + requireValue(target[identity] && assets.libraries[identity]?.type === "package", "Core not resolved as expected package"); + requireValue(Object.keys(assets.packageFolders).length === 1 && + samePath(await fs.realpath(Object.keys(assets.packageFolders)[0]), await fs.realpath(packageRoot)), "nonisolated assets package root"); + const directory = path.join(packageRoot, assets.libraries[identity].path); + for (const [key, library] of Object.entries(assets.libraries)) { + if (key === identity || library.type !== "package") continue; + const [id, version] = key.split("/"); + const frozen = manifest.packages.find(p => p.id.toLowerCase() === id.toLowerCase() && p.version === version); + requireValue(frozen, `unfrozen package: ${key}`); + await checkedFile({ path: path.join(packageRoot, library.path, `${id.toLowerCase()}.${version.toLowerCase()}.nupkg`), + sha256: frozen.sha256 }, packageRoot); + } + const selected = {}; + for (const kind of ["compile", "runtime"]) { + const entries = Object.keys(target[identity][kind] ?? {}).filter(p => /(?:^|\/)FSharp\.Core\.dll$/i.test(p)); + requireValue(entries.length === 1, `missing or ambiguous Core ${kind} asset`); + const location = path.join(directory, entries[0]); + selected[kind] = { path: location, sha256: hash(await fs.readFile(location)) }; + } + const nupkg = path.join(directory, `fsharp.core.${product.corePackageVersion.toLowerCase()}.nupkg`); + return { + ...selected, package: { id: "FSharp.Core", version: product.corePackageVersion, root: packageRoot, + path: nupkg, sha256: hash(await fs.readFile(nupkg)) }, + assetsFile: { path: row.assetsFile, sha256: hash(await fs.readFile(row.assetsFile)) }, + }; +} + +async function loadedCore(file, expected, runtime, architecture, allowedPaths = [expected.path]) { + const records = (await fs.readFile(file, "utf8")).trim().split(/\r?\n/).map(line => JSON.parse(line)); + requireValue(records.length === 1, "missing or ambiguous loaded FSharp.Core"); + const actual = records[0]; + requireValue(actual.complete === true && actual.assemblyName === "FSharp.Core", "incomplete loaded Core evidence"); + requireValue(allowedPaths.some(p => samePath(p, actual.path)), "wrong loaded Core path"); + requireValue(samePath(await fs.realpath(actual.path), actual.path), "loaded Core path redirects through a link"); + requireValue(actual.sha256 === expected.sha256 && actual.assemblyVersion === expected.assemblyVersion && + actual.runtimeVersion === runtime.version && actual.architecture.toLowerCase() === architecture, "wrong loaded Core/runtime/architecture"); + requireValue(samePath(actual.runtimePath, path.join(runtime.path, runtime.version)) && + hash(await fs.readFile(path.join(actual.runtimePath, "System.Private.CoreLib.dll"))) === runtime.sha256, "wrong loaded runtime path/bytes"); + requireValue(hash(await fs.readFile(actual.path)) === expected.sha256, "loaded Core bytes changed"); + return actual; +} + +async function probeFrameworkFsi(selected, flags, directory, command, expected, architecture) { + const probe = path.join(directory, ".runner", "core-probe.fsx"); + await fs.writeFile(probe, [ + "open System", + "open System.IO", + "let core = typeof>.Assembly", + 'let info = typeof.Assembly.GetType("System.Runtime.InteropServices.RuntimeInformation")', + 'let arch = if isNull info then (if IntPtr.Size = 8 then "x64" else "x86") else info.GetProperty("ProcessArchitecture").GetValue(null).ToString().ToLowerInvariant()', + 'printfn "CORE|%s|%O|%s|%s|%s" core.Location (core.GetName().Version) (Path.GetFileName(Path.GetDirectoryName(typeof.Assembly.Location))) arch typeof.Assembly.Location', + ].join("\n")); + const result = await command(selected.executable, [...selected.args, ...flags, "--exec", probe]); + requireValue(result.status === "completed" && result.complete && result.exitCode === 0, "framework FSI identity probe failed"); + const lines = result.stdout.trim().split("\n"); + requireValue(lines.length === 1 && lines[0].startsWith("CORE|"), "ambiguous framework FSI probe"); + const [, location, assemblyVersion, runtimeVersion, actualArchitecture, runtimeLocation] = lines[0].split("|"); + const runtime = selected.identity.runtime; + const runtimePath = runtime.name === ".NETFramework" ? runtime.path : path.join(runtime.path, runtime.version, "System.Private.CoreLib.dll"); + requireValue(samePath(await fs.realpath(location), expected.path) && + hash(await fs.readFile(location)) === expected.sha256 && assemblyVersion === expected.assemblyVersion && + actualArchitecture === architecture && runtimeVersion === selected.identity.runtime.version, "wrong loaded framework Core/runtime"); + requireValue(samePath(await fs.realpath(runtimeLocation), runtimePath) && + hash(await fs.readFile(runtimeLocation)) === runtime.sha256, "wrong loaded executable runtime bytes"); + return { path: location, sha256: expected.sha256, assemblyVersion, runtimeVersion, runtimePath, architecture }; +} + +function diagnosticsExecution(result, diagnostics, root) { + const errors = diagnostics.filter(d => d.severity === "error"); + return { + ...result, stdout: "", stderr: errors.map(d => { + const span = d.endLine > 0 && d.endColumn > 0 ? `${d.line},${d.column},${d.endLine},${d.endColumn}` : `${d.line},${d.column}`; + return `${path.relative(root, d.file).split(path.sep).join("/")}(${span}): error ${d.code}: ${d.message}\n`; + }).join(""), + }; +} + +function expectedBuildManifest(manifest, cell, rows, directory) { + const matrix = [cell]; + for (let i = 0; i < matrix.length; i++) { + const current = matrix[i]; + const matches = rows.filter(r => samePath(r.project, path.join(directory, current.project)) && + r.targetFramework === current.tfm && r.configuration === current.configuration); + requireValue(matches.length === 1, `missing or ambiguous expected inner build: ${current.project} ${current.tfm}`); + for (const reference of matches[0].projectReferences) { + const absolute = path.resolve(path.dirname(matches[0].project), reference); + const relative = path.relative(directory, absolute).split(path.sep).join("/"); + requireValue(manifest.sources.some(s => s.path === relative && s.path.endsWith(".fsproj")), + "undeclared project reference (immutable producers must not be rebuilt)"); + const declared = manifest.matrix.filter(c => c.project === relative && c.configuration === cell.configuration); + const expected = declared.find(c => c.tfm === cell.tfm) ?? (declared.length === 1 ? declared[0] : null); + requireValue(declared.length === 0 || expected, "ambiguous referenced project TFM"); + const next = expected ?? { ...cell, project: relative, command: ["run-project", relative] }; + if (!matrix.some(c => c.project === next.project && c.tfm === next.tfm && c.configuration === next.configuration)) { + matrix.push({ ...next, id: `${cell.id}-dependency-${matrix.length}` }); + } + } + } + const { approvedInputDigest, ...input } = manifest; + const result = { ...input, matrix }; + return { ...result, approvedInputDigest: fingerprintInput(result) }; +} + +async function runReproduction(options, dependencies = {}) { + const cells = []; + let stage = "setup"; + let work, pending; + try { + const { manifest, toolchain, helper } = options; + validateManifest(manifest); + await verifySources(manifest, options.sourceDirectory); + const verification = verifyToolchainIdentity({ policy, toolchain: toolchain.requested, identity: toolchain }); + requireValue(verification.status === "verified", verification.error?.message ?? "unverified acquisition"); + await checkedFile({ path: toolchain.hostPath, sha256: toolchain.hostSha256 }, toolchain.root); + requireValue(["source", "released"].includes(options.product.kind) && options.product.buildInstanceId, "unidentified product build"); + if (options.product.kind === "source") requireValue(/^[0-9a-f]{40}$/.test(options.product.sourceSha) && + (!toolchain.requested.sourceSha || options.product.sourceSha === toolchain.requested.sourceSha), "source build revision mismatch"); + const platform = toolchain.requested.platform; + requireValue(manifest.matrix.every(c => c.architecture === toolchain.architecture && + c.os === platform.os), "matrix requires a different worker platform"); + const run = dependencies.execute ?? execute; + await fs.mkdir(options.workDirectory, { recursive: true }); + work = await fs.realpath(await fs.mkdtemp(path.join(options.workDirectory, "repro-"))); + const env = await prepareEnvironment(work, toolchain, platform); + const settings = { cwd: work, env, timeoutMs: manifest.timeoutSeconds * 1000 }; + const inspect = files => invokeHelper(helper, ["inspect", ...files], settings, run); + stage = "identity"; + const product = await inspectProduct(options.product, inspect); + const frozenProducers = await producers(manifest, options.sourceDirectory, inspect); + const productFingerprint = fingerprintInput(product); + await fs.writeFile(path.join(work, "identities.json"), JSON.stringify({ + product, acquisitionDigest: fingerprintInput(toolchain), + sdk: { version: toolchain.sdkVersion, basePath: toolchain.sdkBasePath, archive: toolchain.archive }, + host: { path: toolchain.hostPath, sha256: toolchain.hostSha256 }, + architecture: toolchain.architecture, runtimes: toolchain.runtimes, producers: frozenProducers, + }, null, 2), { flag: "wx" }); + const repetitions = policy.limits.assertionRepetitions; + for (const cell of manifest.matrix) { + const attempts = [], evidence = []; + for (let repetition = 0; repetition < repetitions; repetition++) { + stage = "setup"; + const directory = path.join(work, `cell-${cells.length}-${repetition}`); + await fs.mkdir(directory); + for (const source of manifest.sources) { + const destination = path.join(directory, source.path); + await fs.mkdir(path.dirname(destination), { recursive: true }); + await fs.copyFile(path.join(options.sourceDirectory, source.path), destination); + } + await fs.writeFile(path.join(directory, "global.json"), JSON.stringify({ + sdk: { version: toolchain.sdkVersion, rollForward: "disable", allowPrerelease: product.kind === "source" }, + }), { flag: "wx" }); + const attemptEnv = await prepareEnvironment(path.join(directory, ".runner"), toolchain, platform); + const launch = { cwd: directory, env: attemptEnv, timeoutMs: settings.timeoutMs }; + const record = { commands: [], productFingerprint, producers: frozenProducers }; + pending = { cell: cell.id, repetition, evidence: record }; + const command = async (executable, args, extraEnv = {}) => { + const entry = { executable, args }; + record.commands.push(entry); + entry.result = await run(executable, args, { ...launch, env: { ...attemptEnv, ...extraEnv } }); + return entry.result; + }; + let execution; + if (manifest.kind === "fsi") { + stage = "identity"; + const selected = await selectFsiCommand({ product, toolchain, cell }); + record.selection = selected.identity; + const coreFile = path.join(directory, ".runner", "fsi-core.jsonl"); + const flags = ["--nologo", "--noninteractive", `--langversion:${cell.languageVersion}`, + `--optimize${cell.configuration === "Release" ? "+" : "-"}`]; + const framework = path.extname(selected.executable).toLowerCase() === ".exe" && selected.args.length === 0; + if (framework) record.loadedCore = await probeFrameworkFsi(selected, flags, directory, command, role(product, "core"), cell.architecture); + else requireValue(helper.hook, "runtime Core load hook missing"); + stage = "runtime"; + execution = await command(selected.executable, [...selected.args, ...flags, "--exec", path.join(directory, cell.command[1])], + framework ? {} : { DOTNET_STARTUP_HOOKS: helper.hook, REPRO_CORE_EVIDENCE: coreFile }); + stage = "identity"; + if (!framework && execution.status === "completed") record.loadedCore = await loadedCore(coreFile, + role(product, "core"), selected.identity.runtime, cell.architecture); + if (compileAssertion(manifest)) execution = { + ...execution, stderr: execution.stderr.split(`${directory}${path.sep}`).join("").replace(/^\n+/, ""), + }; + } else { + const reviewedConfig = manifest.sources.find(s => s.path.toLowerCase() === "nuget.config"); + const restoreConfig = reviewedConfig ? path.join(directory, reviewedConfig.path) : path.join(directory, ".runner", "NuGet.Config"); + if (!reviewedConfig) await fs.writeFile(restoreConfig, + "", { flag: "wx" }); + const targetImport = path.join(directory, ".runner", "Directory.Build.targets"); + const imports = [path.join(__dirname, "ReproEvidence.targets")]; + if (product.kind === "source") imports.unshift(path.join(__dirname, "..", "..", "UseLocalCompiler.Directory.Build.targets")); + const xml = text => text.replace(/&/g, "&").replace(/"/g, """).replace(/${imports.map(file => ``).join("")}`); + attemptEnv.CustomAfterDirectoryBuildTargets = targetImport; + if (product.kind === "source") { + const config = await checkedFile(role(product, "compiler").runtimeConfig, product.root); + const runtimeOptions = JSON.parse(await fs.readFile(config.path, "utf8")).runtimeOptions; + requireValue(/^net\d+\.\d+$/.test(runtimeOptions.tfm) && + ["Debug", "Release"].includes(product.configuration), "missing built compiler layout/configuration"); + Object.assign(attemptEnv, { + LoadLocalFSharpBuild: "True", LocalFSharpCompilerPath: product.root, + LocalFSharpCompilerConfiguration: product.configuration, + FSharpNetCoreProductTargetFramework: runtimeOptions.tfm, + CustomAfterDirectoryBuildProps: path.join(__dirname, "..", "..", "UseLocalCompiler.Directory.Build.props"), + RegressionLocalCore: "true", RegressionLocalCoreVersion: product.corePackageVersion, + RegressionLocalCorePackagesDir: path.dirname(role(product, "corePackage").path), + }); + } + const binlog = path.join(directory, ".runner", "build.binlog"); + const compilerRuntime = await checkedRuntime(toolchain, product.runtime); + const compatible = toolchain.runtimes.filter(r => r.name === product.runtime.name && + r.version.split(".").slice(0, 2).join(".") === product.runtime.version.split(".").slice(0, 2).join(".")); + requireValue(compatible.length === 1, "ambiguous compiler runtime; cannot pin MSBuild's compiler child"); + record.compilerRuntimeSelection = { runtime: compilerRuntime, rollForward: "LatestPatch" }; + stage = "build"; + const build = await command(toolchain.hostPath, ["build", path.join(directory, cell.project), + "--no-incremental", "--nologo", "-v:quiet", "-nr:false", `-bl:${binlog}`, + `-p:TargetFramework=${cell.tfm}`, `-p:Configuration=${cell.configuration}`, `-p:LangVersion=${cell.languageVersion}`, + `-p:RestoreConfigFile=${restoreConfig}`, + "-p:UseSharedCompilation=false"], { + DOTNET_ROLL_FORWARD: "LatestPatch", DOTNET_ROLL_FORWARD_TO_PRERELEASE: "1", + DOTNET_STARTUP_HOOKS: helper.hook, + REPRO_CORE_EVIDENCE: path.join(directory, ".runner", "compiler-core-{pid}.jsonl"), + }); + record.build = build; + if (build.status !== "completed") execution = build; + else { + const structured = await invokeHelper(helper, ["binlog", binlog], launch, run); + record.binlog = { path: binlog, sha256: hash(await fs.readFile(binlog)) }; + if (structured.diagnostics.some(d => d.severity === "error" && + (/^NU\d+$/.test(d.code) || path.basename(d.file ?? "").toLowerCase() === "nuget.targets"))) { + const failed = attemptFor(build); + failed.checks.restore = false; + attempts.push(failed); evidence.push(record); + continue; + } + const rows = structured.cells.map(row => ({ + ...row, fsc: row.fscTasks, + languageVersion: row.expectedLanguageVersion, + optimize: row.expectedOptimize, + defines: row.expectedDefines, + assetsFile: path.resolve(path.dirname(row.project), row.assetsFile), + targetPath: path.resolve(path.dirname(row.project), row.targetPath), + })); + for (const row of rows) row.core = await coreAssets(row, attemptEnv.NUGET_PACKAGES, product, manifest); + stage = "identity"; + const cellManifest = expectedBuildManifest(manifest, cell, rows, directory); + const verified = await verifyProjectIdentities({ + manifest: cellManifest, product, toolchain, sourceDirectory: directory, + packageRoot: attemptEnv.NUGET_PACKAGES, evidence: rows, producerRecords: frozenProducers, + }); + requireValue(verified.status === "verified", verified.error); + record.projects = rows; + const compiler = role(product, "compiler"); + const coreFiles = []; + for (const name of await fs.readdir(path.join(directory, ".runner"))) { + if (!/^compiler-core-\d+\.jsonl$/.test(name)) continue; + const file = path.join(directory, ".runner", name); + const loads = (await fs.readFile(file, "utf8")).trim().split(/\r?\n/).map(line => JSON.parse(line)); + if (loads.some(load => load.entryAssembly && samePath(load.entryAssembly, compiler.path))) coreFiles.push(file); + } + requireValue(coreFiles.length === rows.reduce((count, row) => count + row.fsc.length, 0), + "missing actual compiler runtime/Core load evidence"); + record.compilerCoreLoads = []; + for (const file of coreFiles) record.compilerCoreLoads.push(await loadedCore(file, role(product, "core"), + compilerRuntime, cell.architecture, [path.join(path.dirname(compiler.path), "FSharp.Core.dll")])); + if (compileAssertion(manifest)) execution = diagnosticsExecution(build, structured.diagnostics, directory); + else if (build.exitCode !== 0) { + const failed = attemptFor(build); + failed.checks.build = false; + attempts.push(failed); evidence.push(record); + continue; + } else { + const row = rows.find(r => samePath(r.project, path.join(directory, cell.project)) && + r.targetFramework === cell.tfm && r.configuration === cell.configuration); + requireValue(inside(directory, await fs.realpath(row.targetPath)), "runtime output outside reproduction"); + record.runtimeProducers = []; + for (const producer of frozenProducers) { + const location = path.join(path.dirname(row.targetPath), path.basename(producer.dll.path)); + record.runtimeProducers.push(await checkedFile({ path: location, sha256: producer.dll.sha256 }, directory)); + } + const runtime = options.sampleRuntimes?.[cell.tfm] ?? product.runtime; + const verifiedRuntime = await checkedRuntime(toolchain, runtime); + const coreFile = path.join(directory, ".runner", "app-core.jsonl"); + stage = "runtime"; + execution = await command(toolchain.hostPath, ["exec", "--fx-version", runtime.version, row.targetPath], { + DOTNET_STARTUP_HOOKS: helper.hook, REPRO_CORE_EVIDENCE: coreFile, + }); + stage = "identity"; + if (execution.status === "completed") { + const [core] = await inspect([row.core.runtime.path]); + record.loadedCore = await loadedCore(coreFile, core, verifiedRuntime, cell.architecture, + [row.core.runtime.path, path.join(path.dirname(row.targetPath), "FSharp.Core.dll")]); + } + } + } + } + stage = "identity"; + await verifySources(manifest, directory); + await verifySources(manifest, options.sourceDirectory); + for (const file of product.files) await checkedFile(file, product.root); + await checkedFile({ path: toolchain.hostPath, sha256: toolchain.hostSha256 }, toolchain.root); + for (const file of record.runtimeProducers ?? []) await checkedFile(file, directory); + const attempt = attemptFor(execution); + const outcome = evaluateAssertion(manifest, execution, { phase: compileAssertion(manifest) ? "compile" : "runtime" }); + if (outcome.reason === "unavailable-runtime") attempt.checks.runtime = false; + attempts.push(attempt); evidence.push(record); + } + cells.push({ id: cell.id, attempts, evidence, outcome: classifyAttempts(attempts, manifest) }); + } + const failure = cells.find(c => c.outcome.outcome === "untestable"); + const outcome = failure?.outcome ?? (cells.every(c => c.outcome.outcome === cells[0].outcome.outcome) + ? cells[0].outcome : untestable("flaky")); + const identitiesPath = path.join(work, "identities.json"); + const result = { + outcome, cells, manifestDigest: fingerprintInput(manifest), productFingerprint, workDirectory: work, + identities: { path: identitiesPath, sha256: hash(await fs.readFile(identitiesPath)) }, + }; + await fs.writeFile(path.join(work, "result.json"), JSON.stringify(result, null, 2), { flag: "wx" }); + return result; + } catch (error) { + const result = { + outcome: untestable({ identity: "wrong-identity", build: "build-failure", runtime: "unavailable-runtime" }[stage] ?? "setup-failure"), + cells, error: error.message, ...(pending ? { pending } : {}), ...(work ? { workDirectory: work } : {}), + }; + if (work) await fs.writeFile(path.join(work, "result.json"), JSON.stringify(result, null, 2), { flag: "wx" }); + return result; + } +} + +module.exports = { runReproduction, selectFsiCommand, verifyProjectIdentities, evaluateAssertion }; diff --git a/eng/regression-bisect/repro.smoke.cjs b/eng/regression-bisect/repro.smoke.cjs new file mode 100644 index 00000000000..11dd13ac79f --- /dev/null +++ b/eng/regression-bisect/repro.smoke.cjs @@ -0,0 +1,91 @@ +const fs = require("node:fs/promises"); +const path = require("node:path"); +const { runReproduction } = require("./repro.cjs"); +const { createFixture } = require("./fixtures/smoke.cjs"); +const { requireValue, isolatedEnvironment } = require("./toolchain-io.cjs"); +const { execute } = require("./repro-process.cjs"); + +async function freezeProducer(root, config, results) { + const sourceDirectory = path.join(root, "producer"); + const manifest = await createFixture(sourceDirectory, { kind: "project", tfm: config.released.tfm, scenario: "producer" }); + const result = await runReproduction({ + manifest, sourceDirectory, workDirectory: path.join(root, "executions"), + product: config.released.product, toolchain: config.released.toolchain, helper: config.helper, + sampleRuntimes: config.released.sampleRuntimes, + }); + results.push({ id: "freeze-released-producer", expected: "pass", result }); + requireValue(result.outcome.outcome === "pass", `producer preparation failed: ${result.error ?? result.outcome.reason}`); + const row = result.cells[0].evidence[0].projects[0]; + const nupkg = path.join(path.dirname(path.dirname(row.targetPath)), "FrozenProducer.1.0.0.nupkg"); + const env = isolatedEnvironment(path.join(root, "helper"), config.released.toolchain.requested.platform); + for (const folder of [env.HOME, env.TEMP, env.APPDATA, env.LOCALAPPDATA]) await fs.mkdir(folder, { recursive: true }); + const inspected = await execute(config.helper.executable, + ["exec", "--fx-version", config.helper.runtimeVersion, config.helper.dll, "inspect", row.targetPath, nupkg], + { cwd: root, env, timeoutMs: 120000, maxBytes: 1024 * 1024 }); + requireValue(inspected.complete && inspected.exitCode === 0, `producer inspection failed: ${inspected.stderr}`); + const [dll, pkg] = JSON.parse(inspected.stdout); + const compiler = config.released.product.files.find(f => f.role === "compiler"); + return { + role: "immutable-producer", + identity: { name: "FrozenProducer", version: "1.0.0", sha256: dll.sha256 }, + compiler: { version: compiler.assemblyVersion, sha256: compiler.sha256 }, + flags: [row.fsc[0].commandLine], tfm: row.targetFramework, + packages: [{ id: "FSharp.Core", version: row.core.package.version, sha256: row.core.package.sha256 }], + dll: { ...dll, originalPath: dll.path, path: "producer/FrozenProducer.dll" }, + package: { id: pkg.id, version: pkg.version, sha256: pkg.sha256, + originalPath: pkg.path, path: "producer/FrozenProducer.1.0.0.nupkg" }, + }; +} + +async function main() { + const [configurationFile, outputDirectory, phase] = process.argv.slice(2); + requireValue(configurationFile && outputDirectory && process.argv.length <= 5 && (!phase || ["released", "source"].includes(phase)), + "usage: node repro.smoke.cjs [released|source]"); + const config = JSON.parse(await fs.readFile(configurationFile, "utf8")); + const products = phase ? [phase] : ["released", "source"]; + requireValue(config.released && config.helper && products.every(name => config[name]?.product.kind === name), + "verified products and evidence helper are required (source must be a clean local build)"); + await fs.mkdir(outputDirectory); + const root = await fs.realpath(outputDirectory); + const results = []; + const producer = await freezeProducer(root, config, results); + await fs.writeFile(path.join(root, "frozen-producer.json"), JSON.stringify(producer, null, 2)); + for (const name of products) { + const input = config[name]; + for (const [kind, scenario, configuration] of [ + ["fsi", "pass", "Release"], + ["project", "pass", "Debug"], + ["project", "core-explicit", "Release"], + ["project", "core-central", "Release"], + ["project", "runtime-failure", "Release"], + ["project", "diagnostic", "Release"], + ["project", "late-flags", "Release"], + ["project", "producer-consumer", "Release"], + ...(process.platform === "win32" ? [["project", "multi-tfm", "Release"]] : []), + ["fsi", "producer-control", "Release"], + ]) { + const id = `${name}-${kind}-${scenario}-${configuration}`; + const sourceDirectory = path.join(root, id); + const manifest = await createFixture(sourceDirectory, { + kind, scenario, configuration, tfm: input.tfm, + coreVersion: config.released.product.corePackageVersion, + ...(scenario === "producer-consumer" ? { producer } : {}), + }); + const result = await runReproduction({ + manifest, sourceDirectory, workDirectory: path.join(root, "executions"), + product: input.product, toolchain: input.toolchain, helper: config.helper, + sampleRuntimes: input.sampleRuntimes, + }); + const expected = scenario === "late-flags" ? "untestable" + : ["runtime-failure", "producer-consumer", "diagnostic"].includes(scenario) ? "product-failure" : "pass"; + results.push({ id, expected, result }); + await fs.writeFile(path.join(root, "smoke-results.json"), JSON.stringify(results, null, 2)); + console.log(`${id}: ${result.outcome.outcome} (${result.outcome.reason})${result.error ? `: ${result.error}` : ""}`); + requireValue(result.outcome.outcome === expected, `bounded smoke failed: ${id}`); + if (scenario === "late-flags") requireValue(result.outcome.reason === "wrong-identity" && + result.error === "late compiler flag override", "late override must be rejected for the intended identity mismatch"); + } + } +} + +if (require.main === module) main().catch(error => { console.error(error.message); process.exitCode = 1; }); diff --git a/eng/regression-bisect/repro.test.cjs b/eng/regression-bisect/repro.test.cjs new file mode 100644 index 00000000000..4fb1d181d73 --- /dev/null +++ b/eng/regression-bisect/repro.test.cjs @@ -0,0 +1,324 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs/promises"); +const path = require("node:path"); +const { fixture } = require("./fixtures/reproduction.cjs"); +const { seal, attempt } = require("./fixtures/contracts.cjs"); +const { classifyAttempts } = require("./model.cjs"); +const { selectFsiCommand, verifyProjectIdentities, evaluateAssertion, runReproduction } = require("./repro.cjs"); +const { execute } = require("./repro-process.cjs"); +const { isolatedEnvironment, hash } = require("./toolchain-io.cjs"); + +for (const [name, options] of [ + ["local DLL", {}], ["Windows executable", { executable: true }], ["released SDK", { kind: "released" }], +]) { + test(`FSI selects the inventoried ${name}`, async t => { + const f = await fixture(t, options); + const selected = await selectFsiCommand({ ...f, cell: f.manifest.matrix[0] }); + assert.equal(selected.executable, options.executable + ? f.product.files.find(x => x.role === "fsi").path : f.toolchain.hostPath); + assert.ok(!selected.args.includes("fsi")); + if (!options.executable) { + assert.ok(selected.args.includes(f.product.files.find(x => x.role === "fsi").path)); + assert.deepEqual(selected.args.slice(0, 3), ["exec", "--fx-version", "10.0.0"]); + } + }); +} + +for (const [name, mutate, options] of [ + ["wrong host", f => { f.toolchain.hostPath = path.join(f.root, "dotnet-other", "dotnet.exe"); }], + ["host dotnet fsi", f => { f.product.files.find(x => x.role === "fsi").path = "fsi"; }], + ["missing runtime", f => { f.toolchain.runtimes = []; }], + ["missing executable runtime", f => { f.toolchain.runtimes = []; }, { executable: true }], + ["wrong architecture", f => { f.toolchain.architecture = "arm64"; }], + ["ambiguous layout", f => { f.product.files.push({ ...f.product.files[0] }); }], + ["wrong artifact bytes", async f => { await fs.appendFile(f.product.files[0].path, "changed"); }], + ["wrong runtimeconfig", async f => { await fs.writeFile(f.product.files[0].runtimeConfig.path, "{}"); }], + ["released fallback for local", f => { f.product.root = f.toolchain.sdkBasePath; }], +]) { + test(`FSI rejects ${name}`, async t => { + const f = await fixture(t, options); + await mutate(f); + await assert.rejects(selectFsiCommand({ ...f, cell: f.manifest.matrix[0] })); + }); +} + +test("runtime and diagnostic assertions use the model vocabulary and exact evidence", async t => { + const f = await fixture(t); + const execution = { status: "completed", complete: true, ...f.manifest.assertions.pass }; + for (const [name, phase, changes, expected] of [ + ["runtime pass", "runtime", {}, "pass"], + ["runtime-only failure", "runtime", f.manifest.assertions.failure, "product-failure"], + ["successful build is not runtime success", "build", {}, "untestable"], + ["unrelated exception", "runtime", { stderr: "System.NullReferenceException" }, "untestable"], + ["runtime loader", "runtime", { stderr: "You must install or update .NET to run this application.", exitCode: 1 }, "untestable"], + ["truncation", "runtime", { complete: false }, "untestable"], + ["timeout", "runtime", { status: "timeout" }, "untestable"], + ["crash", "runtime", { status: "exception" }, "untestable"], + ]) { + assert.equal(evaluateAssertion(f.manifest, { ...execution, ...changes }, { phase }).outcome, expected, name); + } + f.manifest.assertions = { + pass: { exitCode: 0, stdout: "", stderr: "" }, + failure: { exitCode: 1, stdout: "", stderr: "repro.fs(3,5,3,9): error FS0001: frozen message\n" }, + }; + seal(f.manifest); + for (const [stderr, outcome] of [ + [f.manifest.assertions.failure.stderr, "product-failure"], + ["repro.fs(3,5,3,9): error FS0001: different message\n", "untestable"], + ["repro.fs(3,6,3,9): error FS0001: frozen message\n", "untestable"], + ["repro.fs(3,5,3,9): error FS0039: frozen message\n", "untestable"], + ["error NU1301: restore failed\n", "untestable"], + ]) assert.equal(evaluateAssertion(f.manifest, { ...execution, exitCode: 1, stdout: "", stderr }, { phase: "compile" }).outcome, outcome); + assert.equal(evaluateAssertion(f.manifest, { ...execution, stdout: "" }, { phase: "compile" }).outcome, "pass"); +}); + +test("mixed repetitions remain untestable", () => { + const { manifest } = require("./fixtures/contracts.cjs"); + assert.deepEqual(classifyAttempts([attempt("pass"), attempt("product-failure")], manifest()), { + schemaVersion: 1, outcome: "untestable", reason: "flaky", + }); +}); + +async function projectFixture(t, options) { + const f = await fixture(t, options); + const project = await f.file("source/App.fsproj", ""); + f.manifest.kind = "project"; + f.manifest.sources = [{ path: "App.fsproj", sha256: project.sha256 }]; + for (const c of f.manifest.matrix) { c.project = "App.fsproj"; c.command = ["run-project", c.project]; } + seal(f.manifest); + const roles = Object.fromEntries(f.product.files.map(x => [x.role, x])); + const sdkImports = []; + for (const name of ["Sdk.props", "Sdk.targets"]) sdkImports.push((await f.file( + path.join("dotnet", "sdk", "10.0.100", "Sdks", "Microsoft.NET.Sdk", "Sdk", name))).path); + const packageRoot = path.join(f.root, "packages"); + const core = await f.file("packages/fsharp.core/10.0.100/lib/netstandard2.0/FSharp.Core.dll", await fs.readFile(roles.core.path)); + const nupkg = await f.file("packages/fsharp.core/10.0.100/fsharp.core.10.0.100.nupkg", await fs.readFile(roles.corePackage.path)); + f.packageRoot = packageRoot; + f.evidence = f.manifest.matrix.map(c => ({ + project: project.path, targetFramework: c.tfm, configuration: c.configuration, languageVersion: c.languageVersion, + optimize: c.configuration === "Release", defines: [], sdkVersion: f.toolchain.sdkVersion, hostPath: f.toolchain.hostPath, + imports: [...["props", "targets", "sdkTargets", "overrides"].map(r => roles[r].path), ...sdkImports], + projectReferences: [], complete: true, + fsc: [{ compilerPath: roles.compiler.path, taskAssembly: roles.tasks.path, targetsPath: roles.targets.path, hostPath: f.toolchain.hostPath, + references: [core.path], languageVersion: c.languageVersion, optimize: c.configuration === "Release", + defines: [], commandLine: `"${f.toolchain.hostPath}" "${roles.compiler.path}" --langversion:${c.languageVersion}` }], + core: { compile: core, runtime: core, package: { ...nupkg, id: "FSharp.Core", version: "10.0.100", root: packageRoot } }, + })); + return f; +} + +test("every project/TFM/configuration cell is verified", async t => { + for (const configuration of ["Debug", "Release"]) { + const f = await projectFixture(t, { tfms: ["net9.0", "net10.0"], configuration }); + const result = await verifyProjectIdentities(f); + assert.equal(result.status, "verified", result.error); + } +}); + +for (const [name, mutate] of [ + ["compiler fallback", f => { f.evidence[0].fsc[0].compilerPath = f.toolchain.hostPath; }], + ["task fallback", f => { f.evidence[0].fsc[0].taskAssembly = f.toolchain.hostPath; }], + ["targets fallback", f => { f.evidence[0].imports.splice(1, 1); }], + ["SDK fallback", f => { f.evidence[0].imports.pop(); }], + ["executed target replaced after import", f => { f.evidence[0].fsc[0].targetsPath = f.toolchain.hostPath; }], + ["Core fallback", f => { f.evidence[0].fsc[0].references = [f.product.files.find(x => x.role === "core").path]; }], + ["same version wrong Core bytes", async f => { await fs.writeFile(f.evidence[0].core.compile.path, "other Core"); }], + ["same version wrong package bytes", async f => { await fs.writeFile(f.evidence[0].core.package.path, "other package"); }], + ["late property override", f => { f.evidence[0].fsc[0].languageVersion = "preview"; }], + ["late optimization override", f => { f.evidence[0].fsc[0].optimize = false; }], + ["late define override", f => { f.evidence[0].fsc[0].defines = ["DIFFERENT"]; }], + ["missing inner TFM", f => { f.evidence.pop(); }], + ["no Fsc invocation", f => { f.evidence[0].fsc = []; }], + ["wrong host", f => { f.evidence[0].fsc[0].hostPath = "dotnet"; }], + ["producer rebuild", f => { f.evidence[0].projectReferences = ["Producer.fsproj"]; }], + ["false prefix", async f => { + const other = await f.file("packages-sibling/FSharp.Core.dll", "local/FSharp.Core.dll"); + f.evidence[0].core.compile = other; f.evidence[0].fsc[0].references = [other.path]; + }], + ["symlink", async f => { + const other = await f.file("outside/FSharp.Core.dll", "local/FSharp.Core.dll"); + const link = path.join(f.packageRoot, "escape"); + await fs.symlink(path.dirname(other.path), link, "junction"); + f.evidence[0].core.compile = { ...other, path: path.join(link, "FSharp.Core.dll") }; + f.evidence[0].fsc[0].references = [f.evidence[0].core.compile.path]; + }], +]) { + test(`project rejects ${name}`, async t => { + const f = await projectFixture(t, { tfms: ["net9.0", "net10.0"] }); + await mutate(f); + assert.equal((await verifyProjectIdentities(f)).status, "untestable"); + }); +} + +test("runner rejects unfrozen sources before invoking any process", async t => { + const f = await fixture(t); + await fs.appendFile(path.join(f.sourceDirectory, "repro.fsx"), " changed"); + let invoked = false; + const result = await runReproduction({ ...f, workDirectory: path.join(f.root, "work") }, { + execute: async () => { invoked = true; throw new Error("must not execute"); }, + }); + assert.equal(result.outcome.outcome, "untestable"); + assert.equal(invoked, false); +}); + +async function runnerFixture(t) { + const f = await fixture(t); + f.workDirectory = path.join(f.root, "work"); + f.helper = { executable: "helper-host", dll: "helper.dll", runtimeVersion: "10.0.0", hook: "CoreHook.dll" }; + let calls = 0; + f.execute = async (executable, args, settings) => { + const result = { status: "completed", complete: true, exitCode: 0, stdout: "", stderr: "" }; + if (executable === f.helper.executable) { + const files = args.slice(args.indexOf("inspect") + 1); + result.stdout = JSON.stringify(files.map(location => { + const record = f.product.files.find(p => p.path === location) ?? f.inspectFiles?.find(p => p.path === location); + if (record.role === "corePackage") return { + ...record, id: "FSharp.Core", version: f.product.corePackageVersion, + entries: [{ path: "lib/netstandard2.0/FSharp.Core.dll", sha256: f.product.files.find(p => p.role === "core").sha256 }], + }; + return record; + })); + } else { + const core = f.product.files.find(p => p.role === "core"); + await fs.writeFile(settings.env.REPRO_CORE_EVIDENCE, JSON.stringify({ + ...core, complete: true, assemblyName: "FSharp.Core", runtimeVersion: "10.0.0", + runtimePath: path.join(f.toolchain.runtimes[0].path, "10.0.0"), architecture: "x64", ...(f.loadedCore ?? {}), + })); + Object.assign(result, f.outputs?.[calls++] ?? f.manifest.assertions.pass); + } + return result; + }; + return f; +} + +test("runner executes inventoried FSI twice in independent credential-free package caches", async t => { + const f = await runnerFixture(t); + const environments = []; + const result = await runReproduction(f, { execute: (file, args, settings) => { + environments.push(settings.env); + return f.execute(file, args, settings); + } }); + assert.equal(result.outcome.outcome, "pass", result.error); + assert.equal(result.cells[0].attempts.length, 2); + const commands = result.cells[0].evidence.map(e => e.commands[0]); + assert.ok(commands.every(c => c.executable === f.toolchain.hostPath && !c.args.includes("fsi"))); + const caches = new Set(environments.map(e => e.NUGET_PACKAGES)); + assert.equal(caches.size, 3); + assert.ok(environments.every(e => !Object.keys(e).some(k => /TOKEN|SECRET|PASSWORD/.test(k) && k !== "NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED"))); + assert.equal(JSON.parse(await fs.readFile(path.join(result.workDirectory, "result.json"))).outcome.outcome, "pass"); +}); + +for (const [name, change, expected] of [ + ["wrong loaded Core", f => { f.loadedCore = { sha256: "0".repeat(64) }; }, "wrong-identity"], + ["wrong loaded Core path", f => { f.loadedCore = { path: f.product.files.find(p => p.role === "fsi").path }; }, "wrong-identity"], + ["wrong loaded Core version", f => { f.loadedCore = { assemblyVersion: "1.0.0.0" }; }, "wrong-identity"], + ["wrong loaded runtime", f => { f.loadedCore = { runtimeVersion: "9.0.0" }; }, "wrong-identity"], + ["wrong loaded runtime path", f => { f.loadedCore = { runtimePath: f.root }; }, "wrong-identity"], + ["wrong loaded architecture", f => { f.loadedCore = { architecture: "arm64" }; }, "wrong-identity"], + ["flaky actual repetitions", f => { f.outputs = [f.manifest.assertions.pass, f.manifest.assertions.failure]; }, "flaky"], + ["truncated assertion output", f => { f.outputs = [{ ...f.manifest.assertions.pass, complete: false }]; }, "incomplete-evidence"], + ["timed out reproduction", f => { f.outputs = [{ ...f.manifest.assertions.pass, status: "timeout", exitCode: null }]; }, "timeout"], +]) { + test(`runner classifies ${name}`, async t => { + const f = await runnerFixture(t); + change(f); + const result = await runReproduction(f, { execute: f.execute }); + assert.equal(result.outcome.reason, expected, result.error); + }); +} + +test("bounded process capture retains no more than the configured bytes", async t => { + const f = await fixture(t); + const result = await execute(process.execPath, ["-e", "process.stdout.write('x'.repeat(100000));process.stderr.write('y'.repeat(100000))"], { + cwd: f.root, env: isolatedEnvironment(f.root, { os: process.platform === "win32" ? "windows" : "linux", architecture: process.arch }), + timeoutMs: 30000, maxBytes: 128, + }); + + test("a frozen managed exception exit code is an assertion, not automatically a setup crash", async t => { + const f = await fixture(t); + f.manifest.assertions.failure = { + exitCode: process.platform === "win32" ? 3762504530 : 82, + stdout: "", stderr: "Unhandled exception. System.InvalidOperationException: FROZEN_EXCEPTION\n", + }; + seal(f.manifest); + const execution = await execute(process.execPath, ["-e", + `process.stderr.write(${JSON.stringify(f.manifest.assertions.failure.stderr)});process.exit(-532462766)`], { + cwd: f.root, env: isolatedEnvironment(f.root, { os: process.platform === "win32" ? "windows" : "linux", architecture: process.arch }), + timeoutMs: 30000, + }); + assert.equal(evaluateAssertion(f.manifest, execution).outcome, "product-failure", JSON.stringify(execution)); + assert.equal(evaluateAssertion(f.manifest, { + status: "completed", complete: true, ...f.manifest.assertions.failure, + stderr: "Unhandled exception. System.NullReferenceException: unexpected\n", + }).outcome, "untestable"); + }); + assert.equal(result.exitCode, 0); + assert.equal(result.complete, false); + assert.equal(Buffer.byteLength(result.stdout), 128); + assert.equal(Buffer.byteLength(result.stderr), 128); +}); + +test("timeout kills a real child process tree", async t => { + const f = await fixture(t); + const result = await execute(process.execPath, ["-e", + "const c=require('node:child_process').spawn(process.execPath,['-e','setInterval(()=>{},1000)'],{stdio:'ignore'});console.log(c.pid);setInterval(()=>{},1000)"], { + cwd: f.root, env: isolatedEnvironment(f.root, { os: process.platform === "win32" ? "windows" : "linux", architecture: process.arch }), + timeoutMs: 3000, + }); + assert.equal(result.status, "timeout"); + const pid = Number(result.stdout.trim()); + assert.ok(Number.isSafeInteger(pid) && pid > 0, result.error); + assert.throws(() => process.kill(pid, 0), { code: "ESRCH" }); +}); + +test("same-version local Core packages never share a cache across observations", async t => { + const f = await runnerFixture(t); + const first = await runReproduction(f, { execute: f.execute }); + for (const role of ["core", "corePackage"]) { + const record = f.product.files.find(p => p.role === role); + const bytes = `second-build-${role}`; + await fs.writeFile(record.path, bytes); + record.sha256 = hash(bytes); + } + const second = await runReproduction(f, { execute: f.execute }); + assert.equal(first.outcome.outcome, "pass", first.error); + assert.equal(second.outcome.outcome, "pass", second.error); + assert.notEqual(first.productFingerprint, second.productFingerprint); + assert.notEqual(first.workDirectory, second.workDirectory); +}); + +for (const [tamper, compressed] of [[false, false], [true, false], [false, true]]) test( + `frozen producer metadata: tamper=${tamper}, compressed=${compressed}`, async t => { + const f = await runnerFixture(t); + const dll = await f.file("source/producer/Old.dll", "old inline metadata"); + const pkg = await f.file("source/producer/Old.1.0.0.nupkg", "old package bytes"); + const resources = [{ name: `FSharpSignature${compressed ? "Compressed" : ""}Data.Old`, sha256: hash("old inline signature") }]; + f.inspectFiles = [{ ...dll, assemblyVersion: "1.0.0.0", resources }]; + const descriptor = { + role: "immutable-producer", compiler: { version: "8.0.100", sha256: hash("released compiler") }, + flags: ["--optimize+", "--define:OLD_PRODUCER"], tfm: "net8.0", packages: [], + dll: { path: "producer/Old.dll", sha256: dll.sha256, resources }, + package: { path: "producer/Old.1.0.0.nupkg", sha256: pkg.sha256, id: "Old", version: "1.0.0" }, + }; + const provenance = await f.file("source/old.producer.json", JSON.stringify(descriptor)); + f.manifest.sources.push( + { path: descriptor.dll.path, sha256: dll.sha256 }, + { path: descriptor.package.path, sha256: pkg.sha256 }, + { path: "old.producer.json", sha256: provenance.sha256 }); + f.manifest.producer = { name: "Old", version: "1.0.0", sha256: dll.sha256 }; + f.manifest.packages = [{ id: "Old", version: "1.0.0", sha256: pkg.sha256 }]; + seal(f.manifest); + const result = await runReproduction(f, { execute: async (file, args, settings) => { + const result = await f.execute(file, args, settings); + if (tamper && file !== f.helper.executable) await fs.writeFile(path.join(settings.cwd, "producer", "Old.dll"), "rebuilt metadata hides failure"); + return result; + } }); + assert.equal(result.outcome.outcome, tamper ? "untestable" : "pass", result.error); + if (tamper) assert.match(result.error, /artifact bytes changed/); + else { + assert.deepEqual(result.cells[0].evidence[0].producers[0], descriptor); + assert.equal(hash(await fs.readFile(dll.path)), dll.sha256); + assert.equal(hash(await fs.readFile(pkg.path)), pkg.sha256); + } +}); From e6ee82e2f1e730ea2ac64a05925dd180a7dfe2df Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 05:10:21 +0200 Subject: [PATCH 15/99] Fix reproduction hook lifetime and nullable identity evidence Use weak assembly tracking, verify generated nullable flags, and strengthen containment and referenced-project coverage. Validate with scoped contracts and released/clean-local product smoke fixtures. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 30 ++- eng/regression-bisect/ReproEvidence.targets | 1 + .../evidence/BinlogEvidence.cs | 29 ++- .../evidence/StartupHook/StartupHook.cs | 6 +- eng/regression-bisect/fixtures/smoke.cjs | 69 ++++++- eng/regression-bisect/repro.cjs | 3 +- eng/regression-bisect/repro.smoke.cjs | 24 ++- eng/regression-bisect/repro.test.cjs | 190 ++++++++++++++++-- 8 files changed, 325 insertions(+), 27 deletions(-) diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index 2f0419eaaf3..bd6d8094055 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -365,6 +365,8 @@ with an absolute acquired host and `exec --fx-version`; it never invokes `dotnet fsi`. Windows executable layouts use a separate fixed Core-load probe. Managed executions capture the actual loaded Core through a startup hook, including its path, digest, assembly version, runtime version and architecture. +The hook deduplicates assemblies weakly so observation does not keep collectible +load contexts alive. Unsupported host/runtime/helper combinations are untestable. Project builds reuse **both** `UseLocalCompiler.Directory.Build` imports. @@ -387,6 +389,11 @@ is insufficient. The pinned structured binlog reader must find the executed Fsc task and correlate its assembly, command, references and flags with imports, assets and the expected project/TFM/configuration cell. Missing inner builds, skipped Fsc tasks, fallbacks and late flag overrides are untestable. +The pre-compile snapshot includes `Nullable`: the reader accounts for Fsc's +generated `NULLABLE` define and compares the executed `--checknulls` setting, +including when that define is also explicitly declared. Referenced projects must +be reviewed sources, with complete compiler/Core evidence for their selected +TFMs; ambiguous or missing dependency builds cannot be ignored. Assertions compare complete, bounded streams and exit codes. Line endings are normalized to LF. Compile-time assertions use a frozen diagnostic of the form @@ -440,7 +447,9 @@ reader's .NET 8+ host. The smoke covers FSI, Debug/Release projects, successful compilation followed by an intended runtime failure, intended diagnostics, late compiler flag overrides, explicit/central Core package overrides, two inner TFMs on Windows, and an immutable released -producer. Its source-evaluation control illustrates how rebuilding a producer +producer. It also covers nullable settings and late overrides, app/library +references with same/different TFMs, and collectible Core unloading with and +without the observation hook. Its source-evaluation control illustrates how rebuilding a producer without its old flags would hide the failure; candidate consumers only receive the frozen DLL/package. Results are written to `smoke-results.json`. An explicit final `released` or `source` argument selects just that phase while @@ -479,6 +488,25 @@ Complete execution evidence is retained at `Q:\rb912-s4-validated-fixtures`; `b182f1a3c504a9d4da94b0cfda10cde5e5280c0e20deb89338160038e3991ff1`. These checks do not imply a full compiler-test-suite pass. +Repair validation on 2026-09-19: **483** scoped Node tests and **67** targeted +reproduction tests passed. Before the fixes, the collectible-Core fixture +reported a product failure only with the hook, and a normal nullable-enabled +project was rejected. Both now pass; late nullable changes in either direction +are rejected even with an explicit `NULLABLE` define. Independently replacing +containment with prefix comparison or bypassing realpath resolution made the +corresponding repaired test fail. + +All **39** real bounded scenarios matched their expected outcomes under both +released and local products, including app/library TFM selection and hook-free +unloading controls. The unchanged clean local product build above was reused +after rechecking its pristine source checkout, successful build binlog and +artifact digests; the repaired evidence helper was rebuilt in Release with +zero warnings/errors. Evidence and exact identities are retained at +`Q:\rb912-s4-repaired-fixtures`, including `validation-summary.json`. +`smoke-results.json` SHA-256 is +`2b61651b316f343a2be775d203467437ab0104d8b6d14765981d1972808ab47c`. +Syntax checks, `dotnet fantomas .` (no changes), and diff checks passed. + ## Complete intake and bounded discovery `intake.cjs` exports `readIssueSnapshot(options)`, diff --git a/eng/regression-bisect/ReproEvidence.targets b/eng/regression-bisect/ReproEvidence.targets index fc0be5a1a0b..526e536bb20 100644 --- a/eng/regression-bisect/ReproEvidence.targets +++ b/eng/regression-bisect/ReproEvidence.targets @@ -12,6 +12,7 @@ + true, + "disable" => false, + _ => null + }; + var defines = snapshot.GetValueOrDefault("DefineConstants") + ?.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) ?? []; + // Fsc adds this define from Nullable independently of DefineConstants. + if (nullable == true) defines = [.. defines, "NULLABLE"]; var expected = ReadFlags( SplitArguments(snapshot.GetValueOrDefault("OtherFlags") ?? "", diagnostics), string.IsNullOrWhiteSpace(snapshot.GetValueOrDefault("LangVersion")) ? "default" : snapshot["LangVersion"], bool.TryParse(snapshot.GetValueOrDefault("Optimize"), out bool optimize) ? optimize : null, - snapshot.GetValueOrDefault("DefineConstants")?.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); + defines, nullable); var cell = new Cell { @@ -62,9 +74,11 @@ internal static object Read(string file) Configuration = Value("Configuration") ?? project.Configuration, LanguageVersion = fscTasks.LastOrDefault()?.LanguageVersion, Optimize = fscTasks.LastOrDefault()?.Optimize, + Nullable = fscTasks.LastOrDefault()?.Nullable, Defines = fscTasks.LastOrDefault()?.Defines ?? [], ExpectedLanguageVersion = expected.LanguageVersion, ExpectedOptimize = expected.Optimize, + ExpectedNullable = expected.Nullable, ExpectedDefines = expected.Defines, SdkVersion = Value("NETCoreSdkVersion"), HostPath = fscTasks.LastOrDefault()?.Executable, @@ -148,7 +162,7 @@ private static FscRecord ReadTask(BuildTask task, string directory) tokens = arguments.Split('\n').Select(t => t.Trim()).Where(t => t.Length > 0).ToArray(); else tokens = SplitArguments(arguments, record.MissingEvidence); - (record.LanguageVersion, record.Optimize, record.Defines) = ReadFlags(tokens); + (record.LanguageVersion, record.Optimize, record.Defines, record.Nullable) = ReadFlags(tokens); var references = new List(); foreach (string token in tokens) { @@ -200,17 +214,19 @@ private static string[] SplitArguments(string arguments, List diagnostic return matches.Select(m => m.Groups[1].Value.Replace("\"", "")).ToArray(); } - private static (string LanguageVersion, bool? Optimize, string[] Defines) ReadFlags( - string[] tokens, string languageVersion = "default", bool? optimize = null, string[]? defines = null) + private static (string LanguageVersion, bool? Optimize, string[] Defines, bool? Nullable) ReadFlags( + string[] tokens, string languageVersion = "default", bool? optimize = null, string[]? defines = null, bool? nullable = null) { var languageFlag = tokens.Select(t => Regex.Match(t, "^(?:--|/)langversion:(.*)$")).LastOrDefault(m => m.Success); var optimizeFlag = tokens.Select(t => Regex.Match(t, "^(?:--optimize|/optimize|-O|/O)([+-]?)$")).LastOrDefault(m => m.Success); + var nullableFlag = tokens.Select(t => Regex.Match(t, "^(?:--|/)checknulls([+-]?)$")).LastOrDefault(m => m.Success); var defineFlags = tokens.Select(t => Regex.Match(t, "^(?:--define|-d|/define|/d):(.*)$")) .Where(m => m.Success).Select(m => m.Groups[1].Value); return ( languageFlag?.Groups[1].Value.Trim('"') ?? languageVersion, optimizeFlag == null ? optimize : optimizeFlag.Groups[1].Value != "-", - (defines ?? []).Concat(defineFlags).Distinct(StringComparer.Ordinal).ToArray()); + (defines ?? []).Concat(defineFlags).Distinct(StringComparer.Ordinal).ToArray(), + nullableFlag == null ? nullable : nullableFlag.Groups[1].Value != "-"); } private static Project? Owner(BaseNode node) @@ -248,9 +264,11 @@ private sealed class Cell public string? Configuration { get; init; } public string? LanguageVersion { get; init; } public bool? Optimize { get; init; } + public bool? Nullable { get; init; } public string[] Defines { get; init; } = []; public string? ExpectedLanguageVersion { get; init; } public bool? ExpectedOptimize { get; init; } + public bool? ExpectedNullable { get; init; } public string[] ExpectedDefines { get; init; } = []; public string? SdkVersion { get; init; } public string? HostPath { get; init; } @@ -275,6 +293,7 @@ private sealed class FscRecord public string? Arguments { get; set; } public string? LanguageVersion { get; set; } public bool? Optimize { get; set; } + public bool? Nullable { get; set; } public string[] Defines { get; set; } = []; public required Dictionary Parameters { get; init; } public string[] References { get; set; } = []; diff --git a/eng/regression-bisect/evidence/StartupHook/StartupHook.cs b/eng/regression-bisect/evidence/StartupHook/StartupHook.cs index 856fd661d76..04993dd3a46 100644 --- a/eng/regression-bisect/evidence/StartupHook/StartupHook.cs +++ b/eng/regression-bisect/evidence/StartupHook/StartupHook.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using System.Reflection; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Serialization.Json; using System.Security.Cryptography; @@ -10,7 +11,7 @@ public static class StartupHook { private static readonly object Gate = new(); - private static readonly HashSet Seen = new(); + private static readonly ConditionalWeakTable Seen = new(); private static string? destination; public static void Initialize() @@ -32,8 +33,9 @@ private static void Record(Assembly assembly) return; lock (Gate) { - if (!Seen.Add(assembly)) + if (Seen.TryGetValue(assembly, out _)) return; + Seen.Add(assembly, new object()); try { string path = assembly.Location; diff --git a/eng/regression-bisect/fixtures/smoke.cjs b/eng/regression-bisect/fixtures/smoke.cjs index 13c428eaf1d..a8b9abbfd57 100644 --- a/eng/regression-bisect/fixtures/smoke.cjs +++ b/eng/regression-bisect/fixtures/smoke.cjs @@ -7,6 +7,15 @@ async function createFixture(directory, { kind, tfm, configuration = "Release", await fs.mkdir(directory, { recursive: true }); const contents = new Map(); const failing = scenario === "runtime-failure"; + const referenced = scenario.startsWith("reference-"); + const nullable = { + "nullable-enabled": ["enable"], + "nullable-disabled": ["disable"], + "late-nullable-enable": ["disable", "enable"], + "late-nullable-disable": ["enable", "disable"], + "late-nullable-explicit": ["enable", "disable"], + }[scenario]; + const projectTfm = scenario === "reference-tfm" ? `${tfm}-windows7.0` : tfm; const producerSource = [ "#if OLD_PRODUCER", 'let inline value () = "OLD_METADATA"', @@ -16,7 +25,8 @@ async function createFixture(directory, { kind, tfm, configuration = "Release", ].join("\n"); const body = producer ? 'if FrozenProducer.value () = "OLD_METADATA" then\n eprintfn "FROZEN_RUNTIME_FAILURE"\n 1\nelse\n printfn "PASS"\n 0' - : failing ? 'eprintfn "FROZEN_RUNTIME_FAILURE"\n1' : 'printfn "PASS"\n0'; + : failing ? 'eprintfn "FROZEN_RUNTIME_FAILURE"\n1' + : referenced ? 'printfn "%s" Library.message\n0' : 'printfn "PASS"\n0'; if (kind === "fsi") contents.set("repro.fsx", scenario === "producer-control" ? `${producerSource}\nprintfn "%s" (value ())\n` : `let result =\n${body.split("\n").map(l => ` ${l}`).join("\n")}\nexit result\n`); @@ -35,11 +45,41 @@ ${body.split("\n").map(l => ` ${l}`).join("\n")} " value", "", ].join("\n")); + if (scenario === "collectible-core") contents.set("Program.fs", `module Repro +open System +open System.IO +open System.Runtime.CompilerServices +open System.Runtime.Loader + +[] +let loadCore () = + let context = new AssemblyLoadContext("collectible-core", true) + context.LoadFromAssemblyPath(Path.Combine(AppContext.BaseDirectory, "FSharp.Core.dll")) |> ignore + let weak = WeakReference(context) + context.Unload() + weak + +[] +let main _ = + let weak = loadCore () + for _ in 1 .. 20 do + GC.Collect() + GC.WaitForPendingFinalizers() + GC.Collect() + if weak.IsAlive then + Console.Error.WriteLine("CONTEXT_RETAINED") + 1 + else + Console.WriteLine("PASS") + 0 +`); contents.set("Repro.fsproj", ` Exe - ${scenario === "multi-tfm" ? `${tfm};${tfm}-windows7.0` : `${tfm}`} + ${scenario === "multi-tfm" ? `${tfm};${tfm}-windows7.0` : `${projectTfm}`} false + ${nullable ? `${nullable[0]}` : ""} + ${scenario === "late-nullable-explicit" ? "$(DefineConstants);NULLABLE" : ""} ${scenario.startsWith("core-") ? "true" : ""} ${scenario === "producer" ? "FrozenProducerFrozenProducer1.0.0$(DefineConstants);OLD_PRODUCERtrue" : ""} @@ -48,10 +88,27 @@ ${body.split("\n").map(l => ` ${l}`).join("\n")} ${scenario === "core-explicit" ? `` : ""} ${scenario === "core-central" ? '' : ""} ${producer ? 'producer/FrozenProducer.dll' : ""} + ${referenced ? '' : ""} ${scenario === "late-flags" ? '$(OtherFlags) --optimize-' : ""} + ${nullable?.[1] ? `${nullable[1]}` : ""} + +`); + if (referenced) { + contents.set("Library/Library.fs", `module Library +let message = "PASS" +[] +let main _ = + printfn "%s" message + 0 +`); + contents.set("Library/Library.fsproj", ` + Exe${tfm}false + + ${scenario === "reference-late" ? '$(OtherFlags) --optimize-' : ""} `); + } if (scenario === "core-central") contents.set("Directory.Packages.props", ` true @@ -60,6 +117,7 @@ ${body.split("\n").map(l => ` ${l}`).join("\n")} } const sources = []; for (const [name, text] of contents) { + await fs.mkdir(path.dirname(path.join(directory, name)), { recursive: true }); await fs.writeFile(path.join(directory, name), text); sources.push({ path: name, sha256: hash(text) }); } @@ -88,7 +146,7 @@ ${body.split("\n").map(l => ` ${l}`).join("\n")} }, matrix: [{ id: `${kind}-${tfm}-${configuration}`, os: { win32: "windows", linux: "linux", darwin: "macos" }[process.platform], - architecture: process.arch, project, tfm, configuration, languageVersion: "8.0", + architecture: process.arch, project, tfm: projectTfm, configuration, languageVersion: nullable ? "9.0" : "8.0", command: [kind === "fsi" ? "fsi" : "run-project", project ?? "repro.fsx"], }], timeoutSeconds: 120, @@ -100,6 +158,11 @@ ${body.split("\n").map(l => ` ${l}`).join("\n")} pass: { exitCode: 0, stdout: "", stderr: "" }, failure: { exitCode: 1, stdout: "", stderr: "Program.fs(6,5): error FS0101: This construct is deprecated. FROZEN_DIAGNOSTIC\n" }, }; + if (scenario === "collectible-core") manifest.assertions.failure.stderr = "CONTEXT_RETAINED\n"; + if (scenario === "reference-tfm") manifest.matrix.push({ + ...manifest.matrix[0], id: `library-${tfm}-${configuration}`, tfm, + project: "Library/Library.fsproj", command: ["run-project", "Library/Library.fsproj"], + }); if (scenario === "multi-tfm") manifest.matrix.push({ ...manifest.matrix[0], id: `${kind}-${tfm}-windows7.0-${configuration}`, tfm: `${tfm}-windows7.0`, }); diff --git a/eng/regression-bisect/repro.cjs b/eng/regression-bisect/repro.cjs index 70829e61867..fbf60553f09 100644 --- a/eng/regression-bisect/repro.cjs +++ b/eng/regression-bisect/repro.cjs @@ -131,7 +131,7 @@ async function verifyProjectIdentities({ manifest, product, toolchain, sourceDir requireValue(samePath(await fs.realpath(fsc[field]), file.path), `${field} fallback or late override`); } requireValue(samePath(fsc.hostPath, toolchain.hostPath) && fsc.commandLine, "wrong Fsc host or missing invocation"); - requireValue(fsc.languageVersion === cell.languageVersion && fsc.optimize === row.optimize && + requireValue(fsc.languageVersion === cell.languageVersion && fsc.optimize === row.optimize && fsc.nullable === row.nullable && fingerprintInput([...fsc.defines].sort()) === fingerprintInput([...row.defines].sort()), "late compiler flag override"); const core = row.core; requireValue(core.package.id === "FSharp.Core" && core.package.version === product.corePackageVersion, "Core package version fallback"); @@ -479,6 +479,7 @@ async function runReproduction(options, dependencies = {}) { ...row, fsc: row.fscTasks, languageVersion: row.expectedLanguageVersion, optimize: row.expectedOptimize, + nullable: row.expectedNullable, defines: row.expectedDefines, assetsFile: path.resolve(path.dirname(row.project), row.assetsFile), targetPath: path.resolve(path.dirname(row.project), row.targetPath), diff --git a/eng/regression-bisect/repro.smoke.cjs b/eng/regression-bisect/repro.smoke.cjs index 11dd13ac79f..b6bb4fbce48 100644 --- a/eng/regression-bisect/repro.smoke.cjs +++ b/eng/regression-bisect/repro.smoke.cjs @@ -60,6 +60,15 @@ async function main() { ["project", "runtime-failure", "Release"], ["project", "diagnostic", "Release"], ["project", "late-flags", "Release"], + ["project", "nullable-enabled", "Release"], + ["project", "nullable-disabled", "Debug"], + ["project", "late-nullable-enable", "Release"], + ["project", "late-nullable-disable", "Release"], + ["project", "late-nullable-explicit", "Release"], + ["project", "collectible-core", "Release"], + ["project", "reference-pass", "Release"], + ["project", "reference-late", "Release"], + ...(process.platform === "win32" ? [["project", "reference-tfm", "Release"]] : []), ["project", "producer-consumer", "Release"], ...(process.platform === "win32" ? [["project", "multi-tfm", "Release"]] : []), ["fsi", "producer-control", "Release"], @@ -76,14 +85,25 @@ async function main() { product: input.product, toolchain: input.toolchain, helper: config.helper, sampleRuntimes: input.sampleRuntimes, }); - const expected = scenario === "late-flags" ? "untestable" + const lateOverride = scenario.startsWith("late-") || scenario === "reference-late"; + const expected = lateOverride ? "untestable" : ["runtime-failure", "producer-consumer", "diagnostic"].includes(scenario) ? "product-failure" : "pass"; results.push({ id, expected, result }); await fs.writeFile(path.join(root, "smoke-results.json"), JSON.stringify(results, null, 2)); console.log(`${id}: ${result.outcome.outcome} (${result.outcome.reason})${result.error ? `: ${result.error}` : ""}`); requireValue(result.outcome.outcome === expected, `bounded smoke failed: ${id}`); - if (scenario === "late-flags") requireValue(result.outcome.reason === "wrong-identity" && + if (lateOverride) requireValue(result.outcome.reason === "wrong-identity" && result.error === "late compiler flag override", "late override must be rejected for the intended identity mismatch"); + if (scenario === "collectible-core") { + const record = result.cells[0].evidence[0]; + const invocation = record.commands.at(-1); + const env = isolatedEnvironment(path.join(root, "helper"), input.toolchain.requested.platform); + const baseline = await execute(invocation.executable, invocation.args, { cwd: root, env, timeoutMs: 30000 }); + results.at(-1).uninstrumented = baseline; + await fs.writeFile(path.join(root, "smoke-results.json"), JSON.stringify(results, null, 2)); + requireValue(baseline.complete && baseline.exitCode === 0 && baseline.stdout === "PASS\n" && baseline.stderr === "", + "collectible context must unload both with and without observation"); + } } } } diff --git a/eng/regression-bisect/repro.test.cjs b/eng/regression-bisect/repro.test.cjs index 4fb1d181d73..64fb1bf878b 100644 --- a/eng/regression-bisect/repro.test.cjs +++ b/eng/regression-bisect/repro.test.cjs @@ -94,6 +94,7 @@ async function projectFixture(t, options) { const packageRoot = path.join(f.root, "packages"); const core = await f.file("packages/fsharp.core/10.0.100/lib/netstandard2.0/FSharp.Core.dll", await fs.readFile(roles.core.path)); const nupkg = await f.file("packages/fsharp.core/10.0.100/fsharp.core.10.0.100.nupkg", await fs.readFile(roles.corePackage.path)); + f.product.coreAssets = [{ path: "lib/netstandard2.0/FSharp.Core.dll", sha256: core.sha256 }]; f.packageRoot = packageRoot; f.evidence = f.manifest.matrix.map(c => ({ project: project.path, targetFramework: c.tfm, configuration: c.configuration, languageVersion: c.languageVersion, @@ -132,17 +133,6 @@ for (const [name, mutate] of [ ["no Fsc invocation", f => { f.evidence[0].fsc = []; }], ["wrong host", f => { f.evidence[0].fsc[0].hostPath = "dotnet"; }], ["producer rebuild", f => { f.evidence[0].projectReferences = ["Producer.fsproj"]; }], - ["false prefix", async f => { - const other = await f.file("packages-sibling/FSharp.Core.dll", "local/FSharp.Core.dll"); - f.evidence[0].core.compile = other; f.evidence[0].fsc[0].references = [other.path]; - }], - ["symlink", async f => { - const other = await f.file("outside/FSharp.Core.dll", "local/FSharp.Core.dll"); - const link = path.join(f.packageRoot, "escape"); - await fs.symlink(path.dirname(other.path), link, "junction"); - f.evidence[0].core.compile = { ...other, path: path.join(link, "FSharp.Core.dll") }; - f.evidence[0].fsc[0].references = [f.evidence[0].core.compile.path]; - }], ]) { test(`project rejects ${name}`, async t => { const f = await projectFixture(t, { tfms: ["net9.0", "net10.0"] }); @@ -151,6 +141,32 @@ for (const [name, mutate] of [ }); } +for (const [name, mutate] of [ + ["false prefix", async f => { + const pkg = f.evidence[0].core.package; + Object.assign(pkg, await f.file(path.join("packages-sibling", "fsharp.core", pkg.version, path.basename(pkg.path)), + await fs.readFile(pkg.path))); + return pkg.path; + }], + ["symlink", async f => { + const asset = f.evidence[0].core.compile; + const directory = path.dirname(asset.path); + const outside = path.join(f.root, "outside"); + await fs.rename(directory, outside); + await fs.symlink(outside, directory, "junction"); + return asset.path; + }], +]) { + test(`project containment rejects ${name}`, async t => { + const f = await projectFixture(t); + assert.equal((await verifyProjectIdentities(f)).status, "verified"); + const rejectedPath = await mutate(f); + const result = await verifyProjectIdentities(f); + assert.equal(result.status, "untestable"); + assert.equal(result.error, `artifact escapes expected root: ${rejectedPath}`); + }); +} + test("runner rejects unfrozen sources before invoking any process", async t => { const f = await fixture(t); await fs.appendFile(path.join(f.sourceDirectory, "repro.fsx"), " changed"); @@ -162,8 +178,8 @@ test("runner rejects unfrozen sources before invoking any process", async t => { assert.equal(invoked, false); }); -async function runnerFixture(t) { - const f = await fixture(t); +async function runnerFixture(t, options = {}) { + const f = await (options.project ? projectFixture : fixture)(t, options); f.workDirectory = path.join(f.root, "work"); f.helper = { executable: "helper-host", dll: "helper.dll", runtimeVersion: "10.0.0", hook: "CoreHook.dll" }; let calls = 0; @@ -192,6 +208,154 @@ async function runnerFixture(t) { return f; } +async function projectRunnerFixture(t, { declaredTfms = [], libraryTfm = "net10.0", mutate, buildExitCode = 0, + undeclared = false, nullable, defines = nullable ? ["NULLABLE"] : [] } = {}) { + const f = await runnerFixture(t, { project: true }); + const library = await f.file("source/Library.fsproj", ""); + if (!undeclared) f.manifest.sources.push({ path: "Library.fsproj", sha256: library.sha256 }); + const app = f.manifest.matrix[0]; + for (const tfm of declaredTfms) f.manifest.matrix.push({ + ...app, id: `library-${tfm}`, project: "Library.fsproj", tfm, languageVersion: "9.0", + command: ["run-project", "Library.fsproj"], + }); + seal(f.manifest); + f.product.configuration = app.configuration; + const roles = Object.fromEntries(f.product.files.map(p => [p.role, p])); + roles.compiler.runtimeConfig = roles.fsi.runtimeConfig; + const baseExecute = f.execute; + f.buildCalls = []; + f.runtimeCalls = []; + let structured; + f.execute = async (executable, args, settings) => { + const success = { status: "completed", complete: true, exitCode: 0, stdout: "", stderr: "" }; + if (executable === f.helper.executable && args.includes("binlog")) { + return { ...success, stdout: JSON.stringify(structured) }; + } + if (args[0] !== "build") { + if (executable !== f.helper.executable) f.runtimeCalls.push(args); + return baseExecute(executable, args, settings); + } + assert.equal(executable, f.toolchain.hostPath); + f.buildCalls.push(args); + const cell = f.manifest.matrix.find(c => path.join(settings.cwd, c.project) === args[1] && + args.includes(`-p:TargetFramework=${c.tfm}`)); + assert.ok(cell, "build must select a declared root cell"); + const write = (location, contents) => f.file(path.relative(f.root, location), contents); + const packageDirectory = path.join(settings.env.NUGET_PACKAGES, "fsharp.core", f.product.corePackageVersion); + const core = await write(path.join(packageDirectory, "lib", "netstandard2.0", "FSharp.Core.dll"), + await fs.readFile(roles.core.path)); + await write(path.join(packageDirectory, "fsharp.core.10.0.100.nupkg"), await fs.readFile(roles.corePackage.path)); + f.inspectFiles = [{ ...roles.core, ...core }]; + f.loadedCore = core; + const builds = [cell]; + if (cell.project === app.project) builds.push(f.manifest.matrix.find(c => + c.project === "Library.fsproj" && c.tfm === libraryTfm) ?? { ...cell, project: "Library.fsproj", tfm: libraryTfm }); + const rows = []; + for (const [index, current] of builds.entries()) { + const row = structuredClone(f.evidence[0]); + Object.assign(row, { + project: path.join(settings.cwd, current.project), targetFramework: current.tfm, + configuration: current.configuration, expectedLanguageVersion: current.languageVersion, + expectedOptimize: current.configuration === "Release", expectedDefines: defines, expectedNullable: nullable, + projectReferences: current.project === app.project ? ["Library.fsproj"] : [], + assetsFile: path.join(settings.cwd, ".runner", `${index}.assets.json`), + targetPath: path.join(settings.cwd, "bin", current.tfm, `${path.parse(current.project).name}.dll`), + fscTasks: row.fsc.map(task => ({ + ...task, references: [core.path], languageVersion: current.languageVersion, defines, nullable, + })), + }); + delete row.fsc; + delete row.core; + const identity = `FSharp.Core/${f.product.corePackageVersion}`; + await write(row.assetsFile, JSON.stringify({ + targets: { [current.tfm]: { [identity]: { + compile: { "lib/netstandard2.0/FSharp.Core.dll": {} }, runtime: { "lib/netstandard2.0/FSharp.Core.dll": {} }, + } } }, + libraries: { [identity]: { type: "package", path: `fsharp.core/${f.product.corePackageVersion}` } }, + packageFolders: { [settings.env.NUGET_PACKAGES]: {} }, + })); + await write(row.targetPath, "synthetic output"); + await write(path.join(settings.cwd, ".runner", `compiler-core-${index}.jsonl`), JSON.stringify({ + ...roles.core, assemblyName: "FSharp.Core", complete: true, entryAssembly: roles.compiler.path, + runtimeVersion: "10.0.0", runtimePath: path.join(f.toolchain.runtimes[0].path, "10.0.0"), architecture: "x64", + })); + rows.push(row); + } + structured = { cells: rows, diagnostics: [] }; + if (mutate) await mutate(structured, f); + await write(args.find(arg => arg.startsWith("-bl:")).slice(4), "synthetic binlog"); + return { ...success, exitCode: buildExitCode }; + }; + return f; +} + +for (const [name, options, error] of [ + ["inherited same-TFM library", {}], + ["declared same-TFM library wins over another declaration", { declaredTfms: ["net9.0", "net10.0"] }], + ["single declared different-TFM library", { declaredTfms: ["net9.0"], libraryTfm: "net9.0" }], + ...[false, true].flatMap(nullable => [ + [`nullable ${nullable ? "enabled" : "disabled"}`, { nullable }], + [`late library nullable ${nullable ? "disable" : "enable"} with explicit NULLABLE define`, { + nullable, defines: ["NULLABLE"], mutate: s => { s.cells[1].fscTasks[0].nullable = !nullable; }, + }, /late compiler flag override/], + [`missing library nullable evidence when ${nullable ? "enabled" : "disabled"}`, { + nullable, mutate: s => { s.cells[1].fscTasks[0].nullable = null; }, + }, /late compiler flag override/], + ]), + ["failed referenced-project evidence", { + buildExitCode: 1, mutate: s => { s.cells[1].complete = false; }, + }, /no unique executed Fsc task: cell-0-dependency-1/], + ["missing referenced-project evidence", { mutate: s => { s.cells.pop(); } }, + /missing or ambiguous expected inner build: Library\.fsproj net10\.0/], + ["wrong referenced-project TFM", { libraryTfm: "net9.0" }, + /missing or ambiguous expected inner build: Library\.fsproj net10\.0/], + ["ambiguous declared TFMs", { declaredTfms: ["net8.0", "net9.0"], libraryTfm: "net9.0" }, + /ambiguous referenced project TFM/], + ["undeclared reference", { undeclared: true }, + /undeclared project reference \(immutable producers must not be rebuilt\)/], + ["library without Fsc", { mutate: s => { s.cells[1].fscTasks = []; } }, + /no unique executed Fsc task: cell-0-dependency-1/], + ["library compiler fallback", { mutate: (s, f) => { s.cells[1].fscTasks[0].compilerPath = f.toolchain.hostPath; } }, + /compilerPath fallback or late override/], + ["library Core fallback", { mutate: (s, f) => { + s.cells[1].fscTasks[0].references = [f.product.files.find(p => p.role === "core").path]; + } }, /Fsc did not reference selected Core/], +]) { + test(`project runner verifies expanded matrix: ${name}`, async t => { + const f = await projectRunnerFixture(t, options); + const result = await runReproduction(f, { execute: f.execute }); + if (error) { + assert.equal(result.outcome.outcome, "untestable", result.error); + assert.equal(result.outcome.reason, "wrong-identity"); + assert.match(result.error, error); + assert.equal(f.buildCalls.length, 1); + assert.equal(f.runtimeCalls.length, 0, "rejected dependency evidence must prevent runtime execution"); + } else { + assert.equal(result.outcome.outcome, "pass", result.error); + assert.equal(result.cells.length, f.manifest.matrix.length); + assert.equal(f.buildCalls.length, result.cells.length * 2); + assert.equal(f.runtimeCalls.length, f.buildCalls.length); + for (const [index, cell] of result.cells.entries()) { + const root = f.manifest.matrix[index]; + const expected = [[root.project, root.tfm, root.configuration, root.languageVersion]]; + if (root.project === "App.fsproj") expected.push([ + "Library.fsproj", options.libraryTfm ?? root.tfm, root.configuration, + options.declaredTfms ? "9.0" : root.languageVersion, + ]); + assert.equal(cell.attempts.length, 2); + for (const record of cell.evidence) { + assert.deepEqual(record.projects.map(p => [ + path.basename(p.project), p.targetFramework, p.configuration, p.languageVersion, + ]), expected); + assert.ok(record.projects.every(p => p.nullable === options.nullable), "nullable snapshot must survive row projection"); + assert.equal(record.compilerCoreLoads.length, expected.length); + assert.equal(record.commands.at(-1).args.at(-1), record.projects[0].targetPath); + } + } + } + }); +} + test("runner executes inventoried FSI twice in independent credential-free package caches", async t => { const f = await runnerFixture(t); const environments = []; From bf13cf234bc47e62166bc82748d6784d0aee086f Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 06:06:39 +0200 Subject: [PATCH 16/99] Add isolated historical source revision workspaces Compose exact toolchain acquisition, clean product builds, artifact inventory and frozen reproduction execution. Add workspace isolation, cancellation and contamination contracts plus a bounded opt-in source smoke. Validation: syntax checks and 498 scoped Node tests passed. The pinned real smoke remains untestable because the declared SDK's public SHA-512 endpoints return 404; retain and document the exact blocker without claiming a successful compiler build. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 101 +++++ eng/regression-bisect/repro-process.cjs | 24 +- eng/regression-bisect/workspace.cjs | 442 ++++++++++++++++++++++ eng/regression-bisect/workspace.smoke.cjs | 35 ++ eng/regression-bisect/workspace.test.cjs | 310 +++++++++++++++ 5 files changed, 904 insertions(+), 8 deletions(-) create mode 100644 eng/regression-bisect/workspace.cjs create mode 100644 eng/regression-bisect/workspace.smoke.cjs create mode 100644 eng/regression-bisect/workspace.test.cjs diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index bd6d8094055..c5885dfeb44 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -327,6 +327,107 @@ was regenerated through its existing harness for the new pinned public-feed policy; an old-policy checkpoint is not silently accepted as a new-policy one. Root compiler Release builds/tests are not this Node adapter's acceptance gate. +## Historical revision workspaces + +`workspace.cjs` composes acquisition and reproduction without admission, Git +bisection, or publishing. `prepareRevisionWorkspace(options, dependencies?)` +returns an owned, fresh workspace handle; `buildRevision(handle, dependencies?)` +consumes it once. `testSourceRevision(options, dependencies?)` performs both and +runs the unchanged frozen manifest. All four phases (`main`, `older-search`, +`bisect`, `confirmation`) use this implementation. Even repeated confirmations +of the same SHA allocate a new UUID, checkout, SDK, package cache and sample. + +Trusted options are `manifest`, its reviewed `sourceDirectory`, an existing +`mirrorDirectory`, separate existing `scratchDirectory` and `evidenceDirectory`, +an absolute `gitExecutable`, optional `signal`, and this request: + +```json +{ + "schemaVersion": 1, + "repository": "dotnet/fsharp", + "sourceSha": "", + "manifestDigest": "", + "approvalDigest": "", + "phase": "confirmation", + "platform": { "os": "windows", "architecture": "x64", "rid": "win-x64" }, + "allocation": { "buildTimeoutMs": 5400000, "workerTimeoutMs": 6600000 } +} +``` + +The controller must associate the trusted mirror with the repository and reserve +the execution allocation before calling. The worker does not fetch issue-supplied +URLs. Git fetches only the pinned object from that mirror into an independent +repository, with empty templates, disabled hooks/credential helpers/submodules, +isolated configuration and Windows long-path support. Tracked links/build output +and unexpected state in a prepared checkout are rejected, not scrubbed or reused. +The environment is constructed from an allowlist, with private homes, caches and +temporary directories; source build scripts cannot print inherited tokens. +This is not an OS sandbox: the credential-free worker/container boundary remains +mandatory before executing historical source or reproductions. + +The explicit adapter supports nested `fsiProject`/`fscProject` Arcade layouts +with `.sln` or `.slnx` entrypoints and evaluated +`artifacts/bin//Release/` output. It reads that checkout's +`global.json`, TFM declaration, bootstrap/build scripts and NuGet configuration. +Windows uses its `eng\Build.ps1 -restore -build -c Release -pack -bootstrap +-noVisualStudio -msbuildEngine dotnet`, restricting `Projects` to that revision's +`Microsoft.FSharp.Compiler.fsproj` product graph rather than building the solution +or tests. Bash uses root `build.sh --configuration Release --pack --bootstrap +--product-build` (the root already supplies restore/build). Unsupported +entrypoints, declarations or SDK property-query support are explicitly untestable. +Historical files, TFMs and package versions are never rewritten. + +MSBuild-evaluated paths and package identity must agree before/after the build. +Inventory verifies bytes, assembly versions, runtimeconfigs and the Core package, +rejecting missing outputs, SDK-bundled compiler fallback and redirected artifacts. +No timestamp selection or shared same-version Core cache is used. Exact acquisition +uses the trusted checkout's existing Arcade acquisition helper, not a helper +assumed to exist in historical source. The evidence helper likewise comes from +the trusted adapter (or the controller's optional `helper` descriptor). Project +reproduction continues to import the trusted root +`UseLocalCompiler.Directory.Build.props` and `.targets`; their digests and the +adapter's layout/version/digest are recorded. + +Results contain the three-way `outcome`, requested/actual identities, commands, +exit codes, durations, build ID and complete reproduction repeats/matrix evidence. +Only `evidence[].path` entries (relative to `evidenceDirectory`, with SHA-256) +are upload candidates, not arbitrary paths appearing in command output. +Build logs/binlogs and observations persist outside scratch. Setup/build errors, +timeouts and wrong/missing identities cannot become product failures even when +their output matches the expected diagnostic. Cancellation terminates process +trees and removes only the UUID-marked, resolved owned scratch child; ordinary +results retain their scratch artifacts for diagnosis. Results expose no mutation +or publisher operation. Controller provenance/admission remains separate. + +### Bounded opt-in clean-build smoke + +```powershell +node eng\regression-bisect\workspace.smoke.cjs --run +node --test eng\regression-bisect\model.test.cjs eng\regression-bisect\toolchains.test.cjs eng\regression-bisect\repro.test.cjs eng\regression-bisect\workspace.test.cjs +``` + +The opt-in smoke freezes a harmless FSI fixture, uses the declared revision SDK, +builds the product graph cleanly and requests two verified local executions. +It prints its retained result/evidence location and exits 2 on an untestable +result. No live issue discovery or full compiler test suite is involved. + +Local verification on 2026-09-19: RED was observed before `workspace.cjs` existed; +the final scoped suite passed **498 tests**, including 15 workspace contracts. +The real Windows x64 smoke pinned +`e6ee82e2f1e730ea2ac64a05925dd180a7dfe2df`, build instance +`38b31a09-c802-495f-ad3a-2e4b006515ab`. Its accurate result is +**untestable / unavailable-sdk**, exit 2: exact SDK +`11.0.100-rc.1.26420.103` has no verified public archive available to acquisition. +Release metadata returned 200, but +`Sdk/11.0.100-rc.1.26420.103/dotnet-sdk-11.0.100-rc.1.26420.103-win-x64.zip.sha512` +returned 404 under both `https://builds.dotnet.microsoft.com/dotnet` and +`https://ci.dot.net/public`. Checkout/source-integrity checks passed and +`global.json` retained its hash. Acquisition stopped before the build entrypoint, +so no build binlog or local FSI result exists; this is not a green clean build +or compiler suite. Retained evidence is +`Q:\fsharp-workspace-smoke-r1n9dF\result.json`, SHA-256 +`4a3bf0528370c54471aa8e81e68f62e4db958a441fab69819e2b540a6c2f7bf5`. + ## Reproduction execution `repro.cjs` is an execution adapter, not an authorization boundary. Its caller diff --git a/eng/regression-bisect/repro-process.cjs b/eng/regression-bisect/repro-process.cjs index 6f0e5166048..d9a114dc30b 100644 --- a/eng/regression-bisect/repro-process.cjs +++ b/eng/regression-bisect/repro-process.cjs @@ -3,12 +3,16 @@ const { spawn } = require("node:child_process"); const { constants } = require("node:os"); const policy = require("../../.github/policies/regression-bisect.json"); -function execute(executable, args, { cwd, env, timeoutMs, maxBytes = policy.evidenceBytesPerStream }) { +function execute(executable, args, { cwd, env, timeoutMs, signal: cancellation, maxBytes = policy.evidenceBytesPerStream }) { return new Promise(resolve => { + if (cancellation?.aborted) { + resolve({ status: "cancelled", complete: false, exitCode: null, stdout: "", stderr: "" }); + return; + } const child = spawn(executable, args, { cwd, env, shell: false, windowsHide: true, detached: process.platform !== "win32" }); const streams = { stdout: [], stderr: [] }; const lengths = { stdout: 0, stderr: 0 }; - let timedOut = false, truncated = false, launchError = null, killing = Promise.resolve(); + let timedOut = false, cancelled = false, terminating = false, truncated = false, launchError = null, killing = Promise.resolve(); for (const name of Object.keys(streams)) child[name].on("data", bytes => { const remaining = Math.max(0, maxBytes - lengths[name]); if (bytes.length > remaining) truncated = true; @@ -16,9 +20,9 @@ function execute(executable, args, { cwd, env, timeoutMs, maxBytes = policy.evid lengths[name] += bytes.length; }); child.on("error", error => { launchError = error.message; }); - const timer = setTimeout(() => { - timedOut = true; - if (!child.pid) return; + const terminate = () => { + if (!child.pid || terminating) return; + terminating = true; if (process.platform === "win32") { killing = new Promise(done => { const killer = spawn(path.join(env.SystemRoot, "System32", "taskkill.exe"), ["/PID", String(child.pid), "/T", "/F"], @@ -29,16 +33,20 @@ function execute(executable, args, { cwd, env, timeoutMs, maxBytes = policy.evid } else { try { process.kill(-child.pid, "SIGKILL"); } catch (error) { launchError = `process-tree termination failed: ${error.message}`; } } - }, timeoutMs); + }; + const abort = () => { cancelled = true; terminate(); }; + cancellation?.addEventListener("abort", abort, { once: true }); + const timer = setTimeout(() => { timedOut = true; terminate(); }, timeoutMs); child.on("close", async (code, signal) => { clearTimeout(timer); + cancellation?.removeEventListener("abort", abort); await killing; const stdout = Buffer.concat(streams.stdout).toString("utf8").replace(/\r\n/g, "\n"); const stderr = Buffer.concat(streams.stderr).toString("utf8").replace(/\r\n/g, "\n"); const managedException = signal === "SIGABRT" && /^Unhandled exception\./.test(stderr); resolve({ - status: timedOut ? "timeout" : launchError || signal && !managedException ? "exception" : "completed", - complete: !truncated && !launchError, exitCode: managedException ? 128 + constants.signals[signal] : code, + status: cancelled ? "cancelled" : timedOut ? "timeout" : launchError || signal && !managedException ? "exception" : "completed", + complete: !truncated && !launchError && !cancelled && !timedOut, exitCode: managedException ? 128 + constants.signals[signal] : code, stdout, stderr, ...(launchError ? { error: launchError } : {}), }); diff --git a/eng/regression-bisect/workspace.cjs b/eng/regression-bisect/workspace.cjs new file mode 100644 index 00000000000..7836d9b974f --- /dev/null +++ b/eng/regression-bisect/workspace.cjs @@ -0,0 +1,442 @@ +const fs = require("node:fs/promises"); +const path = require("node:path"); +const { randomUUID } = require("node:crypto"); +const { validateManifest, fingerprintInput } = require("./model.cjs"); +const { readRevisionToolchain, acquireToolchain } = require("./toolchains.cjs"); +const { runReproduction } = require("./repro.cjs"); +const { execute } = require("./repro-process.cjs"); +const { hash, inside, requireValue, isolatedEnvironment, fileIdentity } = require("./toolchain-io.cjs"); +const policy = require("../../.github/policies/regression-bisect.json"); + +const trustedRoot = path.resolve(__dirname, "..", ".."); +const owned = new WeakSet(); +const relative = (root, file) => path.relative(root, file).split(path.sep).join("/"); +const untestable = reason => ({ schemaVersion: 1, outcome: "untestable", reason }); + +function validateRequest(options) { + validateManifest(options.manifest); + const r = options.request; + requireValue(r?.schemaVersion === 1 && r.repository === policy.repository && + /^[a-f0-9]{40}$/.test(r.sourceSha), "expected trusted repository and full source SHA"); + requireValue(Object.keys(r).sort().join() === + "allocation,approvalDigest,manifestDigest,phase,platform,repository,schemaVersion,sourceSha", "unexpected work request fields"); + requireValue(r.manifestDigest === fingerprintInput(options.manifest) && + r.approvalDigest === options.manifest.approvedInputDigest, "frozen manifest/approval digest mismatch"); + requireValue(["main", "older-search", "bisect", "confirmation"].includes(r.phase), "unsupported source phase"); + const p = r.platform; + requireValue(["windows", "linux", "macos"].includes(p?.os) && ["x64", "arm64"].includes(p.architecture) && + (p.rid === `${{ windows: "win", linux: "linux", macos: "osx" }[p.os]}-${p.architecture}` || + p.os === "linux" && p.rid === `linux-musl-${p.architecture}`) && + options.manifest.matrix.every(c => c.os === p.os && c.architecture === p.architecture), "unapproved platform"); + requireValue(Object.keys(r.allocation).sort().join() === "buildTimeoutMs,workerTimeoutMs" && + Number.isSafeInteger(r.allocation.buildTimeoutMs) && r.allocation.buildTimeoutMs > 0 && + r.allocation.buildTimeoutMs <= policy.limits.buildTimeoutMinutes * 60000 && + Number.isSafeInteger(r.allocation.workerTimeoutMs) && + r.allocation.workerTimeoutMs >= r.allocation.buildTimeoutMs && + r.allocation.workerTimeoutMs <= policy.limits.workerTimeoutMinutes * 60000, "invalid bounded allocation"); + requireValue(path.isAbsolute(options.gitExecutable), "trusted absolute Git executable required"); +} + +async function evidence(w, name, value) { + requireValue(/^[a-z0-9.-]+$/.test(name), "invalid evidence name"); + const bytes = typeof value === "string" || Buffer.isBuffer(value) ? value : JSON.stringify(value, null, 2); + const file = path.join(w.evidenceRoot, name); + await fs.writeFile(file, bytes, { flag: "wx" }); + w.evidence.push({ path: `${w.buildInstanceId}/${name}`, sha256: hash(bytes) }); +} + +function failure(w, stage, error) { + const code = w?.signal?.aborted ? (w.options.signal?.aborted ? "cancelled" : "timeout") : error.code ?? stage; + return { + status: "untestable", outcome: untestable(code === "timeout" ? "timeout" : + { build: "build-failure", identity: "wrong-identity", sdk: "unavailable-sdk", runtime: "unavailable-runtime" }[stage] ?? "setup-failure"), + error: { stage, code, message: error.message }, + }; +} + +async function invoke(w, stage, executable, args, settings = {}, boundary = w.execute) { + w.signal.throwIfAborted(); + const start = Date.now(); + const entry = { stage, executable, args, cwd: settings.cwd ?? w.sourceDirectory }; + w.commands.push(entry); + const result = await boundary(executable, args, { + cwd: entry.cwd, env: w.env, maxBytes: 16 * 1024 * 1024, + ...settings, signal: w.signal, + timeoutMs: Math.max(1, Math.min(w.deadline - start, w.options.request.allocation.buildTimeoutMs, settings.timeoutMs ?? Infinity)), + }); + entry.durationMs = Date.now() - start; + entry.exitCode = result.exitCode; + entry.status = result.status ?? "completed"; + await evidence(w, `command-${w.commands.length}.json`, { ...entry, result }); + if (result.exitCode !== 0 || result.status === "timeout" || result.status === "cancelled" || result.complete === false) { + throw Object.assign(new Error( + `${stage} command failed (${entry.status}, exit ${result.exitCode}): ${executable} ${args.join(" ")}\n${result.stderr}`), + { code: result.status === "timeout" || result.status === "cancelled" ? result.status : "command-failure" }); + } + w.signal.throwIfAborted(); + return result.stdout.trim(); +} + +const gitArgs = w => ["-c", `core.hooksPath=${path.join(w.root, "hooks")}`, "-c", "core.fsmonitor=false", "-c", "core.longpaths=true", + "-c", "credential.helper=", "-c", "submodule.recurse=false", "-c", "protocol.allow=never", "-c", "protocol.file.allow=always"]; +const git = (w, args) => invoke(w, "source", w.options.gitExecutable, [...gitArgs(w), ...args]); + +async function sourceUnchanged(w) { + const after = hash(await fs.readFile(path.join(w.sourceDirectory, "global.json"))); + w.globalJson.after = after; + requireValue(after === w.globalJson.before, "historical global.json changed"); + requireValue(await git(w, ["rev-parse", "HEAD"]) === w.actualSourceSha, "historical checkout SHA changed"); + await git(w, ["diff", "--exit-code", "HEAD", "--"]); +} + +async function prepareRevisionWorkspace(options, dependencies = {}) { + let w; + try { + validateRequest(options); + const parent = await fs.realpath(options.scratchDirectory); + const evidenceParent = await fs.realpath(options.evidenceDirectory); + const mirror = await fs.realpath(options.mirrorDirectory); + requireValue(parent !== mirror && !inside(mirror, parent) && !inside(parent, mirror) && + parent !== evidenceParent && !inside(parent, evidenceParent), "scratch, mirror and evidence must be separate"); + const buildInstanceId = randomUUID(); + const root = path.join(parent, `revision-${buildInstanceId}`); + await fs.mkdir(root); + const signal = AbortSignal.any([AbortSignal.timeout(options.request.allocation.workerTimeoutMs), + ...(options.signal ? [options.signal] : [])]); + w = { + options, root, parent, buildInstanceId, signal, started: Date.now(), deadline: Date.now() + options.request.allocation.workerTimeoutMs, + sourceDirectory: path.join(root, "source"), sampleDirectory: path.join(root, "sample"), + evidenceRoot: path.join(evidenceParent, buildInstanceId), evidence: [], commands: [], + execute: dependencies.execute ?? execute, env: isolatedEnvironment(root, options.request.platform), + }; + owned.add(w); + Object.assign(w.env, { + GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: path.join(root, "gitconfig"), + GIT_TERMINAL_PROMPT: "0", GCM_INTERACTIVE: "Never", + GIT_ALLOW_PROTOCOL: "file", GIT_LFS_SKIP_SMUDGE: "1", + NUGET_PLUGIN_PATHS: "", NUGET_FALLBACK_PACKAGES: "", + }); + if (options.request.platform.os === "windows") Object.assign(w.env, { + OS: "Windows_NT", PATHEXT: ".COM;.EXE;.BAT;.CMD", + PSModulePath: path.join(w.env.SystemRoot, "System32", "WindowsPowerShell", "v1.0", "Modules"), + ProgramFiles: path.join(root, "program-files"), "ProgramFiles(x86)": path.join(root, "program-files-x86"), + ProgramData: path.join(root, "program-data"), ALLUSERSPROFILE: path.join(root, "program-data"), + }); + for (const dir of [w.sourceDirectory, w.sampleDirectory, w.evidenceRoot, path.join(root, "hooks"), + w.env.HOME, w.env.TEMP, w.env.APPDATA, w.env.LOCALAPPDATA, w.env.NUGET_PACKAGES]) await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(w.env.GIT_CONFIG_GLOBAL, "", { flag: "wx" }); + await fs.writeFile(path.join(root, "owner.json"), JSON.stringify({ buildInstanceId, parent }), { flag: "wx" }); + await evidence(w, "request.json", options.request); + await git(w, ["init", "--template="]); + await git(w, ["fetch", "--no-tags", "--depth=1", "--no-recurse-submodules", mirror, options.request.sourceSha]); + const tree = await git(w, ["ls-tree", "-r", "FETCH_HEAD"]); + requireValue(!tree.split("\n").some(line => /^(120000|160000) /.test(line) || + /(?:\t|\/)(?:artifacts|bin|obj|\.dotnet|Bootstrap)\//i.test(line)), "unsupported tracked link/submodule/build output"); + await git(w, ["checkout", "--detach", options.request.sourceSha]); + w.actualSourceSha = await git(w, ["rev-parse", "HEAD"]); + requireValue(w.actualSourceSha === options.request.sourceSha, "wrong checked-out source SHA"); + w.globalJson = { before: hash(await fs.readFile(path.join(w.sourceDirectory, "global.json"))) }; + await sourceUnchanged(w); + for (const source of options.manifest.sources) { + const file = await fileIdentity(path.join(options.sourceDirectory, source.path), await fs.realpath(options.sourceDirectory)); + requireValue(file.sha256 === source.sha256, "frozen reproduction bytes changed"); + const target = path.join(w.sampleDirectory, source.path); + await fs.mkdir(path.dirname(target), { recursive: true }); + await fs.copyFile(file.path, target); + } + w.status = "prepared"; + return w; + } catch (error) { + const result = failure(w, "setup", error); + return w ? finish(w, result) : result; + } +} + +async function layout(w) { + const files = ["build.sh", "Build.cmd", "eng/build.sh", "eng/Build.ps1", "eng/build-utils.ps1", "proto.proj", "NuGet.config", + "src/fsi/fsiProject/fsi.fsproj", "src/fsc/fscProject/fsc.fsproj", "src/FSharp.Core/FSharp.Core.fsproj", + "src/Microsoft.FSharp.Compiler/Microsoft.FSharp.Compiler.fsproj"]; + const declarations = {}; + for (const name of files) { + const file = await fileIdentity(path.join(w.sourceDirectory, name), w.sourceDirectory); + declarations[name] = { sha256: file.sha256, text: await fs.readFile(file.path, "utf8") }; + } + await evidence(w, "layout.json", declarations); + const ps = declarations["eng/Build.ps1"].text, sh = declarations["eng/build.sh"].text; + requireValue(/proto\.proj/.test(declarations["eng/build-utils.ps1"].text) && + /Configuration/.test(declarations["proto.proj"].text), "unsupported bootstrap entrypoint"); + const windows = w.options.request.platform.os === "windows"; + requireValue(windows + ? ["restore", "build", "pack", "bootstrap", "noVisualStudio", "msbuildEngine", "configuration"].every(s => ps.includes(`$${s}`)) && + ps.includes("/p:Projects=") && ps.includes("@properties") + : ["--bootstrap", "--pack", "--product-build", "proto.proj", "Configuration=Proto"].every(s => sh.includes(s)) && + /--restore --build/.test(declarations["build.sh"].text), "unsupported historical build entrypoint"); + requireValue(!//Release/ outputs", + injection: await Promise.all(["UseLocalCompiler.Directory.Build.props", "UseLocalCompiler.Directory.Build.targets"] + .map(async name => ({ path: name, sha256: hash(await fs.readFile(path.join(trustedRoot, name))) }))), + implementationSha256: hash(await fs.readFile(__filename)), + }; +} + +async function evaluated(w, project, targetFramework) { + const properties = ["TargetPath", "TargetFramework", "TargetFrameworks", "PackageId", "PackageVersion", "PackageOutputPath"]; + const args = ["msbuild", path.join(w.sourceDirectory, project), "-nologo", "-p:Configuration=Release", + `-getProperty:${properties.join(",")}`, ...(targetFramework ? [`-p:TargetFramework=${targetFramework}`] : [])]; + const output = await invoke(w, "evaluation", w.toolchain.hostPath, args); + const row = JSON.parse(output).Properties; + requireValue(row && properties.every(p => typeof row[p] === "string"), "unsupported evaluated project properties"); + return row; +} + +async function inspect(w, files) { + const h = w.helper; + return JSON.parse(await invoke(w, "inspection", h.executable, ["exec", "--fx-version", h.runtimeVersion, h.dll, "inspect", ...files])); +} + +async function helper(w) { + if (w.options.helper) { w.helper = w.options.helper; return; } + const directory = path.join(w.root, "helper"); + await fs.cp(path.join(__dirname, "evidence"), directory, { + recursive: true, filter: file => !["bin", "obj"].includes(path.basename(file)), + }); + const runtime = w.toolchain.compilerRuntime.version; + const tfm = `net${runtime.split(".")[0]}.0`; + const output = path.join(directory, "output"); + await fs.writeFile(path.join(directory, "NuGet.Config"), + ''); + await invoke(w, "helper", w.toolchain.hostPath, ["build", path.join(directory, "Evidence.csproj"), + "-c", "Release", "-o", output, `-p:TargetFramework=${tfm}`, "-nr:false", "--nologo"], { cwd: directory }); + w.helper = { executable: w.toolchain.hostPath, dll: path.join(output, "Evidence.dll"), + runtimeVersion: runtime, hook: path.join(output, "Repro.StartupHook.dll") }; +} + +async function inventory(w, rows) { + const files = []; + for (const [role, row] of Object.entries(rows)) { + if (role === "corePackage") { + requireValue(row.PackageId === "FSharp.Core" && /^[\w.+-]+$/.test(row.PackageVersion), "wrong evaluated Core package identity"); + const location = path.resolve(w.sourceDirectory, row.PackageOutputPath, `FSharp.Core.${row.PackageVersion}.nupkg`); + const file = await fileIdentity(location, w.sourceDirectory); + requireValue(file.path === location, "package output redirects through a link"); + files.push({ role, ...file }); + continue; + } + const expectedName = role === "fsi" ? "fsi" : "fsc"; + requireValue(row.TargetPath === path.join(w.sourceDirectory, "artifacts", "bin", expectedName, "Release", + w.targetFramework, `${expectedName}.dll`), + "unsupported or wrong evaluated product output"); + const file = { role, ...await fileIdentity(row.TargetPath, w.sourceDirectory), + runtimeConfig: await fileIdentity(row.TargetPath.replace(/\.dll$/, ".runtimeconfig.json"), w.sourceDirectory) }; + requireValue(file.path === row.TargetPath && file.runtimeConfig.path === row.TargetPath.replace(/\.dll$/, ".runtimeconfig.json"), + "product output redirects through a link"); + requireValue(file.sha256 !== w.toolchain.files[role === "fsi" ? "fsi" : "compiler"].sha256, "stale bundled compiler artifact"); + const config = JSON.parse(await fs.readFile(file.runtimeConfig.path, "utf8")).runtimeOptions; + requireValue(config?.tfm === w.targetFramework && config.framework?.name === "Microsoft.NETCore.App", "wrong product runtimeconfig"); + const runtimes = w.toolchain.runtimes.filter(r => r.name === config.framework.name && + r.version.split(".").slice(0, 2).join(".") === config.framework.version.split(".").slice(0, 2).join(".") && + Number(r.version.split(".")[2].split("-")[0]) >= Number(config.framework.version.split(".")[2].split("-")[0])); + if (runtimes.length !== 1) throw Object.assign(new Error(`missing or ambiguous product runtime: ${config.framework.version}`), + { code: "unavailable-runtime" }); + requireValue(!w.runtime || w.runtime.version === runtimes[0].version, "compiler and FSI require different runtimes"); + w.runtime = { name: runtimes[0].name, version: runtimes[0].version }; + files.push(file); + } + const fscDir = path.dirname(rows.compiler.TargetPath), fsiDir = path.dirname(rows.fsi.TargetPath); + for (const [role, name] of Object.entries({ + core: "FSharp.Core.dll", tasks: "FSharp.Build.dll", props: "Microsoft.FSharp.NetSdk.props", + targets: "Microsoft.FSharp.Targets", sdkTargets: "Microsoft.FSharp.NetSdk.targets", overrides: "Microsoft.FSharp.Overrides.NetSdk.targets", + })) { + const location = path.join(role === "core" ? fsiDir : fscDir, name); + const file = await fileIdentity(location, w.sourceDirectory); + requireValue(file.path === location, "product output redirects through a link"); + files.push({ role, ...file }); + } + for (const directory of [fsiDir, fscDir]) { + for (const name of await fs.readdir(directory, { recursive: true })) { + const location = path.join(directory, name); + const stat = await fs.lstat(location); + requireValue(!stat.isSymbolicLink(), "product inventory contains a link"); + if (stat.isDirectory() || files.some(f => f.path === location)) continue; + files.push({ role: `dependency-${files.length}`, ...await fileIdentity(location, w.sourceDirectory) }); + } + } + const inspected = await inspect(w, files.map(f => f.path)); + for (const file of files) { + const actual = inspected.find(i => path.resolve(i.path) === file.path); + requireValue(actual?.sha256 === file.sha256, "artifact inspection digest mismatch"); + if (["fsi", "compiler", "core", "tasks"].includes(file.role)) requireValue(actual.assemblyVersion && actual.fileVersion && + actual.assemblyName === path.basename(file.path, ".dll"), "artifact assembly identity mismatch"); + Object.assign(file, actual, { role: file.role }); + } + const pkg = files.find(f => f.role === "corePackage"); + requireValue(pkg.id === "FSharp.Core" && pkg.version === rows.corePackage.PackageVersion && + pkg.entries.some(e => /^lib\/[^/]+\/FSharp\.Core\.dll$/.test(e.path) && e.sha256 === files.find(f => f.role === "core").sha256), + "Core package identity/bytes do not match the built FSI"); + return { + kind: "source", root: w.sourceDirectory, sourceSha: w.actualSourceSha, buildInstanceId: w.buildInstanceId, + configuration: "Release", targetFramework: w.targetFramework, runtime: w.runtime, + corePackageVersion: pkg.version, files, + }; +} + +async function buildRevision(w, dependencies = {}) { + let stage = "setup"; + try { + requireValue(owned.has(w) && w.status === "prepared", "workspace is not a fresh prepared build instance"); + w.status = "building"; + await sourceUnchanged(w); + requireValue(await git(w, ["ls-files", "--others", "--exclude-standard"]) === "" && + await git(w, ["ls-files", "--others", "--ignored", "--exclude-standard"]) === "", + "unexpected state in fresh revision checkout"); + stage = "layout"; + await layout(w); + stage = "sdk"; + const declaration = await readRevisionToolchain({ + policy, sourceDirectory: w.sourceDirectory, sourceSha: w.actualSourceSha, platform: w.options.request.platform, + }); + requireValue(declaration.status === "resolved", declaration.error?.message ?? "unsupported source toolchain"); + w.declaration = declaration.toolchain; + await evidence(w, "declarations.json", declaration); + stage = "sdk"; + const acquisition = await acquireToolchain({ + policy, toolchain: declaration.toolchain, storageDirectory: w.root, helperRoot: trustedRoot, signal: w.signal, + }, { + ...dependencies.toolchains, + run: async (exe, args, settings) => { + const stdout = await invoke(w, "acquisition", exe, args, settings, dependencies.toolchains?.run ?? w.execute); + return { exitCode: 0, stdout, stderr: "" }; + }, + }); + await evidence(w, "acquisition.json", acquisition); + if (acquisition.outcome?.reason === "unavailable-runtime") stage = "runtime"; + requireValue(acquisition.status === "acquired", acquisition.error?.message ?? "toolchain acquisition failed"); + w.toolchain = acquisition.identity; + const oldRoot = w.env.DOTNET_ROOT; + for (const [key, value] of Object.entries(w.env)) if (value === oldRoot) w.env[key] = w.toolchain.root; + w.env.DOTNET_HOST_PATH = w.toolchain.hostPath; + w.env.PATH = `${w.toolchain.root}${path.delimiter}${path.dirname(w.options.gitExecutable)}${path.delimiter}${w.env.PATH.split(path.delimiter).slice(1).join(path.delimiter)}`; + // Arcade must use the verified installation, never independently select/download a host. + w.env.DOTNET_INSTALL_DIR = w.toolchain.root; + const sdkPaths = declaration.toolchain.sdkSelection.paths ?? []; + for (const sdkPath of sdkPaths.filter(p => p !== "$host$")) { + const target = path.resolve(w.sourceDirectory, sdkPath); + requireValue(inside(w.sourceDirectory, target), "unsupported historical SDK search path"); + await fs.symlink(w.toolchain.root, target, w.options.request.platform.os === "windows" ? "junction" : "dir"); + } + stage = "setup"; + const projects = { fsi: "src/fsi/fsiProject/fsi.fsproj", compiler: "src/fsc/fscProject/fsc.fsproj", + corePackage: "src/FSharp.Core/FSharp.Core.fsproj" }; + const rows = {}; + for (const [role, project] of Object.entries(projects)) { + let row = await evaluated(w, project); + if (role !== "corePackage") { + const frameworks = (row.TargetFrameworks || row.TargetFramework).split(";").filter(t => /^net\d+\.\d+$/.test(t)); + requireValue(frameworks.length === 1 && (!w.targetFramework || w.targetFramework === frameworks[0]), "unsupported product TFMs"); + w.targetFramework = frameworks[0]; + row = await evaluated(w, project, w.targetFramework); + try { await fs.access(row.TargetPath); throw new Error("product exists before clean build"); } + catch (error) { if (error.code !== "ENOENT") throw error; } + } + rows[role] = row; + } + await evidence(w, "evaluated-products.json", rows); + stage = "build"; + const productProject = path.join(w.sourceDirectory, "src", "Microsoft.FSharp.Compiler", "Microsoft.FSharp.Compiler.fsproj"); + if (w.options.request.platform.os === "windows") { + await invoke(w, "build", path.join(w.env.SystemRoot, "System32", "WindowsPowerShell", "v1.0", "powershell.exe"), + ["-NoProfile", "-ExecutionPolicy", "Bypass", "-File", path.join(w.sourceDirectory, "eng", "Build.ps1"), + "-restore", "-build", "-c", "Release", "-pack", "-bootstrap", "-noVisualStudio", "-msbuildEngine", "dotnet", + `/p:Projects=${productProject}`]); + } else { + await invoke(w, "build", "/bin/bash", [path.join(w.sourceDirectory, "build.sh"), + "--configuration", "Release", "--pack", "--bootstrap", "--product-build"]); + } + stage = "identity"; + await sourceUnchanged(w); + for (const [role, project] of Object.entries(projects)) { + const row = await evaluated(w, project, role === "corePackage" ? undefined : w.targetFramework); + requireValue(fingerprintInput(row) === fingerprintInput(rows[role]), "evaluated product identity changed during build"); + } + stage = "setup"; + await helper(w); + stage = "identity"; + w.product = await inventory(w, rows); + await evidence(w, "product.json", w.product); + w.status = "built"; + return { status: "built", product: w.product, toolchain: w.toolchain, + buildInstanceId: w.buildInstanceId, commands: w.commands, evidence: w.evidence }; + } catch (error) { + return { ...failure(w, error.code === "unavailable-runtime" ? "runtime" : stage, error), + buildInstanceId: w?.buildInstanceId, commands: w?.commands, evidence: w?.evidence }; + } +} + +async function finish(w, result) { + if (w.globalJson) { + try { await sourceUnchanged(w); } + catch (error) { + if (!w.signal.aborted) result = { ...failure(w, "identity", error), ...(result.error ? { priorError: result.error } : {}) }; + } + } + const logs = path.join(w.sourceDirectory, "artifacts", "log"); + let names = []; + try { + names = await fs.readdir(logs, { recursive: true }); + } catch (error) { + if (error.code !== "ENOENT") throw error; + } + for (const name of names.filter(n => /\.(binlog|log)$/i.test(n))) { + const file = await fileIdentity(path.join(logs, name), w.sourceDirectory); + await evidence(w, `build-log-${w.evidence.length}${path.extname(name).toLowerCase()}`, await fs.readFile(file.path)); + w.evidence.at(-1).sourcePath = relative(w.root, file.path); + } + const outcome = { + ...result, buildInstanceId: w.buildInstanceId, actualSourceSha: w.actualSourceSha, + requested: w.options.request, adapter: w.adapter, globalJson: w.globalJson, + toolchain: w.toolchain ? { sdkVersion: w.toolchain.sdkVersion, requested: w.declaration, + acquisitionDigest: fingerprintInput(w.toolchain) } : w.declaration, + product: w.product, commands: w.commands, evidence: w.evidence, durationMs: Date.now() - w.started, + }; + if (w.signal.aborted) { + requireValue(owned.has(w) && path.dirname(w.root) === w.parent && + path.basename(w.root) === `revision-${w.buildInstanceId}` && + !(await fs.lstat(w.root)).isSymbolicLink() && await fs.realpath(w.root) === w.root && + JSON.parse(await fs.readFile(path.join(w.root, "owner.json"))).buildInstanceId === w.buildInstanceId, + "refusing cleanup outside owned resolved scratch tree"); + await fs.rm(w.root, { recursive: true }); + outcome.scratchRemoved = true; + } + await evidence(w, "observation.json", outcome); + return outcome; +} + +async function testSourceRevision(options, dependencies = {}) { + const w = await prepareRevisionWorkspace(options, dependencies); + if (w.status !== "prepared") return w; + const built = await buildRevision(w, dependencies); + if (built.status !== "built") return finish(w, built); + try { + const reproduction = await runReproduction({ + manifest: options.manifest, sourceDirectory: w.sampleDirectory, workDirectory: path.join(w.root, "executions"), + product: w.product, toolchain: w.toolchain, helper: w.helper, + }, { + execute: (exe, args, settings) => w.execute(exe, args, { + ...settings, signal: w.signal, timeoutMs: Math.max(1, Math.min(settings.timeoutMs, w.deadline - Date.now())), + }), + }); + w.signal.throwIfAborted(); + await evidence(w, "reproduction.json", reproduction); + return finish(w, { status: "tested", outcome: reproduction.outcome, reproduction }); + } catch (error) { return finish(w, failure(w, "reproduction", error)); } +} + +module.exports = { prepareRevisionWorkspace, buildRevision, testSourceRevision }; diff --git a/eng/regression-bisect/workspace.smoke.cjs b/eng/regression-bisect/workspace.smoke.cjs new file mode 100644 index 00000000000..9e8419bdefb --- /dev/null +++ b/eng/regression-bisect/workspace.smoke.cjs @@ -0,0 +1,35 @@ +const fs = require("node:fs/promises"); +const path = require("node:path"); +const { createFixture } = require("./fixtures/smoke.cjs"); +const { fingerprintInput } = require("./model.cjs"); +const { requireValue } = require("./toolchain-io.cjs"); +const { testSourceRevision } = require("./workspace.cjs"); + +async function main() { + const [optIn, sha, mirror, storage, gitExecutable] = process.argv.slice(2); + requireValue(optIn === "--run" && /^[a-f0-9]{40}$/.test(sha) && mirror && storage && gitExecutable && process.argv.length === 7, + "usage: node workspace.smoke.cjs --run "); + const root = await fs.mkdtemp(path.join(await fs.realpath(storage), "fsharp-workspace-smoke-")); + const os = { win32: "windows", linux: "linux", darwin: "macos" }[process.platform]; + const platform = { os, architecture: process.arch, rid: `${{ windows: "win", linux: "linux", macos: "osx" }[os]}-${process.arch}` }; + const sourceDirectory = path.join(root, "fixture"); + const manifest = await createFixture(sourceDirectory, { kind: "fsi", tfm: "net8.0" }); + const scratchDirectory = path.join(root, "scratch"), evidenceDirectory = path.join(root, "evidence"); + await fs.mkdir(scratchDirectory); + await fs.mkdir(evidenceDirectory); + console.log(`Pinned clean-build smoke: ${sha}\nEvidence: ${evidenceDirectory}`); + const result = await testSourceRevision({ + request: { + schemaVersion: 1, repository: "dotnet/fsharp", sourceSha: sha, phase: "confirmation", + manifestDigest: fingerprintInput(manifest), approvalDigest: manifest.approvedInputDigest, + platform, allocation: { buildTimeoutMs: 90 * 60000, workerTimeoutMs: 110 * 60000 }, + }, + manifest, sourceDirectory, mirrorDirectory: mirror, scratchDirectory, evidenceDirectory, gitExecutable, + }); + await fs.writeFile(path.join(root, "result.json"), JSON.stringify(result, null, 2), { flag: "wx" }); + console.log(JSON.stringify({ outcome: result.outcome, buildInstanceId: result.buildInstanceId, + actualSourceSha: result.actualSourceSha, error: result.error, result: path.join(root, "result.json") }, null, 2)); + process.exitCode = result.outcome.outcome === "pass" ? 0 : 2; +} + +if (require.main === module) main().catch(error => { console.error(error.message); process.exitCode = 2; }); diff --git a/eng/regression-bisect/workspace.test.cjs b/eng/regression-bisect/workspace.test.cjs new file mode 100644 index 00000000000..b3191c1dd28 --- /dev/null +++ b/eng/regression-bisect/workspace.test.cjs @@ -0,0 +1,310 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs/promises"); +const path = require("node:path"); +const { fixture } = require("./fixtures/reproduction.cjs"); +const { seal } = require("./fixtures/contracts.cjs"); +const { fingerprintInput } = require("./model.cjs"); +const { hash, isolatedEnvironment } = require("./toolchain-io.cjs"); +const { execute } = require("./repro-process.cjs"); +const { prepareRevisionWorkspace, buildRevision, testSourceRevision } = require("./workspace.cjs"); + +const git = process.platform === "win32" ? "C:\\Program Files\\Git\\cmd\\git.exe" : "/usr/bin/git"; +const platform = { os: "windows", architecture: "x64", rid: "win-x64" }; +const ok = stdout => ({ status: "completed", complete: true, exitCode: 0, stdout, stderr: "" }); + +async function controlled(t) { + const f = await fixture(t); + const mirror = path.join(f.root, "mirror"); + await fs.mkdir(mirror); + await fs.writeFile(path.join(f.root, "gitconfig"), ""); + const command = async args => { + const result = await execute(git, ["-c", "core.hooksPath=", ...args], { + cwd: mirror, env: { ...isolatedEnvironment(f.root, platform), + GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: path.join(f.root, "gitconfig") }, timeoutMs: 30000, + }); + assert.equal(result.exitCode, 0, result.stderr); + return result.stdout.trim(); + }; + await command(["init"]); + const shas = []; + for (const [sdk, tfm, solution] of [["10.0.100", "net10.0", "FSharp.slnx"], ["9.0.100", "net9.0", "FSharp.sln"]]) { + for (const [name, content] of Object.entries({ + "global.json": JSON.stringify({ sdk: { version: sdk }, tools: { dotnet: sdk } }), + "eng/TargetFrameworks.props": `${tfm}`, + "eng/Build.ps1": `param($restore,$build,$configuration,$pack,$bootstrap,$noVisualStudio,$msbuildEngine,$properties)\n${solution}\n/p:Projects=$projects\n@properties`, + "eng/build.sh": `--bootstrap --pack --product-build\nproto.proj\nConfiguration=Proto\n${solution}`, + "build.sh": '"$scriptroot/eng/build.sh" --restore --build "$@"', + "Build.cmd": "eng\\Build.ps1 -restore -build", + "eng/build-utils.ps1": 'function Make-BootstrapBuild { "proto.proj"; "Proto"; "Bootstrap" }', + "proto.proj": '', + "NuGet.config": "", + "src/fsi/fsiProject/fsi.fsproj": "", + "src/fsc/fscProject/fsc.fsproj": "", + "src/FSharp.Core/FSharp.Core.fsproj": "", + "src/Microsoft.FSharp.Compiler/Microsoft.FSharp.Compiler.fsproj": "", + })) await f.file(path.join("mirror", name), content); + await command(["add", "."]); + await command(["-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "commit", "-qm", sdk]); + shas.push(await command(["rev-parse", "HEAD"])); + } + f.manifest.matrix[0].languageVersion = "preview"; + seal(f.manifest); + const options = { + request: { + schemaVersion: 1, repository: "dotnet/fsharp", sourceSha: shas[0], phase: "main", + manifestDigest: fingerprintInput(f.manifest), approvalDigest: f.manifest.approvedInputDigest, + platform, allocation: { buildTimeoutMs: 60000, workerTimeoutMs: 120000 }, + }, + manifest: f.manifest, sourceDirectory: f.sourceDirectory, mirrorDirectory: mirror, + scratchDirectory: path.join(f.root, "scratch"), evidenceDirectory: path.join(f.root, "evidence"), + gitExecutable: git, + helper: { executable: "helper-host", dll: "helper.dll", hook: "hook.dll", runtimeVersion: "10.0.0" }, + }; + for (const dir of [options.scratchDirectory, options.evidenceDirectory]) await fs.mkdir(dir); + const builds = []; + let activeSdk; + const toolchains = { + http: async url => { + if (url.endsWith("releases-index.json")) return { status: 200, url, bytes: Buffer.from('{"releases-index":[]}') }; + return { status: 200, url, bytes: Buffer.from("a".repeat(128)) }; + }, + installer: async request => { + activeSdk = request.version; + const runtime = `${activeSdk.split(".")[0]}.0.0`; + for (const [name, text] of Object.entries({ + "dotnet.exe": "host", + [`shared/Microsoft.NETCore.App/${runtime}/System.Private.CoreLib.dll`]: "runtime", + [`sdk/${activeSdk}/FSharp/fsi.runtimeconfig.json`]: JSON.stringify({ runtimeOptions: { framework: { name: "Microsoft.NETCore.App", version: runtime } } }), + [`sdk/${activeSdk}/FSharp/fsi.deps.json`]: '{"libraries":{}}', + [`sdk/${activeSdk}/FSharp/fsi.dll`]: "sdk-fsi", + [`sdk/${activeSdk}/FSharp/fsc.dll`]: "sdk-fsc", + [`sdk/${activeSdk}/FSharp/FSharp.Core.dll`]: "sdk-core", + })) { + const file = path.join(request.root, name); + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, text); + } + return { attempts: [], archive: { ...request.archive, redirectUrl: request.archive.url, + actualSha512: request.archive.sha512, sha256: hash("archive") } }; + }, + run: async (exe, args) => { + const root = path.dirname(exe), runtime = `${activeSdk.split(".")[0]}.0.0`; + if (args[0] === "--info") return ok(`Base Path: ${path.join(root, "sdk", activeSdk)}\nRID: win-x64\n`); + if (args[0] === "--version") return ok(activeSdk); + if (args[0] === "--list-sdks") return ok(`${activeSdk} [${path.join(root, "sdk")}]`); + if (args[0] === "--list-runtimes") return ok(`Microsoft.NETCore.App ${runtime} [${path.join(root, "shared", "Microsoft.NETCore.App")}]`); + return ok(`ARCH|x64\nRUNTIME|${runtime}\n` + + ["fsc", "fsi", "FSharp.Core"].map(n => `FILE|${n}.dll|${n}|1.0.0.0|1.0.0`).join("\n")); + }, + }; + const properties = async (cwd, project) => { + const global = JSON.parse(await fs.readFile(path.join(cwd, "global.json"))); + const tfm = `net${global.sdk.version.split(".")[0]}.0`; + const name = path.basename(project, ".fsproj"); + return { + TargetFramework: tfm, TargetFrameworks: tfm, TargetPath: path.join(cwd, "artifacts", "bin", name, "Release", tfm, `${name}.dll`), + PackageId: "FSharp.Core", PackageVersion: "1.0.0", PackageOutputPath: path.join(cwd, "artifacts", "packages", "Release", "Shipping"), + }; + }; + const deps = { toolchains, execute: async (exe, args, settings) => { + if (exe === git) return execute(exe, args, settings); + for (const key of ["GITHUB_TOKEN", "GH_TOKEN", "NUGET_AUTH_TOKEN", "SSH_AUTH_SOCK", "MSBuildSDKsPath"]) { + assert.equal(settings.env[key], undefined); + } + if (args.includes("inspect")) { + const result = []; + for (const file of args.slice(args.indexOf("inspect") + 1)) { + const bytes = await fs.readFile(file); + result.push({ path: file, sha256: hash(bytes), assemblyVersion: "1.0.0.0", fileVersion: "1.0.0", + assemblyName: path.basename(file, ".dll"), + ...(file.endsWith(".nupkg") ? { id: "FSharp.Core", version: "1.0.0", entries: [ + { path: "lib/netstandard2.0/FSharp.Core.dll", sha256: hash(`core-${builds.at(-1).sha}`) }, + ] } : {}) }); + } + return ok(JSON.stringify(result)); + } + if (args[0] === "msbuild") return ok(JSON.stringify({ Properties: await properties(settings.cwd, args[1]) })); + if (args.includes("--exec")) { + const fsi = args[3]; + const core = path.join(path.dirname(fsi), "FSharp.Core.dll"); + const runtime = args[2]; + await fs.writeFile(settings.env.REPRO_CORE_EVIDENCE, JSON.stringify({ + path: core, sha256: hash(await fs.readFile(core)), assemblyVersion: "1.0.0.0", + assemblyName: "FSharp.Core", complete: true, architecture: "x64", runtimeVersion: runtime, + runtimePath: path.join(path.dirname(exe), "shared", "Microsoft.NETCore.App", runtime), + })); + return ok(f.manifest.assertions.pass.stdout); + } + const source = settings.cwd; + const sha = (await execute(git, ["rev-parse", "HEAD"], { ...settings, timeoutMs: 30000 })).stdout.trim(); + builds.push({ sha, source, env: settings.env, args }); + if (f.fail) { + const log = path.join(source, "artifacts", "log", "Release", "bootstrap.binlog"); + await fs.mkdir(path.dirname(log), { recursive: true }); + await fs.writeFile(log, "failed bootstrap"); + return { ...ok(""), exitCode: 1, stderr: f.manifest.assertions.failure.stderr }; + } + if (f.timeout) return { ...ok(""), status: "timeout", complete: false, exitCode: null }; + if (f.cancel) { f.cancel.abort(); return { ...ok(""), status: "cancelled", exitCode: null }; } + for (const name of ["fsi", "fsc"]) { + const p = await properties(source, `${name}.fsproj`); + const dir = path.dirname(p.TargetPath); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(p.TargetPath, f.stale === name ? `sdk-${name}` : `${name}-${sha}`); + await fs.writeFile(path.join(dir, `${name}.runtimeconfig.json`), JSON.stringify({ + runtimeOptions: { tfm: p.TargetFramework, framework: { name: "Microsoft.NETCore.App", version: `${activeSdk.split(".")[0]}.0.0` } }, + })); + for (const item of ["FSharp.Core.dll", "FSharp.Compiler.Service.dll", "FSharp.Build.dll", "Microsoft.FSharp.NetSdk.props", + "Microsoft.FSharp.Targets", "Microsoft.FSharp.NetSdk.targets", "Microsoft.FSharp.Overrides.NetSdk.targets"]) { + await fs.writeFile(path.join(dir, item), item === "FSharp.Core.dll" ? `core-${sha}` : item); + } + } + const p = await properties(source, "FSharp.Core.fsproj"); + await fs.mkdir(p.PackageOutputPath, { recursive: true }); + if (!f.missingPackage) await fs.writeFile(path.join(p.PackageOutputPath, "FSharp.Core.1.0.0.nupkg"), `package-${sha}`); + await f.afterBuild?.(source); + return ok(""); + } }; + return Object.assign(f, { options, deps, shas, builds, command }); +} + +test("fresh revisions use their exact SHA, SDK, TFM and independent same-version Core bytes", async t => { + const f = await controlled(t); + await f.file("mirror/artifacts/Bootstrap/fsi/fsi.dll", "poison"); + await f.file("mirror/artifacts/bin/fsi/Proto/fsi.dll", "poison"); + await f.file("mirror/bin/fsi.dll", "poison"); + await f.file("mirror/obj/project.assets.json", "poison"); + const hookMarker = path.join(f.root, "hook-ran"); + await f.file("mirror/.git/hooks/post-checkout", `#!/bin/sh\ntouch "${hookMarker.replace(/\\/g, "/")}"\n`); + await f.file("nuget/packages/fsharp.core/1.0.0/FSharp.Core.dll", "host"); + const results = []; + for (const [i, sha] of f.shas.entries()) { + const result = await testSourceRevision({ ...f.options, request: { ...f.options.request, sourceSha: sha } }, f.deps); + assert.equal(result.outcome.outcome, "pass", JSON.stringify(result)); + assert.equal(result.actualSourceSha, sha); + assert.equal(result.toolchain.sdkVersion, i ? "9.0.100" : "10.0.100"); + assert.equal(result.product.targetFramework, i ? "net9.0" : "net10.0"); + assert.equal(result.adapter.layout, i ? "arcade-sln-products" : "arcade-slnx-products"); + assert.equal(result.globalJson.before, result.globalJson.after); + assert.equal(result.reproduction.cells[0].attempts.length, 2); + assert.equal(result.product.files.filter(x => x.path.endsWith("FSharp.Compiler.Service.dll")).length, 2); + results.push(result); + } + assert.notEqual(results[0].product.files.find(x => x.role === "corePackage").sha256, + results[1].product.files.find(x => x.role === "corePackage").sha256); + assert.notEqual(f.builds[0].env.NUGET_PACKAGES, f.builds[1].env.NUGET_PACKAGES); + assert.equal(await fs.readFile(path.join(f.root, "nuget/packages/fsharp.core/1.0.0/FSharp.Core.dll"), "utf8"), "host"); + for (const b of f.builds) await assert.rejects(fs.access(path.join(b.source, "bin", "fsi.dll"))); + await assert.rejects(fs.access(hookMarker)); +}); + +for (const [name, change, reason] of [ + ["failed source build", f => { f.fail = true; }, "build-failure"], + ["timed out source build", f => { f.timeout = true; }, "timeout"], + ["stale fsi", f => { f.stale = "fsi"; }, "wrong-identity"], + ["stale fsc", f => { f.stale = "fsc"; }, "wrong-identity"], + ["missing package", f => { f.missingPackage = true; }, "wrong-identity"], + ["missing FSI", f => { f.afterBuild = source => fs.rm(path.join(source, "artifacts/bin/fsi/Release/net10.0/fsi.dll")); }, "wrong-identity"], + ["changed global.json", f => { f.afterBuild = source => fs.appendFile(path.join(source, "global.json"), " "); }, "wrong-identity"], + ["unavailable SDK", f => { f.deps.toolchains.http = async () => { throw new Error("SDK unavailable"); }; }, "unavailable-sdk"], +]) test(`${name} is untestable, never the reported product failure`, async t => { + const f = await controlled(t); + change(f); + const result = await testSourceRevision(f.options, f.deps); + assert.equal(result.outcome.outcome, "untestable"); + assert.equal(result.outcome.reason, reason, JSON.stringify(result)); + assert.ok(result.error); + if (f.fail) assert.ok(result.evidence.some(e => e.path.endsWith(".binlog"))); + assert.ok(result.evidence.every(e => !path.isAbsolute(e.path) && !e.path.split("/").includes("..") && /^[a-f0-9]{64}$/.test(e.sha256))); +}); + +test("confirmation and all phases always allocate independent build instances", async t => { + const f = await controlled(t); + const ids = new Set(); + for (const phase of ["main", "older-search", "bisect", "confirmation", "confirmation"]) { + const result = await testSourceRevision({ ...f.options, request: { ...f.options.request, phase } }, f.deps); + assert.equal(result.outcome.outcome, "pass", JSON.stringify(result)); + ids.add(result.buildInstanceId); + } + assert.equal(ids.size, 5); +}); + +test("cancellation removes only its scratch tree and retains failure evidence", async t => { + const f = await controlled(t); + f.cancel = new AbortController(); + const sibling = await prepareRevisionWorkspace(f.options, f.deps); + assert.equal(sibling.status, "prepared"); + const result = await testSourceRevision({ ...f.options, signal: f.cancel.signal }, f.deps); + assert.equal(result.outcome.outcome, "untestable"); + assert.equal(result.error.code, "cancelled"); + await fs.access(sibling.root); + assert.deepEqual(await fs.readdir(f.options.scratchDirectory), [path.basename(sibling.root)]); + await fs.access(f.options.mirrorDirectory); +}); + +test("concurrent workspace preparation cannot collide and prepared poison is rejected", async t => { + const f = await controlled(t); + const workspaces = await Promise.all([prepareRevisionWorkspace(f.options, f.deps), prepareRevisionWorkspace(f.options, f.deps)]); + assert.ok(workspaces.every(w => w.status === "prepared")); + assert.notEqual(workspaces[0].root, workspaces[1].root); + for (const name of ["artifacts/Bootstrap/fsc/fsc.dll", "artifacts/bin/fsi/Proto/fsi.dll", "bin/fsi.dll", "obj/project.assets.json"]) { + const location = path.join(workspaces[0].sourceDirectory, name); + await fs.mkdir(path.dirname(location), { recursive: true }); + await fs.writeFile(location, "poison"); + } + const result = await buildRevision(workspaces[0], f.deps); + assert.equal(result.outcome.outcome, "untestable"); + assert.match(result.error.message, /unexpected state/); + assert.equal(f.builds.length, 0); + const built = await buildRevision(workspaces[1], f.deps); + assert.equal(built.status, "built", JSON.stringify(built)); + assert.equal((await buildRevision(workspaces[1], f.deps)).outcome.outcome, "untestable"); +}); + +test("process cancellation is bounded and does not run an already cancelled command", async t => { + const f = await controlled(t); + const settings = { cwd: f.root, env: isolatedEnvironment(f.root, platform), timeoutMs: 30000 }; + const cancelled = AbortSignal.abort(); + assert.equal((await execute(process.execPath, ["-e", "throw Error('must not run')"], + { ...settings, signal: cancelled })).status, "cancelled"); + const controller = new AbortController(); + const running = execute(process.execPath, ["-e", "setInterval(()=>{}, 1000)"], + { ...settings, signal: controller.signal }); + setTimeout(() => controller.abort(), 100); + const result = await running; + assert.equal(result.status, "cancelled"); + assert.equal(result.complete, false); +}); + +test("unsupported layout and invalid digest fail closed before a build", async t => { + const f = await controlled(t); + const invalid = await testSourceRevision({ ...f.options, request: { ...f.options.request, manifestDigest: "a".repeat(64) } }, f.deps); + assert.equal(invalid.outcome.outcome, "untestable"); + await fs.writeFile(path.join(f.options.mirrorDirectory, "eng", "Build.ps1"), "unsupported"); + await f.command(["add", "."]); + await f.command(["-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "commit", "-qm", "unsupported layout"]); + const sourceSha = await f.command(["rev-parse", "HEAD"]); + const result = await testSourceRevision({ ...f.options, request: { ...f.options.request, sourceSha } }, f.deps); + assert.equal(result.outcome.outcome, "untestable"); + assert.equal(result.error.stage, "layout"); + assert.match(result.error.message, /unsupported historical build entrypoint/); + assert.equal(f.builds.length, 0); +}); + +test("evaluated output wins over a newer decoy and host credentials never reach scripts", async t => { + const f = await controlled(t); + const old = process.env.GITHUB_TOKEN; + process.env.GITHUB_TOKEN = "must-not-leak"; + t.after(() => { if (old === undefined) delete process.env.GITHUB_TOKEN; else process.env.GITHUB_TOKEN = old; }); + f.afterBuild = async source => { + const decoy = path.join(source, "artifacts", "bin", "fsi", "Release", "net99.0", "fsi.dll"); + await fs.mkdir(path.dirname(decoy), { recursive: true }); + await fs.writeFile(decoy, "wrong"); + await fs.utimes(decoy, new Date("2099-01-01"), new Date("2099-01-01")); + }; + const result = await testSourceRevision(f.options, f.deps); + assert.equal(result.outcome.outcome, "pass", JSON.stringify(result)); + assert.ok(!JSON.stringify(result).includes("must-not-leak")); + assert.ok(result.product.files.find(x => x.role === "fsi").path.includes("net10.0")); +}); From 5a8b9adb034f420a5c2c1f577670c5eeeb454b7e Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 07:02:34 +0200 Subject: [PATCH 17/99] Fix historical workspace packaging and cancellation Repack built Core into its evaluated standalone output, preserve package identity checks, and invalidate cancelled observations including staged builds. Contain Windows process trees with a job supervisor while allowing detached build servers to exit cleanly. Reuse private NuGet machine paths across worker processes. Validation: 519 scoped tests pass, command/package mutations are rejected, and syntax/format/whitespace checks pass. Pinned clean-build smoke remains untestable: exact SDK checksum feeds return 404; evidence is recorded in the README. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 31 ++++- eng/regression-bisect/repro-process.cjs | 61 ++++++--- eng/regression-bisect/repro-process.ps1 | 69 ++++++++++ eng/regression-bisect/repro.test.cjs | 157 ++++++++++++++++++++++ eng/regression-bisect/toolchain-io.cjs | 6 +- eng/regression-bisect/toolchains.test.cjs | 4 +- eng/regression-bisect/workspace.cjs | 53 +++++--- eng/regression-bisect/workspace.test.cjs | 116 ++++++++++++++-- 8 files changed, 445 insertions(+), 52 deletions(-) create mode 100644 eng/regression-bisect/repro-process.ps1 diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index c5885dfeb44..498d6da99ea 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -378,6 +378,10 @@ entrypoints, declarations or SDK property-query support are explicitly untestabl Historical files, TFMs and package versions are never rewritten. MSBuild-evaluated paths and package identity must agree before/after the build. +After the product graph finishes, the adapter explicitly packs its built Core +with `--no-build --no-restore` into Core's evaluated `PackageOutputPath`. +The graph can place dependency packages under `Dependency/Shipping`; that is not +the standalone Core output, and the adapter does not guess between the two. Inventory verifies bytes, assembly versions, runtimeconfigs and the Core package, rejecting missing outputs, SDK-bundled compiler fallback and redirected artifacts. No timestamp selection or shared same-version Core cache is used. Exact acquisition @@ -398,6 +402,15 @@ their output matches the expected diagnostic. Cancellation terminates process trees and removes only the UUID-marked, resolved owned scratch child; ordinary results retain their scratch artifacts for diagnosis. Results expose no mutation or publisher operation. Controller provenance/admission remains separate. +Cancellation also finalizes staged `buildRevision` calls. An interruption during +final source verification or observation persistence invalidates a completed +reproduction, and repeated finalization reuses the same observation. +On Windows, a trusted PowerShell supervisor joins a kill-on-close Job Object +before launching the command. It drains inherited output even after the parent +exits; timeout/cancellation kills the contained descendants. Successful commands +do not wait for detached build servers that have closed their output pipes. +The shared environment supplies private Windows machine-configuration paths so +NuGet can initialize without reading the developer's machine feed configuration. ### Bounded opt-in clean-build smoke @@ -411,11 +424,17 @@ builds the product graph cleanly and requests two verified local executions. It prints its retained result/evidence location and exits 2 on an untestable result. No live issue discovery or full compiler test suite is involved. -Local verification on 2026-09-19: RED was observed before `workspace.cjs` existed; -the final scoped suite passed **498 tests**, including 15 workspace contracts. +Local verification on 2026-09-19: corrective RED cases reproduced the graph's +`Dependency/Shipping` output mismatch and three staged/finalization cancellation +failures. Real subprocess cases reproduced exited-parent hangs and detached-server +timeouts. The final scoped suite passed **519 tests**, including 28 workspace +contracts. In-memory mutations to use Debug and bypass package validation failed +the command-contract case and all four package-identity cases, respectively. +Changed-script syntax (including the PowerShell helper), formatting, and working/ +sprint-range whitespace checks passed. The real Windows x64 smoke pinned -`e6ee82e2f1e730ea2ac64a05925dd180a7dfe2df`, build instance -`38b31a09-c802-495f-ad3a-2e4b006515ab`. Its accurate result is +`bf13cf234bc47e62166bc82748d6784d0aee086f`, build instance +`20f99ee4-f35e-48c7-8620-b2ecce174da8`. Its accurate result is **untestable / unavailable-sdk**, exit 2: exact SDK `11.0.100-rc.1.26420.103` has no verified public archive available to acquisition. Release metadata returned 200, but @@ -425,8 +444,8 @@ returned 404 under both `https://builds.dotnet.microsoft.com/dotnet` and `global.json` retained its hash. Acquisition stopped before the build entrypoint, so no build binlog or local FSI result exists; this is not a green clean build or compiler suite. Retained evidence is -`Q:\fsharp-workspace-smoke-r1n9dF\result.json`, SHA-256 -`4a3bf0528370c54471aa8e81e68f62e4db958a441fab69819e2b540a6c2f7bf5`. +`Q:\fsharp-workspace-smoke-D70Agl\result.json`, SHA-256 +`28f8a3fe1061f006e8786ef3d426e8e864220f662dd5fc8cb226ef6ff284e302`. ## Reproduction execution diff --git a/eng/regression-bisect/repro-process.cjs b/eng/regression-bisect/repro-process.cjs index d9a114dc30b..72902274407 100644 --- a/eng/regression-bisect/repro-process.cjs +++ b/eng/regression-bisect/repro-process.cjs @@ -1,6 +1,7 @@ const path = require("node:path"); const { spawn } = require("node:child_process"); const { constants } = require("node:os"); +const { randomUUID } = require("node:crypto"); const policy = require("../../.github/policies/regression-bisect.json"); function execute(executable, args, { cwd, env, timeoutMs, signal: cancellation, maxBytes = policy.evidenceBytesPerStream }) { @@ -9,40 +10,69 @@ function execute(executable, args, { cwd, env, timeoutMs, signal: cancellation, resolve({ status: "cancelled", complete: false, exitCode: null, stdout: "", stderr: "" }); return; } - const child = spawn(executable, args, { cwd, env, shell: false, windowsHide: true, detached: process.platform !== "win32" }); + if (!env) { + resolve({ status: "exception", complete: false, exitCode: null, stdout: "", stderr: "", error: "explicit process environment is required" }); + return; + } + const windows = process.platform === "win32"; + const errorMarker = `REPRO_PROCESS_${randomUUID()}:`; + const child = windows + ? spawn(path.join(env.SystemRoot, "System32", "WindowsPowerShell", "v1.0", "powershell.exe"), + ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", path.join(__dirname, "repro-process.ps1"), errorMarker], + { cwd, env, shell: false, windowsHide: true }) + : spawn(executable, args, { cwd, env, shell: false, detached: true }); const streams = { stdout: [], stderr: [] }; const lengths = { stdout: 0, stderr: 0 }; - let timedOut = false, cancelled = false, terminating = false, truncated = false, launchError = null, killing = Promise.resolve(); + let timedOut = false, cancelled = false, terminating = false, truncated = false, launchError = null; + let terminationTimer, finished = false, errorTail = ""; + const helperError = new RegExp(`${errorMarker}([A-Za-z0-9+/=]+)\\r?\\n`); for (const name of Object.keys(streams)) child[name].on("data", bytes => { + if (windows && name === "stderr") { + errorTail = (errorTail + bytes.toString("ascii")).slice(-8192); + const error = helperError.exec(errorTail); + if (error) launchError = Buffer.from(error[1], "base64").toString("utf8"); + } const remaining = Math.max(0, maxBytes - lengths[name]); if (bytes.length > remaining) truncated = true; if (remaining > 0) streams[name].push(bytes.subarray(0, remaining)); lengths[name] += bytes.length; }); child.on("error", error => { launchError = error.message; }); + if (windows) { + child.stdin.on("error", error => { if (!terminating) launchError = error.message; }); + child.stdin.end(JSON.stringify({ + executable, args, environment: Object.entries(env).filter(([, value]) => value !== undefined).map(([key, value]) => `${key}=${value}`), + }) + "\n"); + } const terminate = () => { if (!child.pid || terminating) return; terminating = true; - if (process.platform === "win32") { - killing = new Promise(done => { - const killer = spawn(path.join(env.SystemRoot, "System32", "taskkill.exe"), ["/PID", String(child.pid), "/T", "/F"], - { windowsHide: true, stdio: "ignore", env }); - killer.on("error", error => { launchError = `process-tree termination failed: ${error.message}`; child.kill(); done(); }); - killer.on("close", code => { if (code !== 0) launchError = `process-tree termination failed (${code})`; done(); }); - }); - } else { - try { process.kill(-child.pid, "SIGKILL"); } catch (error) { launchError = `process-tree termination failed: ${error.message}`; } + try { + if (windows) { + if (!child.kill("SIGKILL")) throw new Error("supervisor could not be terminated"); + } else process.kill(-child.pid, "SIGKILL"); + } catch (error) { + launchError = `process-tree termination failed: ${error.message}`; } + terminationTimer = setTimeout(() => { + launchError ??= "process-tree termination did not close its streams within 5 seconds"; + child.stdout.destroy(); + child.stderr.destroy(); + child.stdin.destroy(); + finish(null, null); + }, 5000); }; const abort = () => { cancelled = true; terminate(); }; cancellation?.addEventListener("abort", abort, { once: true }); const timer = setTimeout(() => { timedOut = true; terminate(); }, timeoutMs); - child.on("close", async (code, signal) => { + const finish = (code, signal) => { + if (finished) return; + finished = true; clearTimeout(timer); + clearTimeout(terminationTimer); cancellation?.removeEventListener("abort", abort); - await killing; const stdout = Buffer.concat(streams.stdout).toString("utf8").replace(/\r\n/g, "\n"); - const stderr = Buffer.concat(streams.stderr).toString("utf8").replace(/\r\n/g, "\n"); + const stderr = Buffer.concat(streams.stderr).toString("utf8").replace(helperError, "").replace(/\r\n/g, "\n"); const managedException = signal === "SIGABRT" && /^Unhandled exception\./.test(stderr); resolve({ status: cancelled ? "cancelled" : timedOut ? "timeout" : launchError || signal && !managedException ? "exception" : "completed", @@ -50,7 +80,8 @@ function execute(executable, args, { cwd, env, timeoutMs, signal: cancellation, stdout, stderr, ...(launchError ? { error: launchError } : {}), }); - }); + }; + child.on("close", finish); }); } diff --git a/eng/regression-bisect/repro-process.ps1 b/eng/regression-bisect/repro-process.ps1 new file mode 100644 index 00000000000..320ff652bda --- /dev/null +++ b/eng/regression-bisect/repro-process.ps1 @@ -0,0 +1,69 @@ +param([string] $ErrorMarker) +$ErrorActionPreference = 'Stop' +try { + [Console]::InputEncoding = New-Object System.Text.UTF8Encoding($false) + $request = [Console]::ReadLine() | ConvertFrom-Json + + # Emit only P/Invoke declarations: Add-Type would spawn an uncontained compiler before the job exists. + $assembly = [AppDomain]::CurrentDomain.DefineDynamicAssembly( + (New-Object Reflection.AssemblyName('ReproJob')), [Reflection.Emit.AssemblyBuilderAccess]::Run) + $native = $assembly.DefineDynamicModule('ReproJob').DefineType('ReproJob', 'Public, Abstract, Sealed') + foreach ($signature in @( + @('CreateJobObjectW', [IntPtr], @([IntPtr], [string])), + @('SetInformationJobObject', [bool], @([IntPtr], [int], [IntPtr], [int])), + @('AssignProcessToJobObject', [bool], @([IntPtr], [IntPtr])), + @('TerminateProcess', [bool], @([IntPtr], [int])) + )) { + $method = $native.DefinePInvokeMethod($signature[0], 'kernel32.dll', 'Public, Static, PinvokeImpl', + [Reflection.CallingConventions]::Standard, $signature[1], [Type[]] $signature[2], + [Runtime.InteropServices.CallingConvention]::Winapi, [Runtime.InteropServices.CharSet]::Unicode) + $method.SetImplementationFlags([Reflection.MethodImplAttributes]::PreserveSig) + } + $native.CreateType() | Out-Null + $job = [ReproJob]::CreateJobObjectW([IntPtr]::Zero, $null) + if ($job -eq [IntPtr]::Zero) { throw 'CreateJobObject failed' } + + # JOBOBJECT_EXTENDED_LIMIT_INFORMATION: LimitFlags at byte 16; size is 144 (64-bit) or 112 (32-bit). + $size = if ([IntPtr]::Size -eq 8) { 144 } else { 112 } + $buffer = [Runtime.InteropServices.Marshal]::AllocHGlobal($size) + try { + [Runtime.InteropServices.Marshal]::Copy((New-Object byte[] $size), 0, $buffer, $size) + [Runtime.InteropServices.Marshal]::WriteInt32($buffer, 16, 0x2000) # KILL_ON_JOB_CLOSE + if (-not [ReproJob]::SetInformationJobObject($job, 9, $buffer, $size)) { throw 'SetInformationJobObject failed' } + $self = [Diagnostics.Process]::GetCurrentProcess() + try { + if (-not [ReproJob]::AssignProcessToJobObject($job, $self.Handle)) { throw 'AssignProcessToJobObject failed' } + } finally { $self.Dispose() } + + # The supervisor joins before spawning. Its non-inheritable job handle kills all descendants on exit. + $arguments = foreach ($argument in $request.args) { + '"' + [regex]::Replace([regex]::Replace($argument, '(\\*)"', '$1$1\"'), '(\\+)$', '$1$1') + '"' + } + $start = New-Object Diagnostics.ProcessStartInfo($request.executable, ($arguments -join ' ')) + $start.UseShellExecute = $false + $start.WindowStyle = [Diagnostics.ProcessWindowStyle]::Hidden + $start.RedirectStandardOutput = $true + $start.RedirectStandardError = $true + $start.EnvironmentVariables.Clear() + foreach ($entry in $request.environment) { + $separator = $entry.IndexOf('=') + $start.EnvironmentVariables.Add($entry.Substring(0, $separator), $entry.Substring($separator + 1)) + } + $child = [Diagnostics.Process]::Start($start) + try { + $stdout = $child.StandardOutput.BaseStream.CopyToAsync([Console]::OpenStandardOutput()) + $stderr = $child.StandardError.BaseStream.CopyToAsync([Console]::OpenStandardError()) + $child.WaitForExit() + [Threading.Tasks.Task]::WaitAll([Threading.Tasks.Task[]] @($stdout, $stderr)) + # Native exit avoids CLR shutdown waits and kills any servers that have closed their output pipes. + $self = [Diagnostics.Process]::GetCurrentProcess() + if (-not [ReproJob]::TerminateProcess($self.Handle, $child.ExitCode)) { throw 'TerminateProcess failed' } + } finally { $child.Dispose() } + } finally { [Runtime.InteropServices.Marshal]::FreeHGlobal($buffer) } +} +catch { + $message = $_.Exception.GetBaseException().Message + $bytes = [Text.Encoding]::UTF8.GetBytes($message.Substring(0, [Math]::Min(1024, $message.Length))) + [Console]::Error.WriteLine($ErrorMarker + [Convert]::ToBase64String($bytes)) + [Environment]::Exit(1) +} diff --git a/eng/regression-bisect/repro.test.cjs b/eng/regression-bisect/repro.test.cjs index 64fb1bf878b..e80e7c57649 100644 --- a/eng/regression-bisect/repro.test.cjs +++ b/eng/regression-bisect/repro.test.cjs @@ -486,3 +486,160 @@ for (const [tamper, compressed] of [[false, false], [true, false], [false, true] assert.equal(hash(await fs.readFile(pkg.path)), pkg.sha256); } }); + +for (const mode of ["timeout", "cancelled"]) test(`process ${mode} contains descendants after their parent exits`, async t => { + const f = await fixture(t); + const ready = path.join(f.root, "descendant.json"); + const controller = new AbortController(); + const descendant = ` + const fs = require('node:fs'); + fs.writeFileSync(${JSON.stringify(ready + ".new")}, JSON.stringify({ pid: process.pid, parent: process.ppid })); + fs.renameSync(${JSON.stringify(ready + ".new")}, ${JSON.stringify(ready)}); + process.stdout.write('descendant stdout\\n'); + process.stderr.write('descendant stderr\\n'); + process.send('ready'); + process.disconnect(); + setInterval(() => {}, 1000); + `; + const execution = execute(process.execPath, ["-e", ` + const child = require('node:child_process').spawn(process.execPath, ['-e', ${JSON.stringify(descendant)}], + { stdio: ['ignore', 'inherit', 'inherit', 'ipc'], detached: process.platform === 'win32' }); + child.once('message', () => process.exit(0)); + `], { + cwd: f.root, + env: isolatedEnvironment(f.root, { os: process.platform === "win32" ? "windows" : "linux", architecture: process.arch }), + timeoutMs: mode === "timeout" ? 3000 : 30000, signal: controller.signal, + }); + let pid; + let deadline; + try { + const limit = Date.now() + 10000; + let parent; + while (Date.now() < limit) { + try { + ({ pid, parent } = JSON.parse(await fs.readFile(ready, "utf8"))); + try { process.kill(parent, 0); } catch (error) { if (error.code === "ESRCH") break; throw error; } + } catch (error) { if (error.code !== "ENOENT") throw error; } + await new Promise(resolve => setTimeout(resolve, 20)); + } + assert.ok(pid > 0, "descendant must start"); + assert.throws(() => process.kill(parent, 0), { code: "ESRCH" }, "parent exits before termination"); + process.kill(pid, 0); + if (mode === "cancelled") controller.abort(); + const result = await Promise.race([ + execution, + new Promise((_, reject) => { deadline = setTimeout(() => reject(new Error("execution did not terminate within 6 seconds")), 6000); }), + ]); + assert.equal(result.status, mode, JSON.stringify(result)); + assert.equal(result.complete, false); + assert.equal(result.error, undefined); + assert.equal(result.stdout, "descendant stdout\n"); + assert.equal(result.stderr, "descendant stderr\n"); + assert.throws(() => process.kill(pid, 0), { code: "ESRCH" }, "descendant must not survive"); + } finally { + clearTimeout(deadline); + controller.abort(); + if (pid) { + try { process.kill(pid, "SIGKILL"); } catch (error) { if (error.code !== "ESRCH") throw error; } + } + await execution; + } +}); + +test("process already cancelled does not launch", async t => { + const f = await fixture(t); + const marker = path.join(f.root, "launched"); + const result = await execute(process.execPath, ["-e", `require('node:fs').writeFileSync(${JSON.stringify(marker)}, '')`], { + cwd: f.root, + env: isolatedEnvironment(f.root, { os: process.platform === "win32" ? "windows" : "linux", architecture: process.arch }), + timeoutMs: 3000, signal: AbortSignal.abort(), + }); + assert.deepEqual(result, { status: "cancelled", complete: false, exitCode: null, stdout: "", stderr: "" }); + await assert.rejects(fs.stat(marker), { code: "ENOENT" }); +}); + +test("process preserves arguments, explicit environment, output and exit code", async t => { + const f = await fixture(t); + const args = ["", "with spaces", 'a\\"b', 'trailing\\', "\u96ea", "line\nbreak", "& | < > %PATH%"]; + const env = isolatedEnvironment(f.root, { os: process.platform === "win32" ? "windows" : "linux", architecture: process.arch }); + const previous = process.env.REPRO_PROCESS_SECRET; + process.env.REPRO_PROCESS_SECRET = "must not inherit"; + try { + const result = await execute(process.execPath, ["-e", ` + process.stdout.write(JSON.stringify({ args: process.argv.slice(1), + secret: process.env.REPRO_PROCESS_SECRET, allowed: process.env.REPRO_ALLOWED_ENV })); + process.stderr.write('stderr\\r\\n'); + process.exitCode = 17; + `, ...args], { cwd: f.root, env: { ...env, REPRO_ALLOWED_ENV: "allowed" }, timeoutMs: 30000 }); + assert.equal(result.status, "completed", result.error); + assert.equal(result.complete, true); + assert.equal(result.exitCode, 17); + assert.deepEqual(JSON.parse(result.stdout), { args, allowed: "allowed" }); + assert.equal(result.stderr, "stderr\n"); + } finally { + if (previous === undefined) delete process.env.REPRO_PROCESS_SECRET; + else process.env.REPRO_PROCESS_SECRET = previous; + } +}); + +for (const missing of ["executable", "environment"]) test(`process missing ${missing} remains an exception`, async t => { + const f = await fixture(t); + const result = await execute(path.join(f.root, "does-not-exist.exe"), [], { + cwd: f.root, + env: missing === "environment" ? undefined + : isolatedEnvironment(f.root, { os: process.platform === "win32" ? "windows" : "linux", architecture: process.arch }), + timeoutMs: 30000, maxBytes: 0, + }); + assert.equal(result.status, "exception", JSON.stringify(result)); + assert.equal(result.complete, false); + assert.ok(result.error); +}); + +test("process cancellation during launch is bounded and never launches the target", { skip: process.platform !== "win32" }, async t => { + const f = await fixture(t); + const marker = path.join(f.root, "launched"); + const controller = new AbortController(); + const execution = execute(process.execPath, ["-e", `require('node:fs').writeFileSync(${JSON.stringify(marker)}, '')`], { + cwd: f.root, + env: isolatedEnvironment(f.root, { os: "windows", architecture: process.arch }), + timeoutMs: 30000, signal: controller.signal, + }); + controller.abort(); + const result = await execution; + assert.equal(result.status, "cancelled"); + assert.equal(result.complete, false); + assert.equal(result.error, undefined); + await assert.rejects(fs.stat(marker), { code: "ENOENT" }); +}); + +test("process completes and removes detached servers that close their output pipes", { skip: process.platform !== "win32" }, async t => { + const f = await fixture(t); + const ready = path.join(f.root, "server.pid"); + const server = ` + require('node:fs').writeFileSync(${JSON.stringify(ready)}, String(process.pid)); + process.send('ready'); + process.disconnect(); + setInterval(() => {}, 1000); + `; + try { + const result = await execute(process.execPath, ["-e", ` + const server = require('node:child_process').spawn(process.execPath, ['-e', ${JSON.stringify(server)}], + { detached: true, stdio: ['ignore', 'ignore', 'ignore', 'ipc'] }); + server.once('message', () => { process.stdout.write('build complete\\n'); process.exit(17); }); + `], { + cwd: f.root, env: isolatedEnvironment(f.root, { os: "windows", architecture: process.arch }), timeoutMs: 3000, + }); + const pid = Number(await fs.readFile(ready, "utf8")); + assert.equal(result.status, "completed", JSON.stringify(result)); + assert.equal(result.complete, true); + assert.equal(result.exitCode, 17); + assert.equal(result.stdout, "build complete\n"); + assert.equal(result.stderr, ""); + assert.throws(() => process.kill(pid, 0), { code: "ESRCH" }, "detached build server must not survive"); + } finally { + try { + const pid = Number(await fs.readFile(ready, "utf8")); + if (pid > 0) process.kill(pid, "SIGKILL"); + } catch (error) { if (!["ENOENT", "ESRCH"].includes(error.code)) throw error; } + } +}); diff --git a/eng/regression-bisect/toolchain-io.cjs b/eng/regression-bisect/toolchain-io.cjs index e23e2a868f8..58251426d62 100644 --- a/eng/regression-bisect/toolchain-io.cjs +++ b/eng/regression-bisect/toolchain-io.cjs @@ -80,7 +80,11 @@ function isolatedEnvironment(root, platform) { ? [path.join(systemRoot, "System32"), systemRoot, path.join(systemRoot, "System32", "WindowsPowerShell", "v1.0")].join(path.delimiter) : "/usr/bin:/bin:/usr/sbin:/sbin"; return { - ...(platform.os === "windows" ? { SystemRoot: systemRoot, WINDIR: systemRoot, ComSpec: path.join(systemRoot, "System32", "cmd.exe") } : {}), + ...(platform.os === "windows" ? { + SystemRoot: systemRoot, WINDIR: systemRoot, ComSpec: path.join(systemRoot, "System32", "cmd.exe"), + ProgramFiles: path.join(root, "program-files"), "ProgramFiles(x86)": path.join(root, "program-files-x86"), + ProgramData: path.join(root, "program-data"), ALLUSERSPROFILE: path.join(root, "program-data"), + } : {}), PATH: `${dotnet}${path.delimiter}${systemPath}`, HOME: home, USERPROFILE: home, APPDATA: path.join(home, "AppData", "Roaming"), LOCALAPPDATA: path.join(home, "AppData", "Local"), TMP: path.join(root, "tmp"), TEMP: path.join(root, "tmp"), TMPDIR: path.join(root, "tmp"), diff --git a/eng/regression-bisect/toolchains.test.cjs b/eng/regression-bisect/toolchains.test.cjs index 4d8f59b558d..36815b59899 100644 --- a/eng/regression-bisect/toolchains.test.cjs +++ b/eng/regression-bisect/toolchains.test.cjs @@ -300,8 +300,10 @@ test("two acquisitions with the same versions never share CLI or NuGet state", a for (let i = 0; i < 2; i++) { assert.equal((await acquireToolchain({ policy: policy(), toolchain, storageDirectory: root }, deps)).status, "acquired"); } - for (const key of ["DOTNET_ROOT", "NUGET_PACKAGES", "DOTNET_CLI_HOME", "HOME"]) { + for (const key of ["DOTNET_ROOT", "NUGET_PACKAGES", "DOTNET_CLI_HOME", "HOME", + "ProgramFiles", "ProgramFiles(x86)", "ProgramData", "ALLUSERSPROFILE"]) { assert.notEqual(environments[0][key], environments[1][key]); + for (const env of environments) assert.ok(env[key].startsWith(root + path.sep), `${key} must remain private`); } }); diff --git a/eng/regression-bisect/workspace.cjs b/eng/regression-bisect/workspace.cjs index 7836d9b974f..c2eb35d5060 100644 --- a/eng/regression-bisect/workspace.cjs +++ b/eng/regression-bisect/workspace.cjs @@ -70,7 +70,7 @@ async function invoke(w, stage, executable, args, settings = {}, boundary = w.ex await evidence(w, `command-${w.commands.length}.json`, { ...entry, result }); if (result.exitCode !== 0 || result.status === "timeout" || result.status === "cancelled" || result.complete === false) { throw Object.assign(new Error( - `${stage} command failed (${entry.status}, exit ${result.exitCode}): ${executable} ${args.join(" ")}\n${result.stderr}`), + `${stage} command failed (${entry.status}, exit ${result.exitCode}): ${executable} ${args.join(" ")}\n${result.error ?? result.stderr}`), { code: result.status === "timeout" || result.status === "cancelled" ? result.status : "command-failure" }); } w.signal.throwIfAborted(); @@ -119,8 +119,6 @@ async function prepareRevisionWorkspace(options, dependencies = {}) { if (options.request.platform.os === "windows") Object.assign(w.env, { OS: "Windows_NT", PATHEXT: ".COM;.EXE;.BAT;.CMD", PSModulePath: path.join(w.env.SystemRoot, "System32", "WindowsPowerShell", "v1.0", "Modules"), - ProgramFiles: path.join(root, "program-files"), "ProgramFiles(x86)": path.join(root, "program-files-x86"), - ProgramData: path.join(root, "program-data"), ALLUSERSPROFILE: path.join(root, "program-data"), }); for (const dir of [w.sourceDirectory, w.sampleDirectory, w.evidenceRoot, path.join(root, "hooks"), w.env.HOME, w.env.TEMP, w.env.APPDATA, w.env.LOCALAPPDATA, w.env.NUGET_PACKAGES]) await fs.mkdir(dir, { recursive: true }); @@ -144,6 +142,7 @@ async function prepareRevisionWorkspace(options, dependencies = {}) { await fs.mkdir(path.dirname(target), { recursive: true }); await fs.copyFile(file.path, target); } + w.signal.throwIfAborted(); w.status = "prepared"; return w; } catch (error) { @@ -179,7 +178,7 @@ async function layout(w) { !parsed.hash, "historical feed must be public HTTPS without credentials"); } w.adapter = { - version: 1, layout: /FSharp\.slnx/.test(windows ? ps : sh) ? "arcade-slnx-products" : "arcade-sln-products", + version: 2, layout: /FSharp\.slnx/.test(windows ? ps : sh) ? "arcade-slnx-products" : "arcade-sln-products", mapping: "identity: nested fsi/fsc projects with evaluated artifacts/bin//Release/ outputs", injection: await Promise.all(["UseLocalCompiler.Directory.Build.props", "UseLocalCompiler.Directory.Build.targets"] .map(async name => ({ path: name, sha256: hash(await fs.readFile(path.join(trustedRoot, name))) }))), @@ -360,6 +359,11 @@ async function buildRevision(w, dependencies = {}) { await invoke(w, "build", "/bin/bash", [path.join(w.sourceDirectory, "build.sh"), "--configuration", "Release", "--pack", "--bootstrap", "--product-build"]); } + // Product graphs can pack dependencies elsewhere; repack the built Core into its evaluated standalone output. + const packageOutput = path.resolve(w.sourceDirectory, rows.corePackage.PackageOutputPath); + requireValue(inside(w.sourceDirectory, packageOutput), "Core package output escapes source checkout"); + await invoke(w, "build", w.toolchain.hostPath, ["pack", path.join(w.sourceDirectory, projects.corePackage), + "--configuration", "Release", "--no-build", "--no-restore", "--output", packageOutput]); stage = "identity"; await sourceUnchanged(w); for (const [role, project] of Object.entries(projects)) { @@ -371,20 +375,26 @@ async function buildRevision(w, dependencies = {}) { stage = "identity"; w.product = await inventory(w, rows); await evidence(w, "product.json", w.product); + w.signal.throwIfAborted(); w.status = "built"; return { status: "built", product: w.product, toolchain: w.toolchain, buildInstanceId: w.buildInstanceId, commands: w.commands, evidence: w.evidence }; } catch (error) { - return { ...failure(w, error.code === "unavailable-runtime" ? "runtime" : stage, error), + const result = { ...failure(w, error.code === "unavailable-runtime" ? "runtime" : stage, error), buildInstanceId: w?.buildInstanceId, commands: w?.commands, evidence: w?.evidence }; + return owned.has(w) && w.signal.aborted ? finish(w, result) : result; } } -async function finish(w, result) { +function finish(w, result) { + return w.finalization ??= finalize(w, result); +} + +async function finalize(w, result) { if (w.globalJson) { try { await sourceUnchanged(w); } catch (error) { - if (!w.signal.aborted) result = { ...failure(w, "identity", error), ...(result.error ? { priorError: result.error } : {}) }; + result = { ...failure(w, "identity", error), ...(result.error ? { priorError: result.error } : {}) }; } } const logs = path.join(w.sourceDirectory, "artifacts", "log"); @@ -406,17 +416,26 @@ async function finish(w, result) { acquisitionDigest: fingerprintInput(w.toolchain) } : w.declaration, product: w.product, commands: w.commands, evidence: w.evidence, durationMs: Date.now() - w.started, }; - if (w.signal.aborted) { - requireValue(owned.has(w) && path.dirname(w.root) === w.parent && - path.basename(w.root) === `revision-${w.buildInstanceId}` && - !(await fs.lstat(w.root)).isSymbolicLink() && await fs.realpath(w.root) === w.root && - JSON.parse(await fs.readFile(path.join(w.root, "owner.json"))).buildInstanceId === w.buildInstanceId, - "refusing cleanup outside owned resolved scratch tree"); - await fs.rm(w.root, { recursive: true }); - outcome.scratchRemoved = true; + let written = false; + for (;;) { + if (w.signal.aborted && !outcome.scratchRemoved) { + Object.assign(outcome, failure(w, "finalization", w.signal.reason)); + requireValue(owned.has(w) && path.dirname(w.root) === w.parent && + path.basename(w.root) === `revision-${w.buildInstanceId}` && + !(await fs.lstat(w.root)).isSymbolicLink() && await fs.realpath(w.root) === w.root && + JSON.parse(await fs.readFile(path.join(w.root, "owner.json"))).buildInstanceId === w.buildInstanceId, + "refusing cleanup outside owned resolved scratch tree"); + await fs.rm(w.root, { recursive: true }); + outcome.scratchRemoved = true; + } + const bytes = JSON.stringify(outcome, null, 2); + await fs.writeFile(path.join(w.evidenceRoot, "observation.json"), bytes, { flag: written ? "w" : "wx" }); + written = true; + // An abort during evidence I/O must invalidate the persisted observation too. + if (w.signal.aborted && !outcome.scratchRemoved) continue; + w.evidence.push({ path: `${w.buildInstanceId}/observation.json`, sha256: hash(bytes) }); + return outcome; } - await evidence(w, "observation.json", outcome); - return outcome; } async function testSourceRevision(options, dependencies = {}) { diff --git a/eng/regression-bisect/workspace.test.cjs b/eng/regression-bisect/workspace.test.cjs index b3191c1dd28..5f6439d05c5 100644 --- a/eng/regression-bisect/workspace.test.cjs +++ b/eng/regression-bisect/workspace.test.cjs @@ -5,7 +5,7 @@ const path = require("node:path"); const { fixture } = require("./fixtures/reproduction.cjs"); const { seal } = require("./fixtures/contracts.cjs"); const { fingerprintInput } = require("./model.cjs"); -const { hash, isolatedEnvironment } = require("./toolchain-io.cjs"); +const { hash, isolatedEnvironment, run } = require("./toolchain-io.cjs"); const { execute } = require("./repro-process.cjs"); const { prepareRevisionWorkspace, buildRevision, testSourceRevision } = require("./workspace.cjs"); @@ -13,13 +13,15 @@ const git = process.platform === "win32" ? "C:\\Program Files\\Git\\cmd\\git.exe const platform = { os: "windows", architecture: "x64", rid: "win-x64" }; const ok = stdout => ({ status: "completed", complete: true, exitCode: 0, stdout, stderr: "" }); -async function controlled(t) { +async function controlled(t, os = "windows") { const f = await fixture(t); + const targetPlatform = { ...platform, os, rid: `${os === "windows" ? "win" : "linux"}-x64` }; const mirror = path.join(f.root, "mirror"); await fs.mkdir(mirror); await fs.writeFile(path.join(f.root, "gitconfig"), ""); + const runGit = (args, settings) => run(git, args, settings); const command = async args => { - const result = await execute(git, ["-c", "core.hooksPath=", ...args], { + const result = await runGit(["-c", "core.hooksPath=", ...args], { cwd: mirror, env: { ...isolatedEnvironment(f.root, platform), GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: path.join(f.root, "gitconfig") }, timeoutMs: 30000, }); @@ -49,12 +51,13 @@ async function controlled(t) { shas.push(await command(["rev-parse", "HEAD"])); } f.manifest.matrix[0].languageVersion = "preview"; + f.manifest.matrix[0].os = os; seal(f.manifest); const options = { request: { schemaVersion: 1, repository: "dotnet/fsharp", sourceSha: shas[0], phase: "main", manifestDigest: fingerprintInput(f.manifest), approvalDigest: f.manifest.approvedInputDigest, - platform, allocation: { buildTimeoutMs: 60000, workerTimeoutMs: 120000 }, + platform: targetPlatform, allocation: { buildTimeoutMs: 60000, workerTimeoutMs: 120000 }, }, manifest: f.manifest, sourceDirectory: f.sourceDirectory, mirrorDirectory: mirror, scratchDirectory: path.join(f.root, "scratch"), evidenceDirectory: path.join(f.root, "evidence"), @@ -73,7 +76,7 @@ async function controlled(t) { activeSdk = request.version; const runtime = `${activeSdk.split(".")[0]}.0.0`; for (const [name, text] of Object.entries({ - "dotnet.exe": "host", + [os === "windows" ? "dotnet.exe" : "dotnet"]: "host", [`shared/Microsoft.NETCore.App/${runtime}/System.Private.CoreLib.dll`]: "runtime", [`sdk/${activeSdk}/FSharp/fsi.runtimeconfig.json`]: JSON.stringify({ runtimeOptions: { framework: { name: "Microsoft.NETCore.App", version: runtime } } }), [`sdk/${activeSdk}/FSharp/fsi.deps.json`]: '{"libraries":{}}', @@ -90,7 +93,7 @@ async function controlled(t) { }, run: async (exe, args) => { const root = path.dirname(exe), runtime = `${activeSdk.split(".")[0]}.0.0`; - if (args[0] === "--info") return ok(`Base Path: ${path.join(root, "sdk", activeSdk)}\nRID: win-x64\n`); + if (args[0] === "--info") return ok(`Base Path: ${path.join(root, "sdk", activeSdk)}\nRID: ${targetPlatform.rid}\n`); if (args[0] === "--version") return ok(activeSdk); if (args[0] === "--list-sdks") return ok(`${activeSdk} [${path.join(root, "sdk")}]`); if (args[0] === "--list-runtimes") return ok(`Microsoft.NETCore.App ${runtime} [${path.join(root, "shared", "Microsoft.NETCore.App")}]`); @@ -108,7 +111,11 @@ async function controlled(t) { }; }; const deps = { toolchains, execute: async (exe, args, settings) => { - if (exe === git) return execute(exe, args, settings); + if (exe === git) { + const result = await runGit(args, settings); + if (args.includes("diff") && f.reproductions === 2) f.finalizeCancellation?.abort(); + return result; + } for (const key of ["GITHUB_TOKEN", "GH_TOKEN", "NUGET_AUTH_TOKEN", "SSH_AUTH_SOCK", "MSBuildSDKsPath"]) { assert.equal(settings.env[key], undefined); } @@ -120,12 +127,24 @@ async function controlled(t) { assemblyName: path.basename(file, ".dll"), ...(file.endsWith(".nupkg") ? { id: "FSharp.Core", version: "1.0.0", entries: [ { path: "lib/netstandard2.0/FSharp.Core.dll", sha256: hash(`core-${builds.at(-1).sha}`) }, - ] } : {}) }); + ], ...f.packageIdentity } : {}) }); } return ok(JSON.stringify(result)); } if (args[0] === "msbuild") return ok(JSON.stringify({ Properties: await properties(settings.cwd, args[1]) })); + if (args[0] === "pack") { + assert.deepEqual(args, ["pack", path.join(settings.cwd, "src", "FSharp.Core", "FSharp.Core.fsproj"), + "--configuration", "Release", "--no-build", "--no-restore", + "--output", (await properties(settings.cwd, args[1])).PackageOutputPath]); + if (f.packFail) return { ...ok(""), exitCode: 1, stderr: f.manifest.assertions.failure.stderr }; + const directory = args.at(-1); + await fs.mkdir(directory, { recursive: true }); + if (!f.missingPackage) await fs.writeFile(path.join(directory, "FSharp.Core.1.0.0.nupkg"), `package-${builds.at(-1).sha}`); + f.packs = (f.packs ?? 0) + 1; + return ok(""); + } if (args.includes("--exec")) { + f.reproductions = (f.reproductions ?? 0) + 1; const fsi = args[3]; const core = path.join(path.dirname(fsi), "FSharp.Core.dll"); const runtime = args[2]; @@ -137,8 +156,9 @@ async function controlled(t) { return ok(f.manifest.assertions.pass.stdout); } const source = settings.cwd; - const sha = (await execute(git, ["rev-parse", "HEAD"], { ...settings, timeoutMs: 30000 })).stdout.trim(); - builds.push({ sha, source, env: settings.env, args }); + const sha = (await runGit(["rev-parse", "HEAD"], { ...settings, timeoutMs: 30000 })).stdout.trim(); + builds.push({ sha, source, exe, env: settings.env, args }); + if (f.launchFail) return { ...ok(""), status: "exception", complete: false, exitCode: 1, error: "process supervision failed" }; if (f.fail) { const log = path.join(source, "artifacts", "log", "Release", "bootstrap.binlog"); await fs.mkdir(path.dirname(log), { recursive: true }); @@ -161,8 +181,9 @@ async function controlled(t) { } } const p = await properties(source, "FSharp.Core.fsproj"); - await fs.mkdir(p.PackageOutputPath, { recursive: true }); - if (!f.missingPackage) await fs.writeFile(path.join(p.PackageOutputPath, "FSharp.Core.1.0.0.nupkg"), `package-${sha}`); + const dependencyOutput = path.join(path.dirname(p.PackageOutputPath), "Dependency", "Shipping"); + await fs.mkdir(dependencyOutput, { recursive: true }); + if (!f.missingPackage) await fs.writeFile(path.join(dependencyOutput, "FSharp.Core.1.0.0.nupkg"), `package-${sha}`); await f.afterBuild?.(source); return ok(""); } }; @@ -199,8 +220,48 @@ test("fresh revisions use their exact SHA, SDK, TFM and independent same-version await assert.rejects(fs.access(hookMarker)); }); +for (const os of ["windows", "linux"]) for (const revision of [0, 1]) { + test(`${os} revision ${revision} builds Release products and packs Core into its evaluated output`, async t => { + const f = await controlled(t, os); + const result = await testSourceRevision({ + ...f.options, request: { ...f.options.request, sourceSha: f.shas[revision] }, + }, f.deps); + assert.equal(result.outcome.outcome, "pass", JSON.stringify(result)); + assert.equal(f.builds.length, 1); + const { source, args, exe, env } = f.builds[0]; + assert.equal(exe, os === "windows" + ? path.join(env.SystemRoot, "System32", "WindowsPowerShell", "v1.0", "powershell.exe") : "/bin/bash"); + assert.deepEqual(args, os === "windows" + ? ["-NoProfile", "-ExecutionPolicy", "Bypass", "-File", path.join(source, "eng", "Build.ps1"), + "-restore", "-build", "-c", "Release", "-pack", "-bootstrap", "-noVisualStudio", "-msbuildEngine", "dotnet", + `/p:Projects=${path.join(source, "src", "Microsoft.FSharp.Compiler", "Microsoft.FSharp.Compiler.fsproj")}`] + : [path.join(source, "build.sh"), "--configuration", "Release", "--pack", "--bootstrap", "--product-build"]); + assert.equal(f.packs, 1); + assert.equal(result.product.files.find(x => x.role === "corePackage").path, + path.join(source, "artifacts", "packages", "Release", "Shipping", "FSharp.Core.1.0.0.nupkg")); + }); +} + +for (const [name, identity] of [ + ["ID", { id: "Other.Core" }], + ["version", { version: "2.0.0" }], + ["Core bytes", { entries: [{ path: "lib/netstandard2.0/FSharp.Core.dll", sha256: hash("different build") }] }], + ["Core entry", { entries: [{ path: "tools/FSharp.Core.dll", sha256: hash("different build") }] }], +]) test(`mismatched package ${name} is rejected before reproduction`, async t => { + const f = await controlled(t); + f.packageIdentity = identity; + const result = await testSourceRevision(f.options, f.deps); + assert.equal(result.status, "untestable"); + assert.equal(result.outcome.outcome, "untestable"); + assert.equal(result.outcome.reason, "wrong-identity"); + assert.match(result.error.message, /Core package identity\/bytes/); + assert.equal(f.reproductions, undefined); +}); + for (const [name, change, reason] of [ ["failed source build", f => { f.fail = true; }, "build-failure"], + ["failed process launch", f => { f.launchFail = true; }, "build-failure"], + ["failed Core pack", f => { f.packFail = true; }, "build-failure"], ["timed out source build", f => { f.timeout = true; }, "timeout"], ["stale fsi", f => { f.stale = "fsi"; }, "wrong-identity"], ["stale fsc", f => { f.stale = "fsc"; }, "wrong-identity"], @@ -215,6 +276,7 @@ for (const [name, change, reason] of [ assert.equal(result.outcome.outcome, "untestable"); assert.equal(result.outcome.reason, reason, JSON.stringify(result)); assert.ok(result.error); + if (f.launchFail) assert.match(result.error.message, /process supervision failed/); if (f.fail) assert.ok(result.evidence.some(e => e.path.endsWith(".binlog"))); assert.ok(result.evidence.every(e => !path.isAbsolute(e.path) && !e.path.split("/").includes("..") && /^[a-f0-9]{64}$/.test(e.sha256))); }); @@ -243,6 +305,36 @@ test("cancellation removes only its scratch tree and retains failure evidence", await fs.access(f.options.mirrorDirectory); }); +for (const when of ["before-build", "during-build", "finalization"]) { + test(`${when} cancellation cannot return success or retain its staged scratch tree`, async t => { + const f = await controlled(t); + const controller = new AbortController(); + const sibling = await prepareRevisionWorkspace(f.options, f.deps); + const options = { ...f.options, signal: controller.signal }; + let result; + if (when === "finalization") { + f.finalizeCancellation = controller; + result = await testSourceRevision(options, f.deps); + } else { + const w = await prepareRevisionWorkspace(options, f.deps); + assert.equal(w.status, "prepared"); + if (when === "before-build") controller.abort(); + else f.cancel = controller; + result = await buildRevision(w, f.deps); + } + assert.equal(result.status, "untestable"); + assert.equal(result.outcome.outcome, "untestable"); + assert.equal(result.error.code, "cancelled"); + assert.equal(result.scratchRemoved, true); + assert.deepEqual(await fs.readdir(f.options.scratchDirectory), [path.basename(sibling.root)]); + const observation = result.evidence.find(e => e.path.endsWith("/observation.json")); + assert.ok(observation); + const bytes = await fs.readFile(path.join(f.options.evidenceDirectory, observation.path)); + assert.equal(hash(bytes), observation.sha256); + assert.equal(JSON.parse(bytes).error.code, "cancelled"); + }); +} + test("concurrent workspace preparation cannot collide and prepared poison is rejected", async t => { const f = await controlled(t); const workspaces = await Promise.all([prepareRevisionWorkspace(f.options, f.deps), prepareRevisionWorkspace(f.options, f.deps)]); From d5f65ea0f05a743842347a685bd3955fe73a6c88 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 07:48:18 +0200 Subject: [PATCH 18/99] Cover non-pass historical source observations Exercise expected failures, unexpected diagnostics, timeouts and mixed repeats through the real source worker. Verify returned and persisted outcomes and record scoped validation with the exact-SDK smoke blocker. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 19 +++++++---- eng/regression-bisect/workspace.test.cjs | 41 +++++++++++++++++++++++- 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index 498d6da99ea..606a1f867a1 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -427,14 +427,21 @@ result. No live issue discovery or full compiler test suite is involved. Local verification on 2026-09-19: corrective RED cases reproduced the graph's `Dependency/Shipping` output mismatch and three staged/finalization cancellation failures. Real subprocess cases reproduced exited-parent hangs and detached-server -timeouts. The final scoped suite passed **519 tests**, including 28 workspace +timeouts. The final scoped suite passed **523 tests**, including 32 workspace contracts. In-memory mutations to use Debug and bypass package validation failed the command-contract case and all four package-identity cases, respectively. +Four additional parameterized cases first failed with the pass-only process +fixture, then passed with per-attempt process results: expected failure, +unexpected diagnostic, reproduction timeout and mixed repetitions. They exercise +the real worker/classifier and verify returned outcomes, complete repeats, and +digested `reproduction.json`/`observation.json` evidence. Forcing either the +worker's returned outcome or only its persisted observation to pass made all +four cases fail. Production outcome propagation required no change. Changed-script syntax (including the PowerShell helper), formatting, and working/ sprint-range whitespace checks passed. The real Windows x64 smoke pinned -`bf13cf234bc47e62166bc82748d6784d0aee086f`, build instance -`20f99ee4-f35e-48c7-8620-b2ecce174da8`. Its accurate result is +`5a8b9adb034f420a5c2c1f577670c5eeeb454b7e`, build instance +`2f54fabe-fcb6-424d-a63e-dcd508028681`. Its accurate result is **untestable / unavailable-sdk**, exit 2: exact SDK `11.0.100-rc.1.26420.103` has no verified public archive available to acquisition. Release metadata returned 200, but @@ -443,9 +450,9 @@ returned 404 under both `https://builds.dotnet.microsoft.com/dotnet` and `https://ci.dot.net/public`. Checkout/source-integrity checks passed and `global.json` retained its hash. Acquisition stopped before the build entrypoint, so no build binlog or local FSI result exists; this is not a green clean build -or compiler suite. Retained evidence is -`Q:\fsharp-workspace-smoke-D70Agl\result.json`, SHA-256 -`28f8a3fe1061f006e8786ef3d426e8e864220f662dd5fc8cb226ef6ff284e302`. +or compiler suite. All 18 evidence digests were verified. Retained evidence is +`Q:\fsharp-workspace-smoke-XI1JHA\result.json`, SHA-256 +`9edb1e7f3786d4df56ad42989420e2de81a2cc518e96f9db3ebf8e48ffd609a5`. ## Reproduction execution diff --git a/eng/regression-bisect/workspace.test.cjs b/eng/regression-bisect/workspace.test.cjs index 5f6439d05c5..e477438548d 100644 --- a/eng/regression-bisect/workspace.test.cjs +++ b/eng/regression-bisect/workspace.test.cjs @@ -153,7 +153,7 @@ async function controlled(t, os = "windows") { assemblyName: "FSharp.Core", complete: true, architecture: "x64", runtimeVersion: runtime, runtimePath: path.join(path.dirname(exe), "shared", "Microsoft.NETCore.App", runtime), })); - return ok(f.manifest.assertions.pass.stdout); + return f.outputs ? f.outputs[f.reproductions - 1] : { ...ok(""), ...f.manifest.assertions.pass }; } const source = settings.cwd; const sha = (await runGit(["rev-parse", "HEAD"], { ...settings, timeoutMs: 30000 })).stdout.trim(); @@ -220,6 +220,45 @@ test("fresh revisions use their exact SHA, SDK, TFM and independent same-version await assert.rejects(fs.access(hookMarker)); }); +for (const [name, outputs, outcome, reason] of [ + ["expected failure", a => [a.failure, a.failure], "product-failure", "failure-assertion"], + ["unexpected diagnostic", a => [a.failure, { ...a.failure, stderr: "different regression\n" }], "untestable", "assertion-mismatch"], + ["reproduction timeout", a => [a.failure, { ...a.failure, status: "timeout", complete: false, exitCode: null }], "untestable", "timeout"], + ["mixed repetitions", a => [a.pass, a.failure], "untestable", "flaky"], +]) test(`source worker preserves ${name} in returned and persisted observations`, async t => { + const f = await controlled(t); + f.outputs = outputs(f.manifest.assertions).map(output => ({ ...ok(""), ...output })); + const expected = { schemaVersion: 1, outcome, reason }; + const result = await testSourceRevision(f.options, f.deps); + assert.equal(result.status, "tested", JSON.stringify(result)); + assert.equal(f.builds.length, 1); + assert.equal(f.packs, 1); + assert.equal(f.reproductions, 2); + assert.deepEqual(result.outcome, expected); + assert.deepEqual(result.reproduction.outcome, expected); + assert.equal(result.reproduction.manifestDigest, f.options.request.manifestDigest); + assert.equal(result.reproduction.cells.length, 1); + const cell = result.reproduction.cells[0]; + assert.equal(cell.id, f.manifest.matrix[0].id); + assert.deepEqual(cell.outcome, expected); + assert.deepEqual(cell.attempts.map(a => a.execution), f.outputs); + assert.ok(cell.attempts.every(a => Object.values(a.checks).every(Boolean))); + for (const name of ["reproduction", "observation"]) { + const entry = result.evidence.find(e => e.path === `${result.buildInstanceId}/${name}.json`); + assert.ok(entry, `missing ${name} evidence`); + const bytes = await fs.readFile(path.join(f.options.evidenceDirectory, entry.path)); + assert.equal(hash(bytes), entry.sha256); + const persisted = JSON.parse(bytes); + assert.deepEqual(persisted.outcome, expected); + if (name === "observation") { + assert.equal(persisted.status, "tested"); + assert.equal(persisted.actualSourceSha, f.options.request.sourceSha); + assert.equal(persisted.buildInstanceId, result.buildInstanceId); + assert.deepEqual(persisted.reproduction, result.reproduction); + } else assert.deepEqual(persisted, result.reproduction); + } +}); + for (const os of ["windows", "linux"]) for (const revision of [0, 1]) { test(`${os} revision ${revision} builds Release products and packs Core into its evaluated output`, async t => { const f = await controlled(t, os); From 4c843d17bff39bf7419d49c47df22ec9fd5f5f54 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 10:42:41 +0200 Subject: [PATCH 19/99] Remove duplicate mixed-repetition model assertion Keep flaky repetition classification covered by the parameterized model suite. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/repro.test.cjs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/eng/regression-bisect/repro.test.cjs b/eng/regression-bisect/repro.test.cjs index e80e7c57649..582f981f01e 100644 --- a/eng/regression-bisect/repro.test.cjs +++ b/eng/regression-bisect/repro.test.cjs @@ -3,8 +3,7 @@ const assert = require("node:assert/strict"); const fs = require("node:fs/promises"); const path = require("node:path"); const { fixture } = require("./fixtures/reproduction.cjs"); -const { seal, attempt } = require("./fixtures/contracts.cjs"); -const { classifyAttempts } = require("./model.cjs"); +const { seal } = require("./fixtures/contracts.cjs"); const { selectFsiCommand, verifyProjectIdentities, evaluateAssertion, runReproduction } = require("./repro.cjs"); const { execute } = require("./repro-process.cjs"); const { isolatedEnvironment, hash } = require("./toolchain-io.cjs"); @@ -73,13 +72,6 @@ test("runtime and diagnostic assertions use the model vocabulary and exact evide assert.equal(evaluateAssertion(f.manifest, { ...execution, stdout: "" }, { phase: "compile" }).outcome, "pass"); }); -test("mixed repetitions remain untestable", () => { - const { manifest } = require("./fixtures/contracts.cjs"); - assert.deepEqual(classifyAttempts([attempt("pass"), attempt("product-failure")], manifest()), { - schemaVersion: 1, outcome: "untestable", reason: "flaky", - }); -}); - async function projectFixture(t, options) { const f = await fixture(t, options); const project = await f.file("source/App.fsproj", ""); From 2ec6de6837b5b367e1f1186e850b8faa2d188b16 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 10:42:41 +0200 Subject: [PATCH 20/99] Bound loopback archive reads to the advertised length Avoid a premature-close race when the installer receives all archive bytes before the file stream performs its final EOF read. Reproduce with delayed EOF through the actual loopback transport. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 4 ++- eng/regression-bisect/toolchain-installer.cjs | 2 +- .../toolchain-installer.test.cjs | 32 +++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index 606a1f867a1..60e8906637b 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -239,7 +239,9 @@ installer. This transport boundary records the **actual public download's** redirects/digests without reimplementing extraction or trusting a separate HEAD request. Verification waits for pending server deliveries and file closure, not just installer exit; cleanup also drains deliveries before removing their -archive. Late delivery errors still fail acquisition. The official retained +archive. Reads stop at the advertised archive length so a client's completed +download cannot race a redundant EOF read. Late delivery errors still fail +acquisition. The official retained archive is hashed again. Supplemental runtime installation uses the official skip-non-versioned-files flag to preserve the SDK host. Arcade-owned files are neither edited nor copied. diff --git a/eng/regression-bisect/toolchain-installer.cjs b/eng/regression-bisect/toolchain-installer.cjs index 516812a9911..bbe226840a1 100644 --- a/eng/regression-bisect/toolchain-installer.cjs +++ b/eng/regression-bisect/toolchain-installer.cjs @@ -123,7 +123,7 @@ async function startFeed(fs, download, archiveName, productDirectory, version, p else { const file = await fs.open(download, 'r'); try { - await pipeline(file.createReadStream({ autoClose: false }), res); + await pipeline(file.createReadStream({ autoClose: false, end: size ? size - 1 : undefined }), res); downloads++; } finally { await file.close(); diff --git a/eng/regression-bisect/toolchain-installer.test.cjs b/eng/regression-bisect/toolchain-installer.test.cjs index 7ea55e1eff9..037cbf62bcf 100644 --- a/eng/regression-bisect/toolchain-installer.test.cjs +++ b/eng/regression-bisect/toolchain-installer.test.cjs @@ -3,6 +3,7 @@ const assert = require('node:assert/strict'); const { execFile } = require('node:child_process'); const crypto = require('node:crypto'); +const fileSystem = require('node:fs'); const fs = require('node:fs/promises'); const http = require('node:http'); const path = require('node:path'); @@ -162,6 +163,37 @@ for (const settings of [ }); } +test('archive delivery completes before a slow EOF read after its advertised length', async t => { + const { options, deps } = await fixture(t); + let reads = 0; + deps.fs = { + ...fs, + async open(filename, ...args) { + const file = await fs.open(filename, ...args); + if (filename !== `${options.archivePath}.download` || ++reads !== 2) return file; + return { + createReadStream(settings) { + return fileSystem.createReadStream(filename, { + ...settings, fd: file.fd, + fs: { + close(_fd, callback) { file.close().then(() => callback(null), callback); }, + read(fd, buffer, offset, length, position, callback) { + fileSystem.read(fd, buffer, offset, length, position, (error, bytesRead, data) => { + if (!error && bytesRead === 0) setTimeout(callback, 100, error, bytesRead, data); + else callback(error, bytesRead, data); + }); + }, + }, + }); + }, + close: () => file.close(), + }; + }, + }; + const result = await installArchive(options, deps); + assert.equal(result.archive.actualSha512, options.archive.sha512); +}); + for (const os of ['win', 'linux']) { for (const completion of ['success', 'stream error', 'close error', 'installer error']) { test(`awaits archive delivery after installer exit: ${os}, ${completion}`, async t => { From b00a8d5e74621226a6c74ee6ab1c5e551b74be38 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 20:37:19 +0200 Subject: [PATCH 21/99] Verify historical SDK provenance and clean workspace acceptance Distinguish publisher checksums from exact approved historical HTTPS measurements, retain and verify installer bytes, and preserve explicit blocked reasons. Repair clean-workspace evidence collection and exercise fresh released/source product smokes without SDK or Core fallback. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/policies/regression-bisect.json | 26 ++ eng/regression-bisect/README.md | 137 +++++- eng/regression-bisect/archive-provenance.cjs | 52 +++ .../evidence/Evidence.csproj | 3 +- eng/regression-bisect/fixtures/checkpoint.txt | 2 +- eng/regression-bisect/fixtures/contracts.cjs | 11 +- .../fixtures/reproduction.cjs | 6 +- eng/regression-bisect/fixtures/smoke.cjs | 6 + eng/regression-bisect/model.cjs | 26 +- eng/regression-bisect/model.test.cjs | 63 ++- eng/regression-bisect/repro.cjs | 11 +- eng/regression-bisect/repro.smoke.cjs | 7 +- eng/regression-bisect/repro.test.cjs | 24 ++ eng/regression-bisect/toolchain-installer.cjs | 146 +++++-- .../toolchain-installer.test.cjs | 390 +++++++++++++++++- eng/regression-bisect/toolchain-io.cjs | 85 +++- eng/regression-bisect/toolchains.cjs | 87 +++- eng/regression-bisect/toolchains.test.cjs | 97 ++++- eng/regression-bisect/workspace.cjs | 18 +- eng/regression-bisect/workspace.smoke.cjs | 3 +- eng/regression-bisect/workspace.test.cjs | 56 ++- 21 files changed, 1124 insertions(+), 132 deletions(-) create mode 100644 eng/regression-bisect/archive-provenance.cjs diff --git a/.github/policies/regression-bisect.json b/.github/policies/regression-bisect.json index 95f3d460f4d..aedcad02002 100644 --- a/.github/policies/regression-bisect.json +++ b/.github/policies/regression-bisect.json @@ -42,6 +42,32 @@ "https://ci.dot.net/public", "https://dotnetcli.blob.core.windows.net/dotnet", "https://download.visualstudio.microsoft.com/download/pr" + ], + "measuredArchives": [ + { + "sourceSha": "5a8b9adb034f420a5c2c1f577670c5eeeb454b7e", + "product": "sdk", + "version": "11.0.100-rc.1.26420.103", + "rid": "win-x64", + "origin": "https://builds.dotnet.microsoft.com", + "url": "https://builds.dotnet.microsoft.com/dotnet/Sdk/11.0.100-rc.1.26420.103/dotnet-sdk-11.0.100-rc.1.26420.103-win-x64.zip", + "redirectUrls": [], + "maxBytes": 1073741824, + "timeoutMs": 300000, + "approvalRevision": "2ec6de6837b5b367e1f1186e850b8faa2d188b16" + }, + { + "sourceSha": "5a8b9adb034f420a5c2c1f577670c5eeeb454b7e", + "product": "sdk", + "version": "11.0.100-rc.1.26420.103", + "rid": "win-x64", + "origin": "https://ci.dot.net", + "url": "https://ci.dot.net/public/Sdk/11.0.100-rc.1.26420.103/dotnet-sdk-11.0.100-rc.1.26420.103-win-x64.zip", + "redirectUrls": [], + "maxBytes": 1073741824, + "timeoutMs": 300000, + "approvalRevision": "2ec6de6837b5b367e1f1186e850b8faa2d188b16" + } ] } } diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index 60e8906637b..ef436586538 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -12,7 +12,8 @@ Records under `fixtures` are not reproduction approvals. Run the dependency-free suite from the repository root: ```powershell -node --test eng\regression-bisect\model.test.cjs eng\regression-bisect\intake.test.cjs eng\regression-bisect\state.test.cjs eng\regression-bisect\toolchains.test.cjs eng\regression-bisect\toolchain-installer.test.cjs +$tests = @(Get-ChildItem 'eng\regression-bisect' -Filter '*.test.cjs' | ForEach-Object FullName) +node --test @tests ``` ## Trust boundary @@ -129,7 +130,7 @@ producer, and package fingerprints. Admission also compares runner environments across all endpoints and **reproduction** SDK/runtime identities across the two source endpoints. A source cell can additionally carry `buildToolchain`: `sourceSha`, `globalJsonDigest`, `evidenceDigest`, `requestedSdk`, -`requestedTools`, `requestedCompilerRuntime`, verified `sdk` and +`requestedTools`, `requestedCompilerRuntime`, the full SDK `archive` receipt, verified `sdk` and `compilerRuntime` product identities, and `verified: true`, `supported: true`. Requested/resolved versions and source SHA must agree. Both source endpoints must supply this field or both omit it; legacy observations retain the original @@ -207,16 +208,59 @@ never interchangeable. Missing or inconsistent graph evidence is untestable. | `verifyToolchainIdentity({ policy, toolchain, identity })` | `{ status: "verified" }` only for matching requested/resolved SDK, host/SDK paths, RID/architecture, archive hashes, bundled F# identities, and all exact runtime requirements. The acquisition request binds `toolchain.installationRoot`; evidence cannot nominate a different root. This validates captured evidence, not the authenticity of arbitrary JSON. | Failures return `{ status: "untestable", outcome, error: { stage, code, message }, -attempts }`, not reproduction pass/failure. Missing historical metadata, -unavailable/checksum-less archives, unsupported installer flags or declarations, +attempts }`, not reproduction pass/failure. Missing archives, insufficient +checksum provenance, network failures, unsupported installer flags or declarations, SDK/tool declaration disagreement, changed source files, and unfulfilled runtime requirements fail closed. Historical previews are allowed, but never selected -as released stable. Public preview fallback uses an official archive SHA-512 -sidecar; missing checksum provenance cannot become success through a host SDK. +as released stable. `checksum-provenance` means an available archive lacks +approved provenance or fails its integrity checks; it is distinct from +`unavailable-sdk`/`unavailable-runtime` +(missing archives) and `network-failure`. None is a positive reproduction. Exact `global.json.tools.runtimes` arrays (`dotnet`, `aspnetcore`, `windowsdesktop`, optionally suffixed with the matching architecture) become separate build-runtime requirements. Unevaluated MSBuild version expressions and foreign-architecture requirements are explicitly untestable, not guessed. +Acquisition binds each requirement to the archive that supplied it, including +runtimes already bundled in the SDK; missing supplemental receipts cannot be +hidden by an incomplete installation list. + +Archive provenance is discriminated, not inferred from the existence of a hash: + +* `publisher-checksum` retains the independent SHA-512 in `archive.sha512`. + Release metadata and official sidecars always take precedence. Released + stable selection still requires official release metadata and its checksum. +* `approved-official-https-measured` has **no** publisher `sha512` field. + It requires an exact entry in the pinned policy's `toolchains.measuredArchives`, + including historical source SHA, prerelease SDK version, product, RID, HTTPS + origin and URL, permitted redirect URLs, byte/time bounds and approval revision. + `approvalDigest` is the canonical fingerprint of that entry, not authorization + supplied by an issue. Adding a public feed does not authorize measured downloads. + +The initial exceptions cover only source +`5a8b9adb034f420a5c2c1f577670c5eeeb454b7e`, SDK +`11.0.100-rc.1.26420.103`, product `sdk`, RID `win-x64`, and its exact +`builds.dotnet.microsoft.com` and `ci.dot.net/public` archives. They permit no redirects, at most 1 GiB, +and a five-minute download. Other revisions/platforms need an explicitly reviewed +policy entry or publisher checksum; no version substitution is allowed. +The recorded approval revision is the preserved recovery base +`2ec6de6837b5b367e1f1186e850b8faa2d188b16`; authorization still comes from the +trusted policy checkout, never the existence of that SHA or digest alone. +After both sidecars are checked, bounded availability probes select an approved +exact archive before installation; the installer itself has no public fallback. + +Both branches retain attempted metadata/sidecar sources and HTTP statuses, +requested and final archive URLs, every redirect, `bytesReceived`, receipt time, +measured SHA-256/`actualSha512`, and `retained` path/size/digests. The same +discriminator appears in requested/resolved identities, installation receipts, +workspace results and source observation build-toolchain records. Changing the +receipt or claiming a measured digest is a publisher checksum is rejected. +Before reproduction reuse, retained archives must still exist as regular, +non-linked files and are rehashed; missing or changed bytes invalidate reuse. +Read-only file attributes alone are not proof of immutable identity. +An installation list cannot replace or omit the SDK receipt, and supplemental +runtime receipts are verified separately. Receipt/schema and trusted-policy +changes invalidate old checkpoint evidence rather than upgrading old results +into measured or publisher-verified claims. The shared boundaries are injectable: `fs` (`node:fs/promises`), `clock` (`() => Date`), `http(url, { destination?, method? })`, `run(executable, args, @@ -234,7 +278,7 @@ script parameter support, then invokes the official installer with exact version/architecture and archive-retention flags. Linux-musl archive requests use the official `--os linux-musl` option rather than dropping the ABI from the RID. A loopback-only, bounded -feed serves the already downloaded and SHA-512-verified archive to that +feed serves the already downloaded and provenance-verified archive to that installer. This transport boundary records the **actual public download's** redirects/digests without reimplementing extraction or trusting a separate HEAD request. Verification waits for pending server deliveries and file closure, @@ -242,7 +286,8 @@ not just installer exit; cleanup also drains deliveries before removing their archive. Reads stop at the advertised archive length so a client's completed download cannot race a redundant EOF read. Late delivery errors still fail acquisition. The official retained -archive is hashed again. Supplemental runtime +archive and original download are hashed again and must agree in both SHA-256 +and SHA-512. No second public archive download is permitted. Supplemental runtime installation uses the official skip-non-versioned-files flag to preserve the SDK host. Arcade-owned files are neither edited nor copied. @@ -386,6 +431,9 @@ The graph can place dependency packages under `Dependency/Shipping`; that is not the standalone Core output, and the adapter does not guess between the two. Inventory verifies bytes, assembly versions, runtimeconfigs and the Core package, rejecting missing outputs, SDK-bundled compiler fallback and redirected artifacts. +Only the selected compiler, FSI, tasks, Core and Core package need out-of-process +metadata inspection. Every dependency is still hashed before and after execution; +large localized inventories cannot exceed Windows' process command-line limit. No timestamp selection or shared same-version Core cache is used. Exact acquisition uses the trusted checkout's existing Arcade acquisition helper, not a helper assumed to exist in historical source. The evidence helper likewise comes from @@ -444,8 +492,9 @@ sprint-range whitespace checks passed. The real Windows x64 smoke pinned `5a8b9adb034f420a5c2c1f577670c5eeeb454b7e`, build instance `2f54fabe-fcb6-424d-a63e-dcd508028681`. Its accurate result is -**untestable / unavailable-sdk**, exit 2: exact SDK -`11.0.100-rc.1.26420.103` has no verified public archive available to acquisition. +**untestable / unavailable-sdk**, exit 2, as recorded by the old adapter. +That reason conflated absent checksum provenance with archive availability: +the exact SDK `11.0.100-rc.1.26420.103` archive itself returned HTTP 200. Release metadata returned 200, but `Sdk/11.0.100-rc.1.26420.103/dotnet-sdk-11.0.100-rc.1.26420.103-win-x64.zip.sha512` returned 404 under both `https://builds.dotnet.microsoft.com/dotnet` and @@ -456,6 +505,65 @@ or compiler suite. All 18 evidence digests were verified. Retained evidence is `Q:\fsharp-workspace-smoke-XI1JHA\result.json`, SHA-256 `9edb1e7f3786d4df56ad42989420e2de81a2cc518e96f9db3ebf8e48ffd609a5`. +### Fresh historical acceptance: 2026-09-19 + +Fresh evidence is retained separately under +`Q:\rb912-s5-20260919-acceptance`. The original failed result above was +rehashed and remains unchanged; no earlier reproduction artifacts were reused. +The supplied archive-200/checksum-404 case is a deterministic RED fixture. +During the fresh live run, the exact build-feed archive instead returned 404; +the existing official CI feed returned 200 for the same version/RID. Its separate +exact policy approval was used without substituting SDKs or weakening stable +release checks. + +The clean workspace command completed with exit 0 at source +`5a8b9adb034f420a5c2c1f577670c5eeeb454b7e`, build instance +`6444a4de-52d5-40f6-b065-8969b515357c`. Bootstrap, Release product build, Core +pack, evidence-helper build and two **built FSI** executions succeeded. +Requested/resolved SDK was `11.0.100-rc.1.26420.103`; executing runtime was +`11.0.0-rc.1.26420.103`, Windows x64. Historical declarations stayed unchanged. +The measured CI archive contained 332,886,529 bytes, with SHA-256 +`30a36953c458c07ae367c0642d5922973ad84c56be7c7bb8badb86dfa4fd9369`. +Both measured digests, public sources/statuses, zero redirects, retained archive +identity and same-byte installer receipt are preserved in the acquisition evidence. +This is **approved measured HTTPS**, not a claimed publisher checksum. + +`verified-products.json` combines that fresh product with a newly acquired +publisher-verified stable SDK `10.0.401` / runtime `10.0.12`. +The full released-and-source reproduction command also exited 0: +**43 scenarios matched their expected outcomes**: 23 positive passes, +6 intended product failures and 14 intentional identity rejections. +The latter are negative controls, not accepted product passes. All four source +compiler/tasks/targets/Core fallback controls compiled successfully before +`wrong-identity` rejection. Both phases exercised real separate project/TFM +builds, Debug/Release and language/nullability settings, FSI, runtime execution, +diagnostics, implicit/explicit/central Core references and immutable older-producer +consumption. The final audit rehashed the older DLL/package, both SDK archives, +all inventoried products and all 45 clean-workspace evidence files. + +Principal records, relative to that storage root: + +| Record | SHA-256 | +| --- | --- | +| `fsharp-workspace-smoke-b3FroQ\result.json` | `b9c229bdc3d33001fd4462e6630371e9992b5dfdab070d3668c4be79ed1045b8` | +| `repro-both-01\smoke-results.json` | `0b14541fa738841a53f3c9c87dc4e82ae1070bd1240eee63461110c1724215be` | +| `acceptance-audit-final.json` | `ecdd073f5f1856e4d316ebab4a9324c4c040291cdd7e0c87bd53f290426d030d` | + +Earlier fresh failures and their logs/receipts remain alongside these records: +archive absence, Windows environment case aliases, restore/build helper-TFM +disagreement and an oversized inventory command line. Their corrections have +deterministic regression coverage; none of those failed runs is relabelled green. +The final scoped suite passed **858 tests**, with no failures or skips. +All 17 changed/new JavaScript files passed syntax checks; repository formatting +changed no files, and whitespace checks passed. + +Commands used (the reproduction output directory must be new when repeating): + +```powershell +node eng\regression-bisect\workspace.smoke.cjs --run 5a8b9adb034f420a5c2c1f577670c5eeeb454b7e 'Q:\fsharp-worktrees\.bare' 'Q:\rb912-s5-20260919-acceptance' 'C:\Program Files\Git\cmd\git.exe' +node eng\regression-bisect\repro.smoke.cjs 'Q:\rb912-s5-20260919-acceptance\verified-products.json' 'Q:\rb912-s5-20260919-acceptance\repro-both-01' +``` + ## Reproduction execution `repro.cjs` is an execution adapter, not an authorization boundary. Its caller @@ -572,11 +680,18 @@ Build the reader with the explicit `MSBuild.StructuredLogger` dependency reads binlogs, and its dependency-free `netstandard2.0` hook can run on older Core runtimes independently of the reader's .NET 8+ host. +The workspace builds it against the acquired runtime with +`-p:EvidenceTargetFramework=net.0`; this property survives restore graph +evaluation, unlike overriding the special `TargetFramework` global property. +The hook retains its own `netstandard2.0` target. Helper restore uses the +repository's public `dotnet-public` mirror, not a machine feed or global cache. The smoke covers FSI, Debug/Release projects, successful compilation followed by an intended runtime failure, intended diagnostics, late compiler flag overrides, explicit/central Core package overrides, two inner TFMs on Windows, and an immutable released -producer. It also covers nullable settings and late overrides, app/library +producer. Source-only negative cases successfully compile with SDK compiler, +tasks, targets or Core fallbacks, then require `wrong-identity` rejection. +It also covers nullable settings and late overrides, app/library references with same/different TFMs, and collectible Core unloading with and without the observation hook. Its source-evaluation control illustrates how rebuilding a producer without its old flags would hide the failure; candidate consumers only receive diff --git a/eng/regression-bisect/archive-provenance.cjs b/eng/regression-bisect/archive-provenance.cjs new file mode 100644 index 00000000000..9af6cecf8bf --- /dev/null +++ b/eng/regression-bisect/archive-provenance.cjs @@ -0,0 +1,52 @@ +const path = require("node:path"); +const { requireValue, publicUrl, validateHttpResponse } = require("./toolchain-io.cjs"); + +function validateArchiveRequest({ policy, kind, sourceSha, product, version, platform, archive }) { + const { validatePolicy, fingerprintInput } = require("./model.cjs"); + validatePolicy(policy); + publicUrl(archive.url, policy); + requireValue(archive.rid === platform.rid && archive.architecture === platform.architecture, + "archive RID/architecture does not match request"); + const provenance = archive.provenance; + if (provenance?.kind === "publisher-checksum") { + requireValue(Object.keys(provenance).join() === "kind" && /^[a-f0-9]{128}$/.test(archive.sha512), + "invalid publisher archive checksum provenance"); + return null; + } + requireValue(provenance?.kind === "approved-official-https-measured" && + Object.keys(provenance).sort().join() === "approvalDigest,approvalRevision,kind" && + !Object.hasOwn(archive, "sha512"), "invalid measured archive provenance"); + const approval = policy.toolchains.measuredArchives.find(a => a.url === archive.url && + a.product === product && a.version === version && a.rid === platform.rid && a.sourceSha === sourceSha); + requireValue(kind === "source" && approval && version.includes("-") && + new URL(archive.url).origin === approval.origin && new URL(archive.url).href === archive.url && + provenance.approvalRevision === approval.approvalRevision && + provenance.approvalDigest === fingerprintInput(approval), "historical archive lacks exact policy approval"); + return approval; +} + +function validateArchiveReceipt(options, receipt) { + const { fingerprintInput } = require("./model.cjs"); + const approval = validateArchiveRequest(options); + const { archive, policy } = options; + requireValue(receipt.url === archive.url && receipt.name === archive.name && + receipt.rid === archive.rid && receipt.architecture === archive.architecture && + fingerprintInput(receipt.provenance) === fingerprintInput(archive.provenance) && + receipt.sha512 === archive.sha512 && /^[a-f0-9]{128}$/.test(receipt.actualSha512) && + /^[a-f0-9]{64}$/.test(receipt.sha256) && + (approval || receipt.actualSha512 === archive.sha512), "installer archive checksum or provenance mismatch"); + validateHttpResponse(archive.url, { url: receipt.redirectUrl, redirects: receipt.redirects }, policy, + approval ? { allowedUrls: [approval.url, ...approval.redirectUrls], maxBytes: approval.maxBytes, timeoutMs: approval.timeoutMs } : undefined); + requireValue(receipt.finalOrigin === new URL(receipt.redirectUrl).origin, "archive final origin mismatch"); + requireValue(Number.isSafeInteger(receipt.bytesReceived) && receipt.bytesReceived > 0 && + receipt.bytesReceived <= (approval?.maxBytes ?? 1073741824) && + typeof receipt.receivedAt === "string" && Number.isFinite(Date.parse(receipt.receivedAt)) && + new Date(receipt.receivedAt).toISOString() === receipt.receivedAt, "invalid archive receipt bounds or time"); + const retained = receipt.retained; + requireValue(retained && path.isAbsolute(retained.path) && retained.size === receipt.bytesReceived && + retained.sha256 === receipt.sha256 && retained.sha512 === receipt.actualSha512, + "retained archive identity mismatch"); + return true; +} + +module.exports = { validateArchiveRequest, validateArchiveReceipt }; diff --git a/eng/regression-bisect/evidence/Evidence.csproj b/eng/regression-bisect/evidence/Evidence.csproj index 09c61e83ebb..2da7db1a3f7 100644 --- a/eng/regression-bisect/evidence/Evidence.csproj +++ b/eng/regression-bisect/evidence/Evidence.csproj @@ -7,7 +7,8 @@ Exe - net8.0 + net8.0 + $(EvidenceTargetFramework) enable enable false diff --git a/eng/regression-bisect/fixtures/checkpoint.txt b/eng/regression-bisect/fixtures/checkpoint.txt index 432d66f8863..fcf2150d2ae 100644 --- a/eng/regression-bisect/fixtures/checkpoint.txt +++ b/eng/regression-bisect/fixtures/checkpoint.txt @@ -1,5 +1,5 @@ Regression investigation: discovery (generation 0). ```json -{"schemaVersion":1,"kind":"checkpoint","status":"Regression investigation: discovery (generation 0).","binding":{"repository":"dotnet/fsharp","issueNumber":123,"commentId":456,"workflowPath":".github/workflows/regression-bisect.yml","workflowRef":"refs/heads/main","workflowRevision":"899531e0f885501b5b8520a9bd95bc83345eaa5b","runId":"12345","runAttempt":1,"keyId":"fixture"},"payload":{"root":["@",0],"table":[{"schemaVersion":1,"generation":0,"previousDigest":"657b8c3151a6f2496623f2e58482895222b1808b30eee9cc032e1431a85a9347","state":["@",1],"inputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a","reproductionDigest":"91815101645c42988cb8f780311ed49cd7c179071d76d17298a1c6810a965138","policyDigest":"328aa42989c22950aea4d267acb68a4849932bbdb8ea4d118604f083e7d20738","sequence":0,"pending":null,"records":["@",30],"observations":["@",34],"history":["@",30],"deadline":"2026-10-01T00:00:00Z","publicationStatus":"unpublished"},{"schemaVersion":1,"repository":"dotnet/fsharp","issueNumber":123,"issueFingerprint":"ba79e2bbb07eaf308944401cb8a62139d3eed051e25c88279ce781531fef37f5","approvalFingerprint":"16503092fdfbb9143595739dc023ca6e201f80203f63950dd884370b09878dcb","manifest":["@",2],"budget":["@",18],"pinnedMainSha":"b28b7af69320201d1cf206ebf28373980add1451","epoch":"09eea1ee5aea50efd7ac23639a5050c834e1824d056cbcc4b054532e8c9a02a3","phase":"discovery","endpointEvidence":["@",32],"testedRevisions":["@",30],"skips":["@",30],"interval":null,"publication":["@",33]},{"schemaVersion":1,"repository":"dotnet/fsharp","issueNumber":123,"discussionFingerprint":"ba79e2bbb07eaf308944401cb8a62139d3eed051e25c88279ce781531fef37f5","kind":"fsi","sources":["@",3],"assertions":["@",5],"matrix":["@",8],"timeoutSeconds":300,"producer":["@",11],"packages":["@",12],"searchHints":["@",14],"approvedInputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a"},[["@",4]],{"path":"fixtures/repro.fsx","sha256":"e1c5329a6b6f65fb0733f62c334e0bf4bb91cfa449f5eb213dc67f0d40b440af"},{"pass":["@",6],"failure":["@",7]},{"exitCode":0,"stdout":"ok\n","stderr":""},{"exitCode":1,"stdout":"","stderr":"expected regression\n"},[["@",9]],{"id":"linux-x64","os":"linux","architecture":"x64","project":null,"tfm":"net10.0","configuration":"Release","languageVersion":"10.0","command":["@",10]},["fsi","fixtures/repro.fsx"],{"name":"repro","version":"1.0.0","sha256":"d1990f97a1f4fe7b8e86da8c03c9dd3f7d536c14aa4a6f9eedab07f34cd961b0"},[["@",13]],{"id":"Example","version":"1.0.0","sha256":"bc4a71180870f7945155fbb02f4b0a2e3faa2a62d6d31b7039013055ed19869a"},{"sdkVersions":["@",15],"sourceShas":["@",16],"urls":["@",17]},["9.0.100"],["fc19318dd13128ce14344d066510a982269c241b"],["https://github.com/dotnet/fsharp"],{"schemaVersion":1,"policy":["@",19],"policyRevision":"9f00fad98badc1330e637a0a015d704113498420","workflowRevision":"899531e0f885501b5b8520a9bd95bc83345eaa5b","allowanceId":"maintainer-approved-investigation-1","authorization":["@",26],"stableSdk":["@",27],"authenticatedObservationDigests":["@",30],"consumed":["@",31],"attempts":["@",30],"nextWorkItemId":"none","pendingWork":null},{"schemaVersion":1,"repository":"dotnet/fsharp","selection":["@",20],"authorization":["@",21],"accounting":["@",22],"limits":["@",23],"evidenceBytesPerStream":16384,"toolchains":["@",24]},{"state":"open","label":"Regression","createdAtOrAfter":"2026-09-01T00:00:00Z"},{"mode":"reviewed-manifests-only","policySource":"pinned-trusted-default-branch","approvalDirectory":".github/regression-bisect/approved","assets":"reviewed-repository-paths-and-sha256"},{"scope":"issue-allowance","reserveBeforeExecution":true,"retriesCharged":true,"cancellationRefunds":false,"inputEditsResetAllowance":false,"labelChangesResetAllowance":false,"dispatchResetsAllowance":false,"newAllowanceRequires":"maintainer-authorization"},{"sourceBuilds":24,"workerUnits":40,"reservedMinutes":720,"attemptsPerItem":2,"assertionRepetitions":2,"buildTimeoutMinutes":90,"reproductionTimeoutSeconds":300,"workerTimeoutMinutes":110,"discoveryIssues":3,"discoveryPages":2,"discoveryPageSize":100},{"publicFeeds":["@",25]},["https://builds.dotnet.microsoft.com/dotnet","https://ci.dot.net/public","https://dotnetcli.blob.core.windows.net/dotnet","https://download.visualstudio.microsoft.com/download/pr"],{"repository":"dotnet/fsharp","issueNumber":123,"approvalPath":".github/regression-bisect/approved/123.json","approvalRevision":"9f00fad98badc1330e637a0a015d704113498420","manifestDigest":"16503092fdfbb9143595739dc023ca6e201f80203f63950dd884370b09878dcb","inputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a"},{"version":"10.0.100","origin":"official-release-index","latest":true,"verified":true,"releaseIndexUrl":"https://builds.dotnet.microsoft.com/dotnet/release-metadata/releases-index.json","releaseIndexDigest":"2a21ef65c47867ae1b2056a246bf7c001b12ade326bd0e1fcdfb81af140e65b7","archives":["@",28]},[["@",29]],{"os":"linux","architecture":"x64","sha256":"9d9e20df50a502c04ee629c1282495a65a279635168510572f846de604da046e"},[],{"sourceBuilds":0,"workerUnits":0,"reservedMinutes":0},{"stable":null,"main":null,"good":null},{"digest":null,"commentId":null},{}]},"signature":"f271b6c7deff0f7c41a118baf7a71f0bb43aa32bf35b7347b01c3710f88150a2"} +{"schemaVersion":1,"kind":"checkpoint","status":"Regression investigation: discovery (generation 0).","binding":{"repository":"dotnet/fsharp","issueNumber":123,"commentId":456,"workflowPath":".github/workflows/regression-bisect.yml","workflowRef":"refs/heads/main","workflowRevision":"899531e0f885501b5b8520a9bd95bc83345eaa5b","runId":"12345","runAttempt":1,"keyId":"fixture"},"payload":{"root":["@",0],"table":[{"schemaVersion":1,"generation":0,"previousDigest":"657b8c3151a6f2496623f2e58482895222b1808b30eee9cc032e1431a85a9347","state":["@",1],"inputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a","reproductionDigest":"91815101645c42988cb8f780311ed49cd7c179071d76d17298a1c6810a965138","policyDigest":"e4469202d56c8ad7fbe92d7b4eb4940ace563a83cc06f31da7937446c7e74f74","sequence":0,"pending":null,"records":["@",28],"observations":["@",37],"history":["@",28],"deadline":"2026-10-01T00:00:00Z","publicationStatus":"unpublished"},{"schemaVersion":1,"repository":"dotnet/fsharp","issueNumber":123,"issueFingerprint":"ba79e2bbb07eaf308944401cb8a62139d3eed051e25c88279ce781531fef37f5","approvalFingerprint":"16503092fdfbb9143595739dc023ca6e201f80203f63950dd884370b09878dcb","manifest":["@",2],"budget":["@",18],"pinnedMainSha":"b28b7af69320201d1cf206ebf28373980add1451","epoch":"09eea1ee5aea50efd7ac23639a5050c834e1824d056cbcc4b054532e8c9a02a3","phase":"discovery","endpointEvidence":["@",35],"testedRevisions":["@",28],"skips":["@",28],"interval":null,"publication":["@",36]},{"schemaVersion":1,"repository":"dotnet/fsharp","issueNumber":123,"discussionFingerprint":"ba79e2bbb07eaf308944401cb8a62139d3eed051e25c88279ce781531fef37f5","kind":"fsi","sources":["@",3],"assertions":["@",5],"matrix":["@",8],"timeoutSeconds":300,"producer":["@",11],"packages":["@",12],"searchHints":["@",14],"approvedInputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a"},[["@",4]],{"path":"fixtures/repro.fsx","sha256":"e1c5329a6b6f65fb0733f62c334e0bf4bb91cfa449f5eb213dc67f0d40b440af"},{"pass":["@",6],"failure":["@",7]},{"exitCode":0,"stdout":"ok\n","stderr":""},{"exitCode":1,"stdout":"","stderr":"expected regression\n"},[["@",9]],{"id":"linux-x64","os":"linux","architecture":"x64","project":null,"tfm":"net10.0","configuration":"Release","languageVersion":"10.0","command":["@",10]},["fsi","fixtures/repro.fsx"],{"name":"repro","version":"1.0.0","sha256":"d1990f97a1f4fe7b8e86da8c03c9dd3f7d536c14aa4a6f9eedab07f34cd961b0"},[["@",13]],{"id":"Example","version":"1.0.0","sha256":"bc4a71180870f7945155fbb02f4b0a2e3faa2a62d6d31b7039013055ed19869a"},{"sdkVersions":["@",15],"sourceShas":["@",16],"urls":["@",17]},["9.0.100"],["fc19318dd13128ce14344d066510a982269c241b"],["https://github.com/dotnet/fsharp"],{"schemaVersion":1,"policy":["@",19],"policyRevision":"9f00fad98badc1330e637a0a015d704113498420","workflowRevision":"899531e0f885501b5b8520a9bd95bc83345eaa5b","allowanceId":"maintainer-approved-investigation-1","authorization":["@",30],"stableSdk":["@",31],"authenticatedObservationDigests":["@",28],"consumed":["@",34],"attempts":["@",28],"nextWorkItemId":"none","pendingWork":null},{"schemaVersion":1,"repository":"dotnet/fsharp","selection":["@",20],"authorization":["@",21],"accounting":["@",22],"limits":["@",23],"evidenceBytesPerStream":16384,"toolchains":["@",24]},{"state":"open","label":"Regression","createdAtOrAfter":"2026-09-01T00:00:00Z"},{"mode":"reviewed-manifests-only","policySource":"pinned-trusted-default-branch","approvalDirectory":".github/regression-bisect/approved","assets":"reviewed-repository-paths-and-sha256"},{"scope":"issue-allowance","reserveBeforeExecution":true,"retriesCharged":true,"cancellationRefunds":false,"inputEditsResetAllowance":false,"labelChangesResetAllowance":false,"dispatchResetsAllowance":false,"newAllowanceRequires":"maintainer-authorization"},{"sourceBuilds":24,"workerUnits":40,"reservedMinutes":720,"attemptsPerItem":2,"assertionRepetitions":2,"buildTimeoutMinutes":90,"reproductionTimeoutSeconds":300,"workerTimeoutMinutes":110,"discoveryIssues":3,"discoveryPages":2,"discoveryPageSize":100},{"publicFeeds":["@",25],"measuredArchives":["@",26]},["https://builds.dotnet.microsoft.com/dotnet","https://ci.dot.net/public","https://dotnetcli.blob.core.windows.net/dotnet","https://download.visualstudio.microsoft.com/download/pr"],[["@",27],["@",29]],{"sourceSha":"5a8b9adb034f420a5c2c1f577670c5eeeb454b7e","product":"sdk","version":"11.0.100-rc.1.26420.103","rid":"win-x64","origin":"https://builds.dotnet.microsoft.com","url":"https://builds.dotnet.microsoft.com/dotnet/Sdk/11.0.100-rc.1.26420.103/dotnet-sdk-11.0.100-rc.1.26420.103-win-x64.zip","redirectUrls":["@",28],"maxBytes":1073741824,"timeoutMs":300000,"approvalRevision":"2ec6de6837b5b367e1f1186e850b8faa2d188b16"},[],{"sourceSha":"5a8b9adb034f420a5c2c1f577670c5eeeb454b7e","product":"sdk","version":"11.0.100-rc.1.26420.103","rid":"win-x64","origin":"https://ci.dot.net","url":"https://ci.dot.net/public/Sdk/11.0.100-rc.1.26420.103/dotnet-sdk-11.0.100-rc.1.26420.103-win-x64.zip","redirectUrls":["@",28],"maxBytes":1073741824,"timeoutMs":300000,"approvalRevision":"2ec6de6837b5b367e1f1186e850b8faa2d188b16"},{"repository":"dotnet/fsharp","issueNumber":123,"approvalPath":".github/regression-bisect/approved/123.json","approvalRevision":"9f00fad98badc1330e637a0a015d704113498420","manifestDigest":"16503092fdfbb9143595739dc023ca6e201f80203f63950dd884370b09878dcb","inputDigest":"58d385e0f0da55da5a3bc8bc212e1e8c4edc2a983b9682a0693e909427fa900a"},{"version":"10.0.100","origin":"official-release-index","latest":true,"verified":true,"releaseIndexUrl":"https://builds.dotnet.microsoft.com/dotnet/release-metadata/releases-index.json","releaseIndexDigest":"2a21ef65c47867ae1b2056a246bf7c001b12ade326bd0e1fcdfb81af140e65b7","archives":["@",32]},[["@",33]],{"os":"linux","architecture":"x64","sha256":"9d9e20df50a502c04ee629c1282495a65a279635168510572f846de604da046e"},{"sourceBuilds":0,"workerUnits":0,"reservedMinutes":0},{"stable":null,"main":null,"good":null},{"digest":null,"commentId":null},{}]},"signature":"dcdabd13125fa4aa1ee928e3a9a328a8881ce31c4b96e249bcff17e75fc5fcdd"} ``` diff --git a/eng/regression-bisect/fixtures/contracts.cjs b/eng/regression-bisect/fixtures/contracts.cjs index 94dc798288f..56bf59e8a43 100644 --- a/eng/regression-bisect/fixtures/contracts.cjs +++ b/eng/regression-bisect/fixtures/contracts.cjs @@ -2,6 +2,13 @@ const { createHash } = require("node:crypto"); const policy = () => structuredClone(require("../../../.github/policies/regression-bisect.json")); const digest = text => createHash("sha256").update(text).digest("hex"); const sha = text => createHash("sha1").update(text).digest("hex"); +function archiveReceipt(archive, file, bytes) { + const sha512 = createHash("sha512").update(bytes).digest("hex"); + return { ...archive, sha256: digest(bytes), actualSha512: sha512, redirectUrl: archive.url, + finalOrigin: new URL(archive.url).origin, redirects: [], + bytesReceived: Buffer.byteLength(bytes), receivedAt: "2026-09-19T17:30:00.000Z", + retained: { path: file, sha256: digest(bytes), sha512, size: Buffer.byteLength(bytes) } }; +} const model = () => require("../model.cjs"); const seal = m => { const { approvedInputDigest, ...input } = m; @@ -32,7 +39,7 @@ function manifest() { function attempt(outcome) { return { schemaVersion: 1, - checks: { restore: true, build: true, setup: true, identity: true, sdk: true, runtime: true }, + checks: { restore: true, build: true, setup: true, identity: true, sdk: true, runtime: true, checksumProvenance: true }, execution: { status: "completed", complete: true, ...structuredClone(manifest().assertions[outcome === "pass" ? "pass" : "failure"]), @@ -133,4 +140,4 @@ function statusTransport(context) { }; } -module.exports = { policy, manifest, attempt, observation, admission, state, seal, sha, digest, statusTransport }; +module.exports = { policy, manifest, attempt, observation, admission, state, seal, sha, digest, statusTransport, archiveReceipt }; diff --git a/eng/regression-bisect/fixtures/reproduction.cjs b/eng/regression-bisect/fixtures/reproduction.cjs index 0692c5a6977..e5f2c622f4b 100644 --- a/eng/regression-bisect/fixtures/reproduction.cjs +++ b/eng/regression-bisect/fixtures/reproduction.cjs @@ -2,7 +2,7 @@ const fs = require("node:fs/promises"); const os = require("node:os"); const path = require("node:path"); const { hash } = require("../toolchain-io.cjs"); -const { manifest, seal } = require("./contracts.cjs"); +const { manifest, seal, archiveReceipt } = require("./contracts.cjs"); async function fixture(t, { kind = "source", executable = false, tfms = ["net10.0"], configuration = "Release" } = {}) { const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "repro-contract-"))); @@ -54,7 +54,9 @@ async function fixture(t, { kind = "source", executable = false, tfms = ["net10. path: path.dirname(path.dirname(runtime.path)), sha256: runtime.sha256 }], }; const url = "https://builds.dotnet.microsoft.com/dotnet/Sdk/10.0.100/dotnet-sdk-10.0.100-win-x64.zip"; - toolchain.archive = { url, redirectUrl: url, sha512: "a".repeat(128), actualSha512: "a".repeat(128), sha256: hash("archive") }; + toolchain.archive = archiveReceipt({ name: "sdk.zip", url, rid: "win-x64", architecture: "x64", + provenance: { kind: "publisher-checksum" } }, (await file("sdk.zip", "archive")).path, "archive"); + toolchain.archive.sha512 = toolchain.archive.actualSha512; toolchain.requested = { kind: "released", platform: { os: "windows", architecture: "x64", rid: "win-x64" }, installationRoot: sdkRoot, sdkVersion: "10.0.100", toolsVersion: "10.0.100", diff --git a/eng/regression-bisect/fixtures/smoke.cjs b/eng/regression-bisect/fixtures/smoke.cjs index a8b9abbfd57..19ff94901ce 100644 --- a/eng/regression-bisect/fixtures/smoke.cjs +++ b/eng/regression-bisect/fixtures/smoke.cjs @@ -16,6 +16,7 @@ async function createFixture(directory, { kind, tfm, configuration = "Release", "late-nullable-explicit": ["enable", "disable"], }[scenario]; const projectTfm = scenario === "reference-tfm" ? `${tfm}-windows7.0` : tfm; + const sdkFSharp = "$(MSBuildSDKsPath)/../FSharp"; const producerSource = [ "#if OLD_PRODUCER", 'let inline value () = "OLD_METADATA"', @@ -81,12 +82,17 @@ let main _ = ${nullable ? `${nullable[0]}` : ""} ${scenario === "late-nullable-explicit" ? "$(DefineConstants);NULLABLE" : ""} ${scenario.startsWith("core-") ? "true" : ""} + ${scenario === "fallback-compiler" ? `${sdkFSharp}/fsc.dll${sdkFSharp}/fsc.dll` : ""} + ${scenario === "fallback-tasks" ? `${sdkFSharp}/FSharp.Build.dll` : ""} + ${scenario === "fallback-targets" ? `${sdkFSharp}/Microsoft.FSharp.Targets${sdkFSharp}/Microsoft.FSharp.NetSdk.targets${sdkFSharp}/Microsoft.FSharp.Overrides.NetSdk.targets` : ""} + ${scenario === "fallback-core" ? "true" : ""} ${scenario === "producer" ? "FrozenProducerFrozenProducer1.0.0$(DefineConstants);OLD_PRODUCERtrue" : ""} ${scenario === "core-explicit" ? `` : ""} ${scenario === "core-central" ? '' : ""} + ${scenario === "fallback-core" ? `${sdkFSharp}/FSharp.Core.dll` : ""} ${producer ? 'producer/FrozenProducer.dll' : ""} ${referenced ? '' : ""} diff --git a/eng/regression-bisect/model.cjs b/eng/regression-bisect/model.cjs index dc53bb5d7c2..2852a6a91c5 100644 --- a/eng/regression-bisect/model.cjs +++ b/eng/regression-bisect/model.cjs @@ -158,6 +158,7 @@ function validateManifest(manifest) { const untestableReasons = [ "restore-failure", "build-failure", "setup-failure", "wrong-identity", "unavailable-sdk", "unavailable-runtime", + "checksum-provenance", "network-failure", "timeout", "incomplete-evidence", "flaky", "unrelated-exception", "assertion-mismatch", ]; function validateOutcome(outcome) { @@ -173,6 +174,7 @@ function validateOutcome(outcome) { const outcome = (value, reason) => ({ schemaVersion: 1, outcome: value, reason }); const untestable = reason => outcome("untestable", reason); const checks = { + checksumProvenance: "checksum-provenance", restore: "restore-failure", build: "build-failure", setup: "setup-failure", identity: "wrong-identity", sdk: "unavailable-sdk", runtime: "unavailable-runtime", }; @@ -229,10 +231,21 @@ const artifact = shape({ sha256: digest, origin: oneOf(["local-build", "released-sdk"]), sourceSha: nullable(sourceSha), buildInstanceId: text, }); +const archiveReceipt = (v, p) => object(v, { + name: text, url: httpsUrl, rid: text, architecture, + provenance: (v, p) => object(v, v?.kind === "publisher-checksum" + ? { kind: literal("publisher-checksum") } + : { kind: literal("approved-official-https-measured"), approvalRevision: sourceSha, approvalDigest: digest }, p), + ...(v?.provenance?.kind === "publisher-checksum" ? { sha512: matching(/^[a-f0-9]{128}$/) } : {}), + sha256: digest, actualSha512: matching(/^[a-f0-9]{128}$/), redirectUrl: httpsUrl, finalOrigin: httpsUrl, + redirects: list(shape({ from: httpsUrl, to: httpsUrl, status: oneOf([301, 302, 303, 307, 308]) }), 0, 5), + bytesReceived: positive, receivedAt: (v, p) => check(validTimestamp(v), p, "invalid receipt time"), + retained: shape({ path: text, sha256: digest, sha512: matching(/^[a-f0-9]{128}$/), size: positive }), +}, p); const buildToolchain = shape({ sourceSha, globalJsonDigest: digest, evidenceDigest: digest, requestedSdk: pinnedVersion, requestedTools: pinnedVersion, requestedCompilerRuntime: pinnedVersion, - sdk: productIdentity, compilerRuntime: productIdentity, + sdk: productIdentity, compilerRuntime: productIdentity, archive: archiveReceipt, verified: literal(true), supported: literal(true), }); const observationCell = (v, p) => object(v, { @@ -278,6 +291,17 @@ function validateObservation(observation, manifest, options = { finalConfirmatio build.requestedSdk === build.sdk.version && build.requestedTools === build.sdk.version && build.requestedCompilerRuntime === build.compilerRuntime.version, "observation.cells.buildToolchain", "wrong source declaration or resolved build SDK"); + const { validateArchiveReceipt } = require("./archive-provenance.cjs"); + try { + validateArchiveReceipt({ + policy: trustedPolicy, kind: observation.kind, sourceSha: build.sourceSha, product: "sdk", + version: build.requestedSdk, platform: { ...cell.environment, rid: build.archive.rid }, archive: build.archive, + }, build.archive); + } catch (error) { throw new ContractError(`observation.cells.buildToolchain.archive: ${error.message}`); } + const rid = `${{ windows: "win", linux: "linux", macos: "osx" }[cell.environment.os]}-${cell.environment.architecture}`; + check(build.archive.sha256 === build.sdk.sha256 && build.archive.architecture === cell.environment.architecture && + (build.archive.rid === rid || cell.environment.os === "linux" && build.archive.rid === `linux-musl-${cell.environment.architecture}`), + "observation.cells.buildToolchain.archive", "archive does not match build SDK or platform"); } for (const [name, missing] of Object.entries({ sdk: cell.sdk === null, runtime: cell.runtime === null, diff --git a/eng/regression-bisect/model.test.cjs b/eng/regression-bisect/model.test.cjs index 8ea28974a54..04f2c43fff1 100644 --- a/eng/regression-bisect/model.test.cjs +++ b/eng/regression-bisect/model.test.cjs @@ -4,10 +4,71 @@ const { isEligibleIssue, validatePolicy, validateManifest, fingerprintInput, fingerprintEpoch, classifyAttempts, validateOutcome, validateObservation, validateState, admitBisection, } = require("./model.cjs"); -const { policy, manifest, attempt, observation, admission, state, seal, sha, digest } = require("./fixtures/contracts.cjs"); +const { policy, manifest, attempt, observation, admission, state, seal, sha, digest, archiveReceipt } = require("./fixtures/contracts.cjs"); +const path = require("node:path"); const rejects = (fn, pattern = /./) => assert.throws(fn, pattern); +for (const kind of ["publisher-checksum", "approved-official-https-measured"]) { + test(`observation serializes and validates ${kind} without relabelling hashes`, () => { + const approval = policy().toolchains.measuredArchives[0]; + const m = manifest(); + m.matrix[0].os = "windows"; + seal(m); + const o = observation("source", "pass", m, approval.sourceSha); + const archive = archiveReceipt({ + name: path.basename(new URL(approval.url).pathname), url: approval.url, rid: approval.rid, architecture: "x64", + provenance: kind === "publisher-checksum" ? { kind } : + { kind, approvalRevision: approval.approvalRevision, approvalDigest: fingerprintInput(approval) }, + }, path.resolve("retained-sdk.zip"), "archive"); + if (kind === "publisher-checksum") archive.sha512 = archive.actualSha512; + o.cells[0].buildToolchain = { + sourceSha: approval.sourceSha, globalJsonDigest: digest("global"), evidenceDigest: digest("receipt"), + requestedSdk: approval.version, requestedTools: approval.version, requestedCompilerRuntime: "11.0.0", + sdk: { version: approval.version, sha256: archive.sha256, verified: true }, + compilerRuntime: { version: "11.0.0", sha256: digest("runtime"), verified: true }, + archive, verified: true, supported: true, + }; + assert.equal(validateObservation(JSON.parse(JSON.stringify(o)), m), true); + for (const change of [ + a => { a.provenance.kind = "unknown"; }, + a => { a.provenance.approvalDigest = digest("forged"); }, + a => { a.retained.sha512 = "0".repeat(128); }, + a => { a.bytesReceived++; }, + a => { a.url = a.url.replace("26420", "26421"); }, + a => { a.redirects.push({ from: a.url, to: `${a.url}?other`, status: 302 }); }, + a => { a.architecture = "arm64"; }, + ]) { + const changed = structuredClone(o); + change(changed.cells[0].buildToolchain.archive); + rejects(() => validateObservation(changed, m)); + } + if (kind === "approved-official-https-measured") { + archive.sha512 = archive.actualSha512; + rejects(() => validateObservation(o, m), /missing or unknown fields/); + } + }); +} + +test("checksum provenance is a blocked outcome, not missing SDK or a passing attempt", () => { + const m = manifest(), a = attempt("pass"); + a.checks.checksumProvenance = false; + const blocked = { schemaVersion: 1, outcome: "untestable", reason: "checksum-provenance" }; + assert.deepEqual(classifyAttempts([a, a], m), blocked); + const o = observation("source", "pass", m); + const cell = o.cells[0]; + cell.sdk = cell.runtime = null; + cell.artifacts = Object.fromEntries(Object.keys(cell.artifacts).map(name => [name, null])); + a.checks = Object.fromEntries(Object.keys(a.checks).map(name => [name, false])); + a.execution = { status: "exception", complete: false, exitCode: null, stdout: "", stderr: "checksum provenance absent" }; + cell.attempts = [a, a]; + cell.outcome = o.outcome = blocked; + assert.deepEqual(classifyAttempts(cell.attempts, m), blocked); + assert.equal(validateObservation(JSON.parse(JSON.stringify(o)), m), true); + assert.equal(validateOutcome({ schemaVersion: 1, outcome: "untestable", reason: "network-failure" }), true); + rejects(() => validateOutcome({ schemaVersion: 1, outcome: "pass", reason: "checksum-provenance" })); +}); + for (const [name, patch, expected] of [ ["before cutoff", { created_at: "2026-08-31T23:59:59.999Z" }, false], ["at cutoff", {}, true], diff --git a/eng/regression-bisect/repro.cjs b/eng/regression-bisect/repro.cjs index fbf60553f09..96aa230f34b 100644 --- a/eng/regression-bisect/repro.cjs +++ b/eng/regression-bisect/repro.cjs @@ -3,6 +3,7 @@ const path = require("node:path"); const { execute } = require("./repro-process.cjs"); const { validateManifest, classifyAttempts, fingerprintInput } = require("./model.cjs"); const { verifyToolchainIdentity } = require("./toolchains.cjs"); +const { verifyRetainedArchive } = require("./toolchain-installer.cjs"); const { hash, inside, requireValue, isolatedEnvironment } = require("./toolchain-io.cjs"); const policy = require("../../.github/policies/regression-bisect.json"); @@ -76,7 +77,7 @@ async function selectFsiCommand({ product, toolchain, cell }) { const compileAssertion = manifest => /(?:^|\n)[^\n]+\(\d+,\d+(?:,\d+,\d+)?\): error FS\d{4}: [^\n]+/.test(manifest.assertions.failure.stderr); const attemptFor = ({ status, complete, exitCode, stdout, stderr }) => ({ - schemaVersion: 1, checks: { restore: true, build: true, setup: true, identity: true, sdk: true, runtime: true }, + schemaVersion: 1, checks: { restore: true, build: true, setup: true, identity: true, sdk: true, runtime: true, checksumProvenance: true }, execution: { status, complete, exitCode, stdout, stderr }, }); @@ -200,7 +201,7 @@ async function invokeHelper(helper, args, settings, run) { async function inspectProduct(product, inspect) { for (const file of product.files) await checkedFile(file, product.root); - const files = product.files.filter(f => /\.(dll|exe|nupkg)$/i.test(f.path)); + const files = product.files.filter(f => ["fsi", "compiler", "core", "tasks", "corePackage"].includes(f.role)); const inspected = await inspect(files.map(f => f.path)); for (const file of files.filter(f => f.assemblyVersion)) { const actual = inspected.find(f => samePath(f.path, file.path)); @@ -352,8 +353,12 @@ async function runReproduction(options, dependencies = {}) { const { manifest, toolchain, helper } = options; validateManifest(manifest); await verifySources(manifest, options.sourceDirectory); + stage = "identity"; const verification = verifyToolchainIdentity({ policy, toolchain: toolchain.requested, identity: toolchain }); requireValue(verification.status === "verified", verification.error?.message ?? "unverified acquisition"); + await verifyRetainedArchive(toolchain.archive); + for (const installation of toolchain.installations ?? []) + if (installation.product !== "sdk") await verifyRetainedArchive(installation.archive); await checkedFile({ path: toolchain.hostPath, sha256: toolchain.hostSha256 }, toolchain.root); requireValue(["source", "released"].includes(options.product.kind) && options.product.buildInstanceId, "unidentified product build"); if (options.product.kind === "source") requireValue(/^[0-9a-f]{40}$/.test(options.product.sourceSha) && @@ -484,8 +489,8 @@ async function runReproduction(options, dependencies = {}) { assetsFile: path.resolve(path.dirname(row.project), row.assetsFile), targetPath: path.resolve(path.dirname(row.project), row.targetPath), })); - for (const row of rows) row.core = await coreAssets(row, attemptEnv.NUGET_PACKAGES, product, manifest); stage = "identity"; + for (const row of rows) row.core = await coreAssets(row, attemptEnv.NUGET_PACKAGES, product, manifest); const cellManifest = expectedBuildManifest(manifest, cell, rows, directory); const verified = await verifyProjectIdentities({ manifest: cellManifest, product, toolchain, sourceDirectory: directory, diff --git a/eng/regression-bisect/repro.smoke.cjs b/eng/regression-bisect/repro.smoke.cjs index b6bb4fbce48..33a7f932d83 100644 --- a/eng/regression-bisect/repro.smoke.cjs +++ b/eng/regression-bisect/repro.smoke.cjs @@ -71,6 +71,7 @@ async function main() { ...(process.platform === "win32" ? [["project", "reference-tfm", "Release"]] : []), ["project", "producer-consumer", "Release"], ...(process.platform === "win32" ? [["project", "multi-tfm", "Release"]] : []), + ...(name === "source" ? ["compiler", "tasks", "targets", "core"].map(role => ["project", `fallback-${role}`, "Release"]) : []), ["fsi", "producer-control", "Release"], ]) { const id = `${name}-${kind}-${scenario}-${configuration}`; @@ -86,7 +87,8 @@ async function main() { sampleRuntimes: input.sampleRuntimes, }); const lateOverride = scenario.startsWith("late-") || scenario === "reference-late"; - const expected = lateOverride ? "untestable" + const fallback = scenario.startsWith("fallback-"); + const expected = lateOverride || fallback ? "untestable" : ["runtime-failure", "producer-consumer", "diagnostic"].includes(scenario) ? "product-failure" : "pass"; results.push({ id, expected, result }); await fs.writeFile(path.join(root, "smoke-results.json"), JSON.stringify(results, null, 2)); @@ -94,6 +96,9 @@ async function main() { requireValue(result.outcome.outcome === expected, `bounded smoke failed: ${id}`); if (lateOverride) requireValue(result.outcome.reason === "wrong-identity" && result.error === "late compiler flag override", "late override must be rejected for the intended identity mismatch"); + if (fallback) requireValue(result.outcome.reason === "wrong-identity" && + result.pending?.evidence.build?.exitCode === 0, + "fallback must compile successfully before product identity rejects it"); if (scenario === "collectible-core") { const record = result.cells[0].evidence[0]; const invocation = record.commands.at(-1); diff --git a/eng/regression-bisect/repro.test.cjs b/eng/regression-bisect/repro.test.cjs index 582f981f01e..221f536571b 100644 --- a/eng/regression-bisect/repro.test.cjs +++ b/eng/regression-bisect/repro.test.cjs @@ -24,6 +24,24 @@ for (const [name, options] of [ }); } +for (const [changed, installations] of [[false, undefined], [true, undefined], [false, []], [false, "runtime-only"]]) { + test(`reproduction refuses ${changed ? "changed" : "missing"} retained toolchain bytes with ${JSON.stringify(installations)}`, async t => { + const f = await fixture(t); + const archive = f.toolchain.archive.retained.path; + if (changed) await fs.appendFile(archive, "changed"); + else await fs.rm(archive); + if (installations !== undefined) f.toolchain.installations = installations === "runtime-only" + ? [{ product: "dotnet", version: "10.0.0", archive: { ...f.toolchain.archive, + retained: { ...f.toolchain.archive.retained, path: (await f.file("runtime.zip", "archive")).path } } }] + : installations; + const result = await runReproduction({ ...f, workDirectory: path.join(f.root, "executions") }, + { execute: () => assert.fail("invalid cached acquisition must not execute") }); + assert.equal(result.outcome.outcome, "untestable"); + assert.equal(result.outcome.reason, "wrong-identity"); + assert.equal(result.workDirectory, undefined, "missing retained SDK must fail before creating an execution workspace"); + }); +} + for (const [name, mutate, options] of [ ["wrong host", f => { f.toolchain.hostPath = path.join(f.root, "dotnet-other", "dotnet.exe"); }], ["host dotnet fsi", f => { f.product.files.find(x => x.role === "fsi").path = "fsi"; }], @@ -312,6 +330,12 @@ for (const [name, options, error] of [ ["library Core fallback", { mutate: (s, f) => { s.cells[1].fscTasks[0].references = [f.product.files.find(p => p.role === "core").path]; } }, /Fsc did not reference selected Core/], + ["bare Core reference after successful compilation", { mutate: async s => { + const file = s.cells[0].assetsFile; + const assets = JSON.parse(await fs.readFile(file)); + assets.libraries = {}; + await fs.writeFile(file, JSON.stringify(assets)); + } }, /Core not resolved as expected package/], ]) { test(`project runner verifies expanded matrix: ${name}`, async t => { const f = await projectRunnerFixture(t, options); diff --git a/eng/regression-bisect/toolchain-installer.cjs b/eng/regression-bisect/toolchain-installer.cjs index bbe226840a1..66ee0c15a73 100644 --- a/eng/regression-bisect/toolchain-installer.cjs +++ b/eng/regression-bisect/toolchain-installer.cjs @@ -3,7 +3,10 @@ const crypto = require('node:crypto'); const http = require('node:http'); const path = require('node:path'); +const { Transform } = require('node:stream'); const { pipeline } = require('node:stream/promises'); +const { validateArchiveRequest, validateArchiveReceipt } = require('./archive-provenance.cjs'); +const { publicUrl, validateHttpResponse } = require('./toolchain-io.cjs'); const marker = 'INSTALLER_HELPER_RESULT='; const helperPS = ` @@ -42,26 +45,46 @@ printf 'INSTALLER_HELPER_RESULT=%s\\n' "$_GetDotNetInstallScript" const hash = (bytes, algorithm) => crypto.createHash(algorithm).update(bytes).digest('hex'); -async function hashFile(fs, filename) { +async function hashFile(fs, filename, maxBytes = 1073741824) { + const before = await fs.lstat(filename); + if (!before.isFile() || before.isSymbolicLink() || before.size > maxBytes) + throw new Error('Archive must be a bounded regular nonlink file'); const sha256 = crypto.createHash('sha256'); const sha512 = crypto.createHash('sha512'); + let size = 0; const file = await fs.open(filename, 'r'); try { for await (const chunk of file.createReadStream({ autoClose: false })) { + size += chunk.length; + if (size > maxBytes) throw new Error('Archive exceeds byte limit'); sha256.update(chunk); sha512.update(chunk); } - return { sha256: sha256.digest('hex'), sha512: sha512.digest('hex') }; + const after = await fs.lstat(filename); + if (!after.isFile() || after.isSymbolicLink() || size !== before.size || + ['size', 'ino', 'mtimeMs'].some(key => before[key] !== after[key])) + throw new Error('Archive changed while hashing bytes'); + return { sha256: sha256.digest('hex'), sha512: sha512.digest('hex'), size }; } finally { await file.close(); } } -function publicUrl(value) { - const url = new URL(value); - if (url.protocol !== 'https:' || url.username || url.password || url.hash) - throw new Error('Acquisition requires a public HTTPS URL without credentials or fragment'); - return url; +async function verifyRetainedArchive(archive, deps = {}) { + const fs = deps.fs || require('node:fs/promises'); + const retained = archive?.retained; + if (!retained || typeof retained.path !== 'string' || !path.isAbsolute(retained.path) || /[\r\n\0]/.test(retained.path) || + !/^[0-9a-f]{64}$/.test(retained.sha256) || !/^[0-9a-f]{128}$/.test(retained.sha512) || + !Number.isSafeInteger(retained.size) || retained.size <= 0 || + archive.sha256 !== retained.sha256 || archive.actualSha512 !== retained.sha512 || archive.bytesReceived !== retained.size || + (Object.hasOwn(archive, 'sha512') && archive.sha512.toLowerCase() !== retained.sha512)) + throw new Error('Retained archive receipt fields do not match'); + if (await fs.realpath(retained.path) !== path.resolve(retained.path)) + throw new Error('Retained archive path must not contain links'); + const actual = await hashFile(fs, retained.path, retained.size); + if (actual.sha256 !== retained.sha256 || actual.sha512 !== retained.sha512 || actual.size !== retained.size) + throw new Error('Retained archive digest or size does not match verified download'); + return actual; } function isolatedEnvironment(env, windows) { @@ -71,15 +94,20 @@ function isolatedEnvironment(env, windows) { 'TEMP', 'TMP', 'TMPDIR', 'DOTNET_CLI_HOME', 'NUGET_PACKAGES', 'LANG', 'LC_ALL', ]); const result = {}; - for (const [key, value] of Object.entries(env)) - if (allowed.has(key.toUpperCase()) && typeof value === 'string') result[key] = value; + for (const [key, value] of Object.entries(env)) { + if (!allowed.has(key.toUpperCase()) || typeof value !== 'string') continue; + const name = windows ? (key.toUpperCase() === 'SYSTEMROOT' ? 'SystemRoot' : key.toUpperCase()) : key; + if (Object.hasOwn(result, name) && result[name] !== value) throw new Error('Conflicting Windows environment aliases'); + result[name] = value; + } if (windows) { const systemRoot = Object.entries(result).find(([key]) => key.toUpperCase() === 'SYSTEMROOT')?.[1]; if (!systemRoot || !path.win32.isAbsolute(systemRoot)) throw new Error('Absolute env.SystemRoot is required on Windows'); result.SystemRoot = systemRoot; result.PSModulePath = path.win32.join(systemRoot, 'System32', 'WindowsPowerShell', 'v1.0', 'Modules'); } - result.NO_PROXY = result.no_proxy = '127.0.0.1,localhost'; + result.NO_PROXY = '127.0.0.1,localhost'; + if (!windows) result.no_proxy = result.NO_PROXY; result.DOTNET_CLI_TELEMETRY_OPTOUT = '1'; result.DOTNET_SKIP_FIRST_TIME_EXPERIENCE = '1'; result.DOTNET_MULTILEVEL_LOOKUP = '0'; @@ -89,7 +117,7 @@ function isolatedEnvironment(env, windows) { // This is a transport for already verified bytes, not an extractor or a public-feed // fallback implementation. Only the official installer unpacks the archive. The // caller's HTTP boundary validates every public hop and streams the original GET. -async function startFeed(fs, download, archiveName, productDirectory, version, product) { +async function startFeed(fs, download, archiveName, productDirectory, version, product, digests) { const prefix = `/${crypto.randomBytes(24).toString('hex')}`; const directory = `${prefix}/${productDirectory}/${version}/`; const archiveRoute = directory + archiveName; @@ -97,7 +125,7 @@ async function startFeed(fs, download, archiveName, productDirectory, version, p `${directory}productVersion.txt`, `${directory}${product === 'sdk' ? 'sdk' : product === 'dotnet' ? 'runtime' : product}-productVersion.txt`, ]); - const size = (await fs.stat(download)).size; + const size = digests.size; let requests = 0; let active = 0; let downloads = 0; @@ -123,7 +151,21 @@ async function startFeed(fs, download, archiveName, productDirectory, version, p else { const file = await fs.open(download, 'r'); try { - await pipeline(file.createReadStream({ autoClose: false, end: size ? size - 1 : undefined }), res); + const sha256 = crypto.createHash('sha256'); + const sha512 = crypto.createHash('sha512'); + let delivered = 0; + const verify = new Transform({ + transform(chunk, _encoding, callback) { + delivered += chunk.length; + sha256.update(chunk); sha512.update(chunk); + callback(null, chunk); + }, + flush(callback) { + callback(delivered === size && sha256.digest('hex') === digests.sha256 && sha512.digest('hex') === digests.sha512 + ? null : new Error('Archive bytes changed during installer delivery')); + }, + }); + await pipeline(file.createReadStream({ autoClose: false, end: size - 1 }), verify, res); downloads++; } finally { await file.close(); @@ -168,9 +210,11 @@ async function startFeed(fs, download, archiveName, productDirectory, version, p * to destination when supplied), and run is the caller's bounded, trusted process * boundary. No process.env is inherited. The acquisition receipt is persisted at * archivePath + '.provenance.json' before any acquired script executes. + * policy/kind/sourceSha bind measured archives to an exact approval; archive.sha512 + * is reserved for publisher checksums. Both modes retain independently hashed bytes. */ async function installArchive(options, deps) { - const { helperRoot, root, archivePath, product, version, platform, archive, cwd } = options; + const { helperRoot, root, archivePath, product, version, platform, archive, cwd, policy, kind, sourceSha } = options; const fs = deps.fs || require('node:fs/promises'); if (typeof deps.http !== 'function' || typeof deps.run !== 'function') throw new Error('Policy-constrained http and trusted run boundaries are required'); @@ -185,11 +229,15 @@ async function installArchive(options, deps) { throw new Error('An exact SDK/runtime version is required, not a channel or latest'); if (!['x64', 'x86', 'arm', 'arm64', 's390x', 'ppc64le', 'loongarch64', 'riscv64'].includes(platform.architecture)) throw new Error('Unsupported installer architecture'); - if (platform.rid !== `${os}-${platform.architecture}`) - throw new Error('Installer OS/RID/architecture mismatch'); - if (!archive || !/^[0-9a-f]{128}$/i.test(archive.sha512)) - throw new Error('Official archive SHA512 is required'); - const archiveName = decodeURIComponent(publicUrl(archive.url).pathname.split('/').pop()); + if (platform.rid !== `${os}-${platform.architecture}` || archive?.rid !== platform.rid || archive.architecture !== platform.architecture) + throw new Error('Installer archive OS/RID/architecture mismatch'); + if (!['source', 'released'].includes(kind) || (kind === 'source' && !/^[0-9a-f]{40}$/.test(sourceSha))) + throw new Error('Explicit installer kind and source revision are required'); + const approval = validateArchiveRequest({ policy, kind, sourceSha, product, version, platform, archive }); + const constraints = approval && { + allowedUrls: [approval.url, ...approval.redirectUrls], maxBytes: approval.maxBytes, timeoutMs: approval.timeoutMs, + }; + const archiveName = new URL(publicUrl(archive.url, policy)).pathname.split('/').pop(); if (!/^[A-Za-z0-9][A-Za-z0-9_.-]*\.(zip|tar\.gz)$/.test(archiveName) || (!windows && !archiveName.endsWith('.tar.gz'))) throw new Error('Unsupported official archive filename'); for (const [name, value] of Object.entries({ helperRoot, root, archivePath, cwd })) @@ -209,16 +257,17 @@ async function installArchive(options, deps) { const download = `${archivePath}.download`; let feed; let receipt; - async function get(url, destination) { + let stage = 'checksum-provenance'; + async function get(url, destination, constraints) { const attempt = { url, version, status: 'pending' }; attempts.push(attempt); try { - const response = await deps.http(url, { method: 'GET', ...(destination ? { destination } : {}) }); + const response = await deps.http(url, { method: 'GET', ...(destination ? { destination } : {}), ...(constraints ? { constraints } : {}) }); attempt.status = response.status; - if (response.status !== 200) throw new Error(`Acquisition HTTP ${response.status}: ${url}`); - publicUrl(response.url); - if (!Array.isArray(response.redirects)) throw new Error('HTTP acquisition omitted redirect provenance'); - return response; + if (response.status !== 200) throw Object.assign(new Error(`Acquisition HTTP ${response.status}: ${url}`), { + code: response.status === 404 ? 'archive-unavailable' : 'network-failure', + }); + return validateHttpResponse(url, response, policy, constraints); } catch (error) { attempt.error = error.message; throw error; @@ -233,11 +282,16 @@ async function installArchive(options, deps) { try { await fs.mkdir(path.dirname(archivePath), { recursive: true }); await fs.mkdir(root, { recursive: true }); - const response = await get(archive.url, download); - const digests = await hashFile(fs, download); - if (digests.sha512 !== archive.sha512.toLowerCase()) throw new Error('Downloaded archive SHA512 does not match official metadata'); + const response = await get(archive.url, download, constraints); + const receivedAt = new Date().toISOString(); + const digests = await hashFile(fs, download, approval?.maxBytes); + if (!digests.size) throw new Error('Downloaded archive is empty'); + if (!approval && digests.sha512 !== archive.sha512.toLowerCase()) throw new Error('Downloaded archive SHA512 does not match official metadata'); + if ((approval || response.size !== undefined) && response.size !== digests.size) + throw new Error('HTTP byte count does not match downloaded archive bytes'); for (const name of ['sha256', 'sha512']) - if (response[name] && response[name].toLowerCase() !== digests[name]) + if ((approval || response[name] !== undefined) && + (typeof response[name] !== 'string' || response[name].toLowerCase() !== digests[name])) throw new Error(`HTTP ${name} digest does not match downloaded archive bytes`); const scriptResponse = await get(installerUrl); if (!Buffer.isBuffer(scriptResponse.bytes) || scriptResponse.bytes.length === 0 || scriptResponse.bytes.length > 1_048_576) @@ -251,14 +305,19 @@ async function installArchive(options, deps) { await fs.writeFile(scriptPath, scriptResponse.bytes, { flag: 'wx' }); if (!windows) await fs.writeFile(path.join(cache, '.dotnet-install.timestamp'), `${Math.floor(Date.now() / 1000)}\n`, { flag: 'wx' }); receipt = { - archive: { ...archive, sha256: digests.sha256, actualSha512: digests.sha512, redirectUrl: response.url, redirects: response.redirects }, + archive: { + ...archive, sha256: digests.sha256, actualSha512: digests.sha512, redirectUrl: response.url, redirects: response.redirects, + finalOrigin: new URL(response.url).origin, bytesReceived: digests.size, receivedAt, retained: { path: archivePath, ...digests }, + }, installer: { url: installerUrl, sha256: scriptDigest, helperPath, helperSha256: helperDigest, path: scriptPath, redirectUrl: scriptResponse.url, redirects: scriptResponse.redirects, }, - attempts, + attempts, sources: structuredClone(options.sources ?? []), }; + validateArchiveReceipt(options, receipt.archive); await fs.writeFile(`${archivePath}.provenance.json`, JSON.stringify(receipt, null, 2)); + stage = 'installation-failure'; Object.assign(env, { RB_HELPER_PATH: helperPath, RB_INSTALLER_CACHE: cache, RB_INSTALLER_PATH: scriptPath }); const helperOutput = await run(windows ? ['-Command', helperPS] : ['-c', helperSH], 60_000); const markers = helperOutput.split(/\r?\n/).filter(line => line.startsWith(marker)); @@ -280,20 +339,35 @@ async function installArchive(options, deps) { if (!supported.has(flag.toLowerCase())) throw new Error(`Official installer does not support required flag ${flag}`); const productDirectory = product === 'sdk' ? 'Sdk' : product === 'aspnetcore' ? 'aspnetcore/Runtime' : product === 'windowsdesktop' && Number(version.split('.')[0]) >= 5 ? 'WindowsDesktop' : 'Runtime'; - feed = await startFeed(fs, download, archiveName, productDirectory, version, product); + stage = 'checksum-provenance'; + const beforeInstall = await hashFile(fs, download, digests.size); + if (beforeInstall.sha256 !== digests.sha256 || beforeInstall.sha512 !== digests.sha512 || beforeInstall.size !== digests.size) + throw new Error('Downloaded archive changed before installer execution'); + feed = await startFeed(fs, download, archiveName, productDirectory, version, product, digests); const args = windows ? ['-File', scriptPath, '-Version', version, '-InstallDir', root, '-Architecture', platform.architecture, '-NoPath', '-KeepZip', '-ZipPath', archivePath, '-AzureFeed', feed.url] : [scriptPath, '--version', version, '--install-dir', root, '--architecture', platform.architecture, '--no-path', '--keep-zip', '--zip-path', archivePath, '--azure-feed', feed.url, '--os', os]; if (product !== 'sdk') args.push(windows ? '-Runtime' : '--runtime', product, windows ? '-SkipNonVersionedFiles' : '--skip-non-versioned-files'); + stage = 'installation-failure'; await run(args, 1_200_000); + stage = 'checksum-provenance'; await feed.verify(); - const retained = await hashFile(fs, archivePath); - if (retained.sha512 !== digests.sha512 || retained.sha256 !== digests.sha256) - throw new Error('Retained archive digest does not match verified download'); + const afterInstall = await hashFile(fs, download, digests.size); + if (afterInstall.sha512 !== digests.sha512 || afterInstall.sha256 !== digests.sha256 || afterInstall.size !== digests.size) + throw new Error('Downloaded archive changed after installer execution'); + await verifyRetainedArchive(receipt.archive, { fs }); + if (typeof fs.chmod === 'function') { + try { await fs.chmod(archivePath, 0o444); } catch (error) { + if (!['ENOTSUP', 'EOPNOTSUPP', 'ENOSYS'].includes(error.code)) throw error; + } + } + await verifyRetainedArchive(receipt.archive, { fs }); + await fs.writeFile(`${archivePath}.provenance.json`, JSON.stringify(receipt, null, 2)); return { ...receipt, logs }; } catch (error) { + error.code ??= stage; error.attempts = attempts; error.logs = logs; if (receipt) error.acquisition = receipt; @@ -304,4 +378,4 @@ async function installArchive(options, deps) { } } -module.exports = { installArchive }; +module.exports = { installArchive, verifyRetainedArchive }; diff --git a/eng/regression-bisect/toolchain-installer.test.cjs b/eng/regression-bisect/toolchain-installer.test.cjs index 037cbf62bcf..4261809ab92 100644 --- a/eng/regression-bisect/toolchain-installer.test.cjs +++ b/eng/regression-bisect/toolchain-installer.test.cjs @@ -9,18 +9,44 @@ const http = require('node:http'); const path = require('node:path'); const { test } = require('node:test'); const { promisify } = require('node:util'); -const { installArchive } = require('./toolchain-installer.cjs'); +const { installArchive, verifyRetainedArchive } = require('./toolchain-installer.cjs'); +const { createHttp, publicUrl } = require('./toolchain-io.cjs'); +const { policy } = require('./fixtures/contracts.cjs'); +const { fingerprintInput } = require('./model.cjs'); const digest = (bytes, algorithm) => crypto.createHash(algorithm).update(bytes).digest('hex'); const parameters = ['Version', 'InstallDir', 'Architecture', 'Runtime', 'NoPath', 'KeepZip', 'ZipPath', 'AzureFeed', 'SkipNonVersionedFiles']; const help = '--version --install-dir --architecture --runtime --no-path --keep-zip --zip-path --azure-feed --os --skip-non-versioned-files'; const payload = Buffer.from('verified archive bytes'); +const approval = { + sourceSha: '5a8b9adb034f420a5c2c1f577670c5eeeb454b7e', product: 'sdk', version: '11.0.100-rc.1.26420.103', + rid: 'win-x64', origin: 'https://builds.dotnet.microsoft.com', + url: 'https://builds.dotnet.microsoft.com/dotnet/Sdk/11.0.100-rc.1.26420.103/dotnet-sdk-11.0.100-rc.1.26420.103-win-x64.zip', + redirectUrls: [], maxBytes: 1073741824, timeoutMs: 300000, + approvalRevision: '2ec6de6837b5b367e1f1186e850b8faa2d188b16', +}; + +for (const oversized of [false, true]) { + test(`HTTP cleanup failure is visible after ${oversized ? 'bounded rejection' : 'complete download'}`, async t => { + const response = new Response(payload); + t.mock.method(response.body, 'cancel', async () => { throw new Error('controlled response cleanup failure'); }); + t.mock.method(globalThis, 'fetch', async () => response); + await assert.rejects(createHttp(policy())(approval.url, { + constraints: { allowedUrls: [approval.url], maxBytes: oversized ? 1 : 100, timeoutMs: 1000 }, + }), error => { + assert.match(error.message, /controlled response cleanup failure/); + if (oversized) assert.match(error.message, /download exceeds byte limit/); + return true; + }); + }); +} function request(url, method = 'GET') { return new Promise((resolve, reject) => { const req = http.request(url, { method, agent: false }, response => { const chunks = []; response.on('data', chunk => chunks.push(chunk)); + response.on('error', reject); response.on('end', () => resolve({ status: response.statusCode, bytes: Buffer.concat(chunks) })); }); req.on('error', reject); @@ -39,8 +65,11 @@ async function fixture(t, { product = 'sdk', os = 'win', architecture = 'x64', r await fs.writeFile(helperPath, '# trusted repository helper'); const options = { helperRoot, root: path.join(directory, 'sdk'), archivePath: path.join(directory, filename), - product, version, platform: { os, rid, architecture }, - archive: { name: `${product}-${rid}`, url: `https://builds.dotnet.microsoft.com/dotnet/${filename}`, sha512: digest(payload, 'sha512') }, + product, version, platform: { os, rid, architecture }, policy: policy(), kind: 'source', sourceSha: approval.sourceSha, + archive: { + name: `${product}-${rid}`, rid, architecture, url: `https://builds.dotnet.microsoft.com/dotnet/${filename}`, + sha512: digest(payload, 'sha512'), provenance: { kind: 'publisher-checksum' }, + }, cwd: directory, env: { SystemRoot: 'C:\\Windows', PATH: 'trusted-system-path', @@ -61,7 +90,8 @@ async function fixture(t, { product = 'sdk', os = 'win', architecture = 'x64', r await fs.writeFile(transport.destination, state.payload); return { status: 200, sha256: digest(state.payload, 'sha256'), sha512: digest(state.payload, 'sha512'), - url: 'https://ci.dot.net/public/archive', redirects: ['https://ci.dot.net/public/archive'], + size: state.payload.length, url: state.finalUrl || 'https://ci.dot.net/public/archive', + redirects: state.redirects || [{ from: url, to: 'https://ci.dot.net/public/archive', status: 302 }], }; } return { bytes: script, url, redirects: [], status: 200 }; @@ -75,8 +105,12 @@ async function fixture(t, { product = 'sdk', os = 'win', architecture = 'x64', r assert.notEqual(execution.env.PSModulePath, 'untrusted'); const provenance = JSON.parse(await fs.readFile(`${options.archivePath}.provenance.json`, 'utf8')); assert.equal(provenance.archive.actualSha512, digest(payload, 'sha512')); - assert.equal(provenance.archive.redirectUrl, 'https://ci.dot.net/public/archive'); + assert.equal(provenance.archive.redirectUrl, state.finalUrl || 'https://ci.dot.net/public/archive'); if (args.some(arg => arg.includes('GetDotNetInstallScript'))) { + if (windows) { + const keys = Object.keys(execution.env).map(key => key.toUpperCase()); + assert.equal(new Set(keys).size, keys.length, 'Windows supervised environment must not contain case aliases'); + } assert.ok(args.some(arg => arg.includes(windows ? '$prepareMachine = $false' : 'prepare_machine=false'))); assert.ok(args.some(arg => arg.includes(windows ? '$ci = $false' : 'ci=false'))); assert.ok(!args.some(arg => arg.includes('InitializeDotNetCli'))); @@ -132,6 +166,205 @@ async function fixture(t, { product = 'sdk', os = 'win', architecture = 'x64', r return { options, deps, state, script, helperPath }; } +async function measuredFixture(t) { + const result = await fixture(t, { version: approval.version }); + const { options, state } = result; + options.archive.url = approval.url; + delete options.archive.sha512; + options.archive.provenance = { + kind: 'approved-official-https-measured', approvalRevision: approval.approvalRevision, + approvalDigest: fingerprintInput(approval), + }; + assert.deepEqual(options.policy.toolchains.measuredArchives[0], approval); + state.finalUrl = approval.url; + state.redirects = []; + return result; +} + +test('measured byte mismatch reports checksum provenance, not archive absence', async t => { + const { options, deps } = await measuredFixture(t); + const transport = deps.http; + deps.http = async (...args) => { + const response = await transport(...args); + if (args[1].destination) response.sha256 = '0'.repeat(64); + return response; + }; + await assert.rejects(installArchive(options, deps), error => { + assert.match(error.message, /HTTP sha256 digest/); + assert.equal(error.code, 'checksum-provenance'); + return true; + }); +}); + +for (const conflicting of [false, true]) { + test(`Windows installer canonicalizes environment aliases: conflicting=${conflicting}`, async t => { + const { options, deps, state } = await fixture(t); + options.env.SYSTEMROOT = options.env.SystemRoot; + delete options.env.SystemRoot; + options.env.path = conflicting ? 'different-path' : options.env.PATH; + if (conflicting) { + await assert.rejects(installArchive(options, deps), /Conflicting Windows environment aliases/); + assert.equal(state.requests.length, 0); + } else await installArchive(options, deps); + }); +} + +test('approved exact historical preview measures bytes without claiming a publisher SHA512', async t => { + const { options, deps, state } = await measuredFixture(t); + options.sources = [{ url: `${approval.url}.sha512`, status: 404, version: approval.version }]; + const result = await installArchive(options, deps); + assert.equal(Object.hasOwn(result.archive, 'sha512'), false); + assert.deepEqual(result.archive.provenance, options.archive.provenance); + assert.equal(result.archive.actualSha512, digest(payload, 'sha512')); + assert.equal(result.archive.finalOrigin, approval.origin); + assert.equal(result.archive.bytesReceived, payload.length); + assert.equal(new Date(result.archive.receivedAt).toISOString(), result.archive.receivedAt); + assert.deepEqual(result.archive.retained, { + path: options.archivePath, sha256: digest(payload, 'sha256'), sha512: digest(payload, 'sha512'), size: payload.length, + }); + assert.deepEqual(state.requests[0].constraints, { + allowedUrls: [approval.url], maxBytes: approval.maxBytes, timeoutMs: approval.timeoutMs, + }); + await verifyRetainedArchive(result.archive); + assert.deepEqual(result.sources, options.sources); + assert.notEqual(result.sources, options.sources); + assert.deepEqual(JSON.parse(await fs.readFile(`${options.archivePath}.provenance.json`, 'utf8')).archive, result.archive); + assert.deepEqual(JSON.parse(await fs.readFile(`${options.archivePath}.provenance.json`, 'utf8')).sources, options.sources); +}); + +test('released publisher-checksum installation remains available without measured approval', async t => { + const { options, deps } = await fixture(t, { version: '10.0.401' }); + options.kind = 'released'; + delete options.sourceSha; + deps.fs = { ...fs, chmod: undefined }; + const result = await installArchive(options, deps); + assert.deepEqual(result.sources, []); + assert.equal(result.archive.sha512, digest(payload, 'sha512')); + assert.deepEqual(result.archive.provenance, { kind: 'publisher-checksum' }); + await verifyRetainedArchive(result.archive); +}); + +for (const [name, mutate] of [ + ['missing policy', o => { delete o.policy; }], + ['missing approval', o => { o.policy.toolchains.measuredArchives = []; }], + ['changed approval revision', o => { o.policy.toolchains.measuredArchives[0].approvalRevision = '0'.repeat(40); }], + ['widened approval redirects', o => { o.policy.toolchains.measuredArchives[0].redirectUrls.push('https://ci.dot.net/public/other.zip'); }], + ['changed source', o => { o.sourceSha = '0'.repeat(40); }], + ['released mode', o => { o.kind = 'released'; }], + ['missing mode', o => { delete o.kind; }], + ['wrong version', o => { o.version = '11.0.100'; }], + ['wrong product', o => { o.product = 'dotnet'; }], + ['wrong RID', o => { o.archive.rid = 'win-arm64'; }], + ['wrong architecture', o => { o.archive.architecture = 'arm64'; }], + ['missing provenance', o => { delete o.archive.provenance; }], + ['unapproved provenance kind', o => { o.archive.provenance.kind = 'measured'; }], + ['wrong approval digest', o => { o.archive.provenance.approvalDigest = '0'.repeat(64); }], + ['wrong provenance revision', o => { o.archive.provenance.approvalRevision = '0'.repeat(40); }], + ['measured SHA512 relabeled as publisher', o => { o.archive.sha512 = digest(payload, 'sha512'); }], + ['different URL on same origin', o => { o.archive.url = o.archive.url.replace('/Sdk/', '/Runtime/'); }], +]) { + test(`measured request fails closed before network: ${name}`, async t => { + const { options, deps, state } = await measuredFixture(t); + mutate(options); + await assert.rejects(installArchive(options, deps)); + assert.equal(state.requests.length, 0); + assert.equal(state.calls.length, 0); + }); +} + +for (const [name, mutate] of [ + ['missing sha256', r => { delete r.sha256; }], + ['missing sha512', r => { delete r.sha512; }], + ['wrong sha256', r => { r.sha256 = '0'.repeat(64); }], + ['wrong sha512', r => { r.sha512 = '0'.repeat(128); }], + ['missing size', r => { delete r.size; }], + ['truncated bytes', r => { r.size++; }], + ['oversized bytes', r => { r.size = approval.maxBytes + 1; }], + ['partial status', r => { r.status = 206; }], + ['missing redirects', r => { delete r.redirects; }], + ['omitted hops', r => { r.url = 'https://ci.dot.net/public/other.zip'; }], + ['wrong hop source', r => { r.redirects = [{ from: `${approval.url}.other`, to: approval.url, status: 302 }]; }], + ['invalid hop status', r => { r.redirects = [{ from: approval.url, to: approval.url, status: 200 }]; }], + ['unapproved intermediate hop', r => { + const other = 'https://ci.dot.net/public/other.zip'; + r.redirects = [{ from: approval.url, to: other, status: 302 }, { from: other, to: approval.url, status: 302 }]; + }], + ['downgrade', r => { + r.url = approval.url.replace('https:', 'http:'); + r.redirects = [{ from: approval.url, to: r.url, status: 302 }]; + }], +]) { + test(`measured adapter independently rejects HTTP claims: ${name}`, async t => { + const { options, deps, state } = await measuredFixture(t); + const transport = deps.http; + deps.http = async (...args) => { + const response = await transport(...args); + if (args[1].destination) mutate(response); + return response; + }; + await assert.rejects(installArchive(options, deps)); + assert.equal(state.calls.length, 0); + }); +} + +for (const phase of ['download', 'helper', 'delivery', 'installer exit', 'retention protection']) { + test(`measured archive mutation fails closed during ${phase}`, async t => { + const { options, deps, state } = await measuredFixture(t); + const changed = Buffer.from(payload); + changed[0] ^= 1; + if (phase === 'download') state.payload = Buffer.alloc(0); + else if (phase === 'delivery') { + let reads = 0; + deps.fs = { + ...fs, + async open(filename, ...args) { + if (filename === `${options.archivePath}.download` && ++reads === 3) + await fs.writeFile(filename, changed); + return fs.open(filename, ...args); + }, + }; + } else if (phase === 'retention protection') { + deps.fs = { ...fs, chmod: filename => fs.writeFile(filename, changed) }; + } + const run = deps.run; + deps.run = async (exe, args, execution) => { + if (phase === 'delivery' && args.includes('-AzureFeed')) { + const url = `${args[args.indexOf('-AzureFeed') + 1]}/Sdk/${options.version}/${path.basename(options.archivePath)}`; + await request(url).catch(() => {}); + await fs.writeFile(`${options.archivePath}.download`, payload); + await fs.writeFile(options.archivePath, payload); + return { exitCode: 0 }; + } + const result = await run(exe, args, execution); + if ((phase === 'helper' && args.includes('-Command')) || (phase === 'installer exit' && args.includes('-AzureFeed'))) + await fs.writeFile(`${options.archivePath}.download`, changed); + return result; + }; + await assert.rejects(installArchive(options, deps), + phase === 'delivery' ? /changed during installer delivery/ : /changed|digest|empty|delivery/i); + await assert.rejects(fs.stat(`${options.archivePath}.download`), { code: 'ENOENT' }); + }); +} + +for (const mutation of ['bytes', 'missing', 'receipt sha256', 'receipt sha512', 'receipt size', 'relative path', 'nonlink', 'directory']) { + test(`retained archive reuse rejects ${mutation}`, async t => { + const { options, deps } = await measuredFixture(t); + const { archive } = await installArchive(options, deps); + let verifyFs = fs; + if (mutation === 'bytes') { + await fs.chmod(options.archivePath, 0o666); + await fs.writeFile(options.archivePath, Buffer.alloc(payload.length)); + } else if (mutation === 'missing') await fs.rm(options.archivePath, { force: true }); + else if (mutation === 'receipt sha256') archive.sha256 = '0'.repeat(64); + else if (mutation === 'receipt sha512') archive.actualSha512 = '0'.repeat(128); + else if (mutation === 'receipt size') archive.bytesReceived++; + else if (mutation === 'relative path') archive.retained.path = path.basename(options.archivePath); + else if (mutation === 'directory') archive.retained.path = options.cwd; + else verifyFs = { ...fs, lstat: async filename => ({ ...await fs.lstat(filename), isFile: () => true, isSymbolicLink: () => true }) }; + await assert.rejects(verifyRetainedArchive(archive, { fs: verifyFs }), /archive|ENOENT|regular/i); + }); +} + for (const settings of [ { product: 'sdk' }, { product: 'dotnet' }, { product: 'aspnetcore' }, { product: 'windowsdesktop' }, { product: 'windowsdesktop', version: '3.1.32' }, @@ -148,7 +381,7 @@ for (const settings of [ assert.equal(result.archive.name, options.archive.name); assert.equal(result.archive.url, options.archive.url); assert.equal(result.archive.redirectUrl, 'https://ci.dot.net/public/archive'); - assert.deepEqual(result.archive.redirects, ['https://ci.dot.net/public/archive']); + assert.deepEqual(result.archive.redirects, [{ from: options.archive.url, to: 'https://ci.dot.net/public/archive', status: 302 }]); assert.equal(result.installer.sha256, digest(script, 'sha256')); assert.equal(result.installer.helperPath, helperPath); assert.equal(result.installer.helperSha256, digest(await fs.readFile(helperPath), 'sha256')); @@ -170,7 +403,7 @@ test('archive delivery completes before a slow EOF read after its advertised len ...fs, async open(filename, ...args) { const file = await fs.open(filename, ...args); - if (filename !== `${options.archivePath}.download` || ++reads !== 2) return file; + if (filename !== `${options.archivePath}.download` || ++reads !== 3) return file; return { createReadStream(settings) { return fileSystem.createReadStream(filename, { @@ -210,7 +443,7 @@ for (const os of ['win', 'linux']) { async open(filename, ...args) { if (filename === options.archivePath) assert.equal(deliveryFinished, true); const file = await fs.open(filename, ...args); - if (filename !== `${options.archivePath}.download` || ++reads !== 2) return file; + if (filename !== `${options.archivePath}.download` || ++reads !== 3) return file; signalDeliveryReady(); await deliveryAllowed; return { @@ -355,7 +588,8 @@ for (const mutation of [ o => { o.archive.sha512 = ''; }, o => { o.platform.architecture = 'x64; bad'; }, o => { o.platform.rid = 'win-arm64'; }, o => { o.platform.rid = 'linux-x64'; }, o => { o.env = undefined; }, o => { o.archive.url = 'http://example.com/file.zip'; }, - o => { o.env.SystemRoot = 'relative'; }, + o => { o.env.SystemRoot = 'relative'; }, o => { delete o.kind; }, o => { delete o.sourceSha; }, + o => { delete o.archive.provenance; }, o => { delete o.policy; }, ]) { test(`rejects invalid input: ${mutation}`, async t => { const { options, deps, state } = await fixture(t); @@ -366,16 +600,20 @@ for (const mutation of [ }); } -test('unavailable archive is explicit, with no fallback or process execution', async t => { - const { options, deps, state } = await fixture(t); - deps.http = async url => ({ status: 404, url, redirects: [] }); - await assert.rejects(installArchive(options, deps), error => { - assert.match(error.message, /404/); - assert.equal(error.attempts[0].url, options.archive.url); - return true; +for (const status of [404, 429, 500, 503]) { + test(`HTTP ${status} acquisition fails explicitly, with no fallback or process execution`, async t => { + const { options, deps, state } = await fixture(t); + deps.http = async url => ({ status, url, redirects: [] }); + await assert.rejects(installArchive(options, deps), error => { + assert.match(error.message, new RegExp(String(status))); + assert.equal(error.code, status === 404 ? 'archive-unavailable' : 'network-failure'); + assert.equal(error.attempts[0].url, options.archive.url); + assert.equal(error.attempts.length, 1); + return true; + }); + assert.equal(state.calls.length, 0); }); - assert.equal(state.calls.length, 0); -}); +} test('real repository helper returns seeded script and parses actual PS parameters', { skip: process.platform !== 'win32' }, async t => { const { options, deps } = await fixture(t); @@ -396,3 +634,119 @@ test('real repository helper returns seeded script and parses actual PS paramete await installArchive(options, deps); assert.equal(helperRan, true); }); + +for (const url of [ + approval.url.replace('https:', 'http:'), approval.url.replace('/Sdk/', '/Runtime/'), + approval.url.replace('/Sdk/', '/dotnet/../Sdk/'), approval.url.replace('/Sdk/', '/%53dk/'), + approval.url.replace('https://', 'https://user:pass@'), `${approval.url}?query=1`, `${approval.url}#fragment`, + approval.url.replace('https://builds.', 'https://BUILDS.'), approval.url.replace('/Sdk/', '\\Sdk\\'), +]) { + test(`constrained HTTP rejects nonexact archive URL before network: ${url}`, async t => { + const fetch = t.mock.method(globalThis, 'fetch', () => assert.fail('must reject before network')); + await assert.rejects(createHttp(policy())(url, { + constraints: { allowedUrls: [approval.url], maxBytes: 100, timeoutMs: 1000 }, + }), /unapproved|exact approved/); + assert.equal(fetch.mock.callCount(), 0); + }); +} + +for (const redirect of [ + 'http://builds.dotnet.microsoft.com/dotnet/other.zip', 'https://ci.dot.net/public/other.zip', + `${approval.url}?query=1`, `${approval.url}#fragment`, approval.url.replace('/Sdk/', '/%53dk/'), + approval.url.replace('/Sdk/', '/nested/../Sdk/'), +]) { + test(`constrained HTTP rejects redirect before network: ${redirect}`, async t => { + const fetch = t.mock.method(globalThis, 'fetch', async () => new Response(null, { status: 302, headers: { location: redirect } })); + await assert.rejects(createHttp(policy())(approval.url, { + constraints: { allowedUrls: [approval.url], maxBytes: 100, timeoutMs: 1000 }, + }), /unapproved|exact approved/); + assert.equal(fetch.mock.callCount(), 1); + }); +} + +test('constrained HTTP follows only exact allowed hops with one total deadline and streamed digests', async t => { + const directory = await fs.mkdtemp(path.join(__dirname, '.installer-http-')); + t.after(() => fs.rm(directory, { recursive: true, force: true })); + const redirect = 'https://ci.dot.net/public/approved.zip'; + const signals = []; + const fetch = t.mock.method(globalThis, 'fetch', async (url, options) => { + signals.push(options.signal); + assert.equal(options.redirect, 'manual'); + return url === approval.url ? new Response(null, { status: 307, headers: { location: redirect } }) + : new Response(payload, { headers: { 'content-length': String(payload.length) } }); + }); + const destination = path.join(directory, 'archive.zip'); + const result = await createHttp(policy())(approval.url, { + destination, constraints: { allowedUrls: [approval.url, redirect], maxBytes: payload.length, timeoutMs: 1000 }, + }); + assert.equal(fetch.mock.callCount(), 2); + assert.equal(signals[0], signals[1]); + assert.deepEqual(result.redirects, [{ from: approval.url, to: redirect, status: 307 }]); + assert.deepEqual(await fs.readFile(destination), payload); + assert.equal(result.bytes, undefined); + assert.equal(result.size, payload.length); + assert.equal(result.sha256, digest(payload, 'sha256')); + assert.equal(result.sha512, digest(payload, 'sha512')); +}); + +for (const [name, headers, maxBytes] of [ + ['advertised oversize', { 'content-length': '100' }, 4], + ['streamed oversize', {}, 4], + ['truncated body', { 'content-length': String(payload.length + 1) }, 100], + ['overlong body', { 'content-length': '1' }, 100], + ['invalid length', { 'content-length': 'invalid' }, 100], + ['infinite limit', {}, Infinity], + ['zero limit', {}, 0], +]) { + test(`constrained HTTP rejects ${name}`, async t => { + t.mock.method(globalThis, 'fetch', async () => new Response(payload, { headers })); + await assert.rejects(createHttp(policy())(approval.url, { + constraints: { allowedUrls: [approval.url], maxBytes, timeoutMs: 1000 }, + }), /limit|Content-Length|constraints/); + }); +} + +test('constrained HTTP timeout aborts the download rather than resetting on a redirect', async t => { + const keepAlive = setTimeout(() => {}, 1000); + t.after(() => clearTimeout(keepAlive)); + t.mock.method(globalThis, 'fetch', (_url, { signal }) => new Promise((_, reject) => { + signal.addEventListener('abort', () => reject(signal.reason), { once: true }); + })); + await assert.rejects(createHttp(policy())(approval.url, { + constraints: { allowedUrls: [approval.url], maxBytes: 100, timeoutMs: 10 }, + }), { name: 'TimeoutError', code: 'network-failure' }); +}); + +for (const phase of ['request', 'body']) { + test(`HTTP native ${phase} failure retains network classification and cause`, async t => { + const cause = new TypeError('fetch failed'); + t.mock.method(globalThis, 'fetch', async () => { + if (phase === 'request') throw cause; + return new Response(new ReadableStream({ start(controller) { controller.error(cause); } })); + }); + await assert.rejects(createHttp(policy())(approval.url), error => + error.code === 'network-failure' && error.cause === cause && error.name === 'TypeError'); + await assert.rejects(createHttp(policy())(approval.url.replace('https:', 'http:')), error => + error.code !== 'network-failure' && /unapproved/.test(error.message)); + }); +} + +test('HTTP availability HEAD does not consume a body or allocate an archive file', async t => { + const body = new Response(payload, { headers: { 'content-length': String(approval.maxBytes + 1) } }); + const cancel = t.mock.method(body.body, 'cancel'); + t.mock.method(globalThis, 'fetch', async (_url, options) => { + assert.equal(options.method, 'HEAD'); + return body; + }); + const result = await createHttp(policy(), { open: () => assert.fail('HEAD must not allocate a file') })(approval.url, { + destination: 'unused.zip', method: 'HEAD', + }); + assert.equal(cancel.mock.callCount(), 1); + assert.equal(result.size, 0); + assert.equal(result.bytes.length, 0); +}); + +test('public URL canonicalization rejects escaped traversal rather than normalizing it', () => { + for (const url of [approval.url.replace('/Sdk/', '/x/../Sdk/'), approval.url.replace('/Sdk/', '/%53dk/')]) + assert.throws(() => publicUrl(url, policy()), /unapproved/); +}); diff --git a/eng/regression-bisect/toolchain-io.cjs b/eng/regression-bisect/toolchain-io.cjs index 58251426d62..cedb57fc152 100644 --- a/eng/regression-bisect/toolchain-io.cjs +++ b/eng/regression-bisect/toolchain-io.cjs @@ -12,45 +12,106 @@ const inside = (root, file) => { function publicUrl(url, policy) { const parsed = new URL(url); - requireValue(parsed.protocol === "https:" && !parsed.username && !parsed.password && !parsed.search && + requireValue(typeof url === "string" && parsed.href === url && !/[%\\]/.test(url) && + parsed.protocol === "https:" && !parsed.username && !parsed.password && !parsed.search && !parsed.hash && !parsed.port && policy.toolchains.publicFeeds.some(feed => parsed.href === feed || parsed.href.startsWith(`${feed}/`)), `unapproved public download URL: ${url}`); return url; } +function downloadUrl(url, policy, constraints) { + publicUrl(url, policy); + if (constraints) { + requireValue(Array.isArray(constraints.allowedUrls) && constraints.allowedUrls.includes(url), + `URL is not an exact approved download: ${url}`); + requireValue(Number.isSafeInteger(constraints.maxBytes) && constraints.maxBytes > 0 && constraints.maxBytes <= 1073741824 && + Number.isSafeInteger(constraints.timeoutMs) && constraints.timeoutMs > 0 && constraints.timeoutMs <= 300000, + "invalid download byte or timeout constraints"); + } + return url; +} + +function validateHttpResponse(url, response, policy, constraints) { + let current = downloadUrl(url, policy, constraints); + requireValue(Array.isArray(response.redirects) && response.redirects.length <= 5, "HTTP acquisition omitted or exceeded redirect provenance"); + for (const hop of response.redirects) { + requireValue(hop && hop.from === current && [301, 302, 303, 307, 308].includes(hop.status), + "HTTP redirect provenance is not a contiguous chain"); + current = downloadUrl(hop.to, policy, constraints); + } + requireValue(response.url === current, "HTTP final URL does not match redirect provenance"); + return response; +} + function createHttp(policy, fileSystem = fs, signal) { - return async (url, { destination, method = "GET" } = {}) => { + const networkFailure = error => { + throw Object.assign(new Error(error.message, { cause: error }), { name: error.name, code: "network-failure" }); + }; + return async (url, { destination, method = "GET", constraints } = {}) => { const redirects = []; - let current = publicUrl(url, policy); + let current = downloadUrl(url, policy, constraints); + requireValue(["GET", "HEAD"].includes(method), "unsupported download method"); + const deadline = AbortSignal.timeout(constraints?.timeoutMs ?? 300000); + const downloadSignal = signal ? AbortSignal.any([signal, deadline]) : deadline; for (let hop = 0; hop <= 5; hop++) { + if (downloadSignal.aborted) networkFailure(downloadSignal.reason); const response = await fetch(current, { - method, redirect: "manual", - signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(300000)]) : AbortSignal.timeout(300000), - }); + method, redirect: "manual", signal: downloadSignal, headers: { "Accept-Encoding": "identity" }, + }).catch(networkFailure); + if (downloadSignal.aborted) networkFailure(downloadSignal.reason); if ([301, 302, 303, 307, 308].includes(response.status)) { await response.body?.cancel(); const location = response.headers.get("location"); requireValue(location?.trim(), "redirect response is missing Location"); - const next = publicUrl(new URL(location, current).href, policy); + requireValue(!/[%\\\s]/.test(location) && !location.split(/[/?#]/).some(segment => segment === "." || segment === ".."), + `unapproved public download URL: ${location}`); + const next = downloadUrl(new URL(location, current).href, policy, constraints); + if (/^https?:/i.test(location)) requireValue(location === next, `unapproved public download URL: ${location}`); redirects.push({ from: current, to: next, status: response.status }); current = next; continue; } + const maxBytes = Math.min(constraints?.maxBytes ?? Infinity, destination && response.ok ? 1073741824 : 16777216); + const length = response.headers.get("content-length"); const sha256 = createHash("sha256"); const sha512 = createHash("sha512"); const chunks = []; let size = 0; - const file = destination && response.ok ? await fileSystem.open(destination, "wx") : null; + let file; + let reader; + let failure; try { - for await (const chunk of response.body ?? []) { + if (method !== "HEAD" && length !== null) + requireValue(/^\d+$/.test(length) && Number.isSafeInteger(Number(length)) && Number(length) <= maxBytes, + "invalid or oversized download Content-Length"); + if (method === "GET" && destination && response.ok) file = await fileSystem.open(destination, "wx"); + if (method === "GET") reader = response.body?.getReader(); + while (reader) { + const { done, value: chunk } = await reader.read().catch(networkFailure); + if (downloadSignal.aborted) networkFailure(downloadSignal.reason); + if (done) break; size += chunk.length; - requireValue(size <= (file ? 1024 * 1024 * 1024 : 16 * 1024 * 1024), `download exceeds byte limit: ${current}`); + requireValue(size <= maxBytes, `download exceeds byte limit: ${current}`); sha256.update(chunk); sha512.update(chunk); if (file) await file.writeFile(chunk); else chunks.push(chunk); } + if (downloadSignal.aborted) networkFailure(downloadSignal.reason); + if (method !== "HEAD" && length !== null) + requireValue(size === Number(length), "download Content-Length truncation or mismatch"); + } catch (error) { + failure = error; + throw error; } finally { - await file?.close(); + reader?.releaseLock(); + try { + await response.body?.cancel(); + } catch (error) { + if (!failure) throw error; + if (error !== failure && error !== failure.cause) failure.message += `; response cleanup failed: ${error.message}`; + } finally { + await file?.close(); + } } return { status: response.status, url: current, redirects, @@ -243,4 +304,4 @@ async function capture(request, deps) { }; } -module.exports = { hash, requireValue, inside, publicUrl, createHttp, run, isolatedEnvironment, fileIdentity, capture }; +module.exports = { hash, requireValue, inside, publicUrl, validateHttpResponse, createHttp, run, isolatedEnvironment, fileIdentity, capture }; diff --git a/eng/regression-bisect/toolchains.cjs b/eng/regression-bisect/toolchains.cjs index 17923b75217..c78de250d2a 100644 --- a/eng/regression-bisect/toolchains.cjs +++ b/eng/regression-bisect/toolchains.cjs @@ -1,7 +1,8 @@ const fs = require("node:fs/promises"); const path = require("node:path"); const { validatePolicy, fingerprintInput } = require("./model.cjs"); -const { hash, requireValue, inside, publicUrl, createHttp, run, isolatedEnvironment, capture } = require("./toolchain-io.cjs"); +const { hash, requireValue, inside, publicUrl, createHttp, validateHttpResponse, run, isolatedEnvironment, capture } = require("./toolchain-io.cjs"); +const { validateArchiveReceipt } = require("./archive-provenance.cjs"); const indexUrl = "https://builds.dotnet.microsoft.com/dotnet/release-metadata/releases-index.json"; const exactVersion = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$/; @@ -22,9 +23,13 @@ function failure(stage, error, attempts = []) { status: "untestable", outcome: { schemaVersion: 1, outcome: "untestable", - reason: { identity: "wrong-identity", sdk: "unavailable-sdk", runtime: "unavailable-runtime" }[stage] ?? "setup-failure", + reason: ["checksum-provenance", "network-failure"].includes(error.code) ? error.code : + stage === "sdk" && ["command-failure", "installation-failure"].includes(error.code) ? "setup-failure" : + { identity: "wrong-identity", sdk: "unavailable-sdk", runtime: "unavailable-runtime" }[stage] ?? "setup-failure", }, - error: { stage, code: error.code ?? "toolchain-unavailable", message: error.message }, attempts, + error: { stage, code: error.code ?? "toolchain-unavailable", message: error.message }, + attempts: attempts.map(a => a.status === "pending" ? { ...a, status: "error", error: error.message } : a), + ...(error.acquisition ? { acquisition: error.acquisition } : {}), }; } function validatePlatform(platform) { @@ -47,7 +52,7 @@ async function document(url, options, deps, records, attempts) { const response = await deps.http(url); attempt.status = response.status; validateResponse(url, response, options.policy); - requireValue(response.status === 200, `metadata HTTP ${response.status}: ${url}`); + if (response.status !== 200) throw Object.assign(new Error(`metadata HTTP ${response.status}: ${url}`), { code: "network-failure" }); const bytes = Buffer.from(response.bytes); records.push({ url, redirectUrl: response.url, sha256: hash(bytes), bytesBase64: bytes.toString("base64") }); return JSON.parse(bytes.toString("utf8")); @@ -72,8 +77,10 @@ function selectArchive(product, platform, policy) { requireValue(files?.length === 1, `missing or ambiguous ${platform.rid} ${extension} archive for ${product.version}`); const file = files[0]; publicUrl(file.url, policy); - requireValue(/^[0-9a-f]{128}$/i.test(file.hash), `malformed SHA-512 for ${file.name}`); - return { name: file.name, url: file.url, sha512: file.hash.toLowerCase(), rid: file.rid, architecture: platform.architecture }; + if (!/^[0-9a-f]{128}$/i.test(file.hash)) throw Object.assign(new Error(`malformed SHA-512 for ${file.name}`), + { code: "checksum-provenance" }); + return { name: file.name, url: file.url, sha512: file.hash.toLowerCase(), rid: file.rid, architecture: platform.architecture, + provenance: { kind: "publisher-checksum" } }; } function sdkEntries(release) { return [release.sdk, ...(release.sdks ?? [])].filter(Boolean); @@ -206,26 +213,50 @@ async function exactArchive(toolchain, product, version, options, deps, attempts bundledRuntime: { version: found["runtime-version"] ?? row.release.runtime?.version }, }; } - // Public unreleased build archives have no release record. Require their official - // sidecar checksum, never a hash computed from an otherwise unauthenticated archive. const family = { sdk: "Sdk", dotnet: "Runtime", aspnetcore: "aspnetcore/Runtime", windowsdesktop: "WindowsDesktop" }[product]; const prefix = { sdk: "dotnet-sdk", dotnet: "dotnet-runtime", aspnetcore: "aspnetcore-runtime", windowsdesktop: "windowsdesktop-runtime" }[product]; const name = `${prefix}-${version}-${toolchain.platform.rid}${toolchain.platform.os === "windows" ? ".zip" : ".tar.gz"}`; - for (const feed of options.policy.toolchains.publicFeeds.filter(f => /(?:builds\.dotnet\.microsoft\.com\/dotnet|ci\.dot\.net\/public)$/.test(f))) { - const url = `${feed}/${family}/${version}/${name}`; + const urls = options.policy.toolchains.publicFeeds + .filter(f => /(?:builds\.dotnet\.microsoft\.com\/dotnet|ci\.dot\.net\/public)$/.test(f)) + .map(feed => `${feed}/${family}/${version}/${name}`); + for (const url of urls) { const attempt = { url: `${url}.sha512`, version, status: "pending" }; attempts.push(attempt); const response = await deps.http(attempt.url); attempt.status = response.status; validateResponse(attempt.url, response, options.policy); if (response.status === 404) continue; - requireValue(response.status === 200, `checksum HTTP ${response.status}: ${attempt.url}`); + if (response.status !== 200) throw Object.assign(new Error(`checksum HTTP ${response.status}: ${attempt.url}`), { code: "network-failure" }); const checksum = Buffer.from(response.bytes).toString("utf8").trim().split(/\s+/)[0]; - requireValue(/^[a-f0-9]{128}$/i.test(checksum), `malformed historical checksum: ${attempt.url}`); + if (!/^[a-f0-9]{128}$/i.test(checksum)) throw Object.assign(new Error(`malformed historical checksum: ${attempt.url}`), + { code: "checksum-provenance" }); metadata.push({ url: attempt.url, redirectUrl: response.url, sha256: hash(response.bytes), bytesBase64: Buffer.from(response.bytes).toString("base64") }); - return { archive: { name, url, sha512: checksum.toLowerCase(), rid: toolchain.platform.rid, architecture: toolchain.platform.architecture }, metadata }; + return { archive: { name, url, sha512: checksum.toLowerCase(), rid: toolchain.platform.rid, + architecture: toolchain.platform.architecture, provenance: { kind: "publisher-checksum" } }, metadata }; } - throw new Error(`exact ${product} ${version} unavailable from permitted public feeds`); + // Availability is not checksum provenance. HEAD never downloads or executes unapproved bytes. + for (const url of urls) { + const approval = options.policy.toolchains.measuredArchives.find(a => toolchain.kind === "source" && + a.sourceSha === toolchain.sourceSha && a.product === product && a.version === version && + a.rid === toolchain.platform.rid && a.url === url); + const constraints = approval ? { allowedUrls: [url, ...approval.redirectUrls], + maxBytes: approval.maxBytes, timeoutMs: approval.timeoutMs } : undefined; + const attempt = { url, version, method: "HEAD", status: "pending" }; + attempts.push(attempt); + const response = await deps.http(url, { method: "HEAD", ...(constraints ? { constraints } : {}) }); + attempt.status = response.status; + validateHttpResponse(url, response, options.policy, constraints); + if (response.status === 404) continue; + if (response.status === 200 && approval) return { + archive: { name, url, rid: toolchain.platform.rid, architecture: toolchain.platform.architecture, + provenance: { kind: "approved-official-https-measured", approvalRevision: approval.approvalRevision, + approvalDigest: fingerprintInput(approval) } }, metadata, + }; + if (response.status === 200) throw Object.assign(new Error(`exact ${product} ${version} is available but lacks approved checksum provenance`), + { code: "checksum-provenance" }); + throw Object.assign(new Error(`archive availability HTTP ${response.status}: ${url}`), { code: "network-failure" }); + } + throw Object.assign(new Error(`exact ${product} ${version} archive unavailable from permitted public feeds`), { code: "archive-unavailable" }); } function verifyHostRid(toolchain, identity) { @@ -269,9 +300,24 @@ function verifyToolchainIdentity({ policy, toolchain, identity }) { verifyHostRid(toolchain, identity); requireValue(identity.sdks.length === 1 && identity.sdks[0].version === toolchain.sdkVersion && identity.sdks[0].path === path.join(root, "sdk"), "unexpected installed SDK or SDK location"); - requireValue(identity.archive?.sha512 === toolchain.archive.sha512 && identity.archive.actualSha512 === toolchain.archive.sha512 && - identity.archive.url === toolchain.archive.url && /^[a-f0-9]{64}$/.test(identity.archive.sha256), "archive identity mismatch"); - publicUrl(identity.archive.redirectUrl, policy); + validateArchiveReceipt({ policy, ...toolchain, version: toolchain.sdkVersion, product: "sdk" }, identity.archive); + if (Object.hasOwn(identity, "installations")) { + requireValue(Array.isArray(identity.installations), "missing installation receipts"); + const sdks = identity.installations.filter(i => i.product === "sdk"); + requireValue(sdks.length === 1 && sdks[0].version === toolchain.sdkVersion && + fingerprintInput(sdks[0].archive) === fingerprintInput(identity.archive), "missing or mismatched SDK installation receipt"); + const runtimes = identity.installations.filter(i => i.product !== "sdk"); + for (const installed of runtimes) { + requireValue(toolchain.runtimeRequirements.some(r => runtimeProducts[r.name] === installed.product && r.version === installed.version), + "unexpected supplemental installation receipt"); + validateArchiveReceipt({ policy, ...toolchain, version: installed.version, product: installed.product, + archive: installed.archive }, installed.archive); + } + for (const required of toolchain.runtimeRequirements) + requireValue(identity.installations.some(i => i.archive.url === required.archiveUrl && + (i.product === "sdk" || i.product === runtimeProducts[required.name] && i.version === required.version)), + "missing required runtime archive receipt"); + } for (const [name, assemblyName] of Object.entries({ compiler: "fsc", fsi: "fsi", core: "FSharp.Core" })) { const file = identity.files[name]; requireValue(file && inside(path.join(root, "sdk", toolchain.sdkVersion, "FSharp"), file.path) && @@ -349,10 +395,10 @@ async function acquireToolchain(options, injected = {}) { "eng", "common"), root, archivePath: path.join(work, `${product}-${version}${toolchain.platform.os === "windows" ? ".zip" : ".tar.gz"}`), product, version, platform: toolchain.platform, archive, env, cwd: sampleDirectory, + policy: options.policy, kind: toolchain.kind, sourceSha: toolchain.sourceSha, sources: structuredClone(attempts), }, deps); attempts.push(...result.attempts); - requireValue(result.archive.actualSha512 === archive.sha512 && result.archive.url === archive.url, "installer archive checksum or URL mismatch"); - validateResponse(archive.url, { url: result.archive.redirectUrl, redirects: result.archive.redirects }, options.policy); + validateArchiveReceipt({ policy: options.policy, ...toolchain, version, product, archive }, result.archive); installed.push({ product, version, ...result }); }; stage = "sdk"; @@ -376,6 +422,8 @@ async function acquireToolchain(options, injected = {}) { const exact = await exactArchive(toolchain, product, required.version, options, deps, attempts); await install(product, required.version, exact.archive); } + required.archiveUrl = (installed.find(i => i.product === runtimeProducts[required.name] && + i.version === required.version) ?? installed[0]).archive.url; } stage = "identity"; const executable = path.join(root, toolchain.platform.os === "windows" ? "dotnet.exe" : "dotnet"); @@ -396,6 +444,7 @@ async function acquireToolchain(options, injected = {}) { evidenceDigest: fingerprintInput(identity), requestedSdk: toolchain.sdkVersion, requestedTools: toolchain.toolsVersion, requestedCompilerRuntime: toolchain.runtimeRequirements.find(r => r.role === "compiler" && r.name === "Microsoft.NETCore.App").version, sdk: { version: identity.sdkVersion, sha256: identity.archive.sha256, verified: true }, + archive: identity.archive, compilerRuntime: { version: localCompilerRuntime.version, sha256: identity.runtimes.find(r => r.name === localCompilerRuntime.name && r.version === localCompilerRuntime.version).sha256, diff --git a/eng/regression-bisect/toolchains.test.cjs b/eng/regression-bisect/toolchains.test.cjs index 36815b59899..3e9483d0183 100644 --- a/eng/regression-bisect/toolchains.test.cjs +++ b/eng/regression-bisect/toolchains.test.cjs @@ -5,7 +5,7 @@ const os = require("node:os"); const path = require("node:path"); const { createHash } = require("node:crypto"); const { resolveLatestStableSdk, readRevisionToolchain, acquireToolchain, verifyToolchainIdentity } = require("./toolchains.cjs"); -const { policy, admission, observation, seal, digest } = require("./fixtures/contracts.cjs"); +const { policy, admission, observation, seal, digest, archiveReceipt } = require("./fixtures/contracts.cjs"); const { admitBisection, fingerprintInput } = require("./model.cjs"); const { capture, publicUrl, createHttp, isolatedEnvironment } = require("./toolchain-io.cjs"); @@ -207,22 +207,23 @@ for (const [declarations, expected] of [ } function captured(toolchain, root) { + const runtime = toolchain.bundledRuntime?.version ?? "10.0.0"; return { sdkVersion: toolchain.sdkVersion, architecture: platform.architecture, rid: platform.rid, hostPath: path.join(root, "dotnet.exe"), root, sdkBasePath: path.join(root, "sdk", toolchain.sdkVersion), sdks: [{ version: toolchain.sdkVersion, path: path.join(root, "sdk") }], - runtimes: [{ name: "Microsoft.NETCore.App", version: "10.0.0", path: path.join(root, "shared", "Microsoft.NETCore.App"), sha256: digest("runtime") }], - compilerRuntime: { name: "Microsoft.NETCore.App", version: "10.0.0" }, + runtimes: [{ name: "Microsoft.NETCore.App", version: runtime, path: path.join(root, "shared", "Microsoft.NETCore.App"), sha256: digest("runtime") }], + compilerRuntime: { name: "Microsoft.NETCore.App", version: runtime }, runtimeExecutions: [], files: Object.fromEntries(Object.entries({ compiler: "fsc", fsi: "fsi", core: "FSharp.Core" }).map(([name, assemblyName]) => [name, { path: path.join(root, "sdk", toolchain.sdkVersion, "FSharp", `${assemblyName}.dll`), sha256: digest(name), assemblyName, assemblyVersion: "10.0.0.0", fileVersion: "10.0.0.0", }])), - archive: { ...toolchain.archive, sha256: digest(archiveBytes), actualSha512: hash512(archiveBytes), redirectUrl: toolchain.archive.url }, + archive: archiveReceipt(toolchain.archive, path.join(root, "sdk.zip"), archiveBytes), }; } -for (const [name, edit] of [ +const identityMutations = [ ["wrong architecture", i => i.architecture = "arm64"], ["wrong resolved SDK", i => i.sdkVersion = "11.0.100-rc.1"], ["host SDK leakage", i => i.sdks.push({ version: "11.0.100", path: path.resolve("host-sdk") })], @@ -230,13 +231,16 @@ for (const [name, edit] of [ ["self-asserted host root", i => { i.root = path.resolve("machine"); i.hostPath = path.join(i.root, "dotnet.exe"); }], ["same-version SDK outside root", i => i.sdkBasePath = path.resolve("machine", "sdk", "10.0.401")], ["archive mismatch", i => i.archive.actualSha512 = "0".repeat(128)], + ["wrong archive RID", i => i.archive.rid = "win-arm64"], + ["wrong archive architecture", i => i.archive.architecture = "arm64"], ["wrong archive URL", i => i.archive.url = `${feed}/Sdk/wrong.zip`], ["unapproved redirect destination", i => i.archive.redirectUrl = "https://unapproved.example/sdk.zip"], ["missing runtime", i => i.runtimes = []], ["wrong compiler runtime", i => i.compilerRuntime.version = "9.0.0"], ["outside Core path", i => i.files.core.path = path.resolve("host-core.dll")], ["wrong F# assembly", i => i.files.core.assemblyName = "Other.Assembly"], -]) { +]; +for (const [name, edit] of identityMutations) { test(`identity fails closed: ${name}`, async () => { const { toolchain } = await resolveLatestStableSdk(options(), metadata()); const identity = captured(toolchain, path.resolve("isolated")); @@ -325,6 +329,67 @@ test("unavailable historical SDK records public fallback attempts without runnin assert.ok(result.attempts.some(a => a.status === 404)); }); +for (const [approved, checksum, archiveStatus = 200, ciStatus = archiveStatus] of [ + [true, null], [true, null, 404, 200], [false, null], [false, null, 404], [false, null, 503], [true, "sidecar"], [true, "manifest"], +]) { + test(`observed historical archive: approved=${approved}, checksum=${checksum}, HTTP=${archiveStatus}/${ciStatus}`, async t => { + const version = "11.0.100-rc.1.26420.103"; + const { result: { toolchain } } = await source(t, version); + toolchain.sourceSha = approved ? "5a8b9adb034f420a5c2c1f577670c5eeeb454b7e" : "b".repeat(40); + const url = archive(version).url; + const requests = []; + const deps = checksum === "manifest" ? metadata([["11.0", "sts", [release(version)]]]) : metadata(); + const documents = deps.http; + deps.http = async (requested, settings = {}) => { + requests.push({ url: requested, method: settings.method ?? "GET" }); + if (requested.endsWith(".sha512")) return { status: checksum === "sidecar" ? 200 : 404, url: requested, + redirects: [], bytes: Buffer.from(checksum === "sidecar" ? hash512(archiveBytes) : "") }; + if (requested.endsWith(".zip")) return { + status: requested.startsWith(feed) ? archiveStatus : ciStatus, url: requested, redirects: [], + bytes: archiveBytes, size: archiveBytes.length, + sha256: digest(archiveBytes), sha512: hash512(archiveBytes), + }; + return documents(requested); + }; + deps.installer = async request => { + assert.equal(approved, true, "unapproved bytes must never execute"); + assert.equal(request.archive.sha512, checksum ? hash512(archiveBytes) : undefined, "measured digest is not a publisher checksum"); + assert.equal(request.archive.provenance.kind, checksum ? "publisher-checksum" : "approved-official-https-measured"); + await deps.http(request.archive.url); + const fsharp = path.join(request.root, "sdk", version, "FSharp"); + await fs.mkdir(fsharp, { recursive: true }); + await fs.writeFile(path.join(fsharp, "fsi.runtimeconfig.json"), JSON.stringify({ + runtimeOptions: { framework: { name: "Microsoft.NETCore.App", version: "10.0.0" } }, + })); + await fs.mkdir(path.join(request.root, "shared", "Microsoft.NETCore.App", "10.0.0"), { recursive: true }); + await fs.writeFile(request.archivePath, archiveBytes); + return { attempts: [{ url: request.archive.url, status: 200 }], archive: archiveReceipt(request.archive, request.archivePath, archiveBytes) }; + }; + deps.capture = async request => captured(request.toolchain, request.root); + const result = await acquireToolchain({ policy: policy(), toolchain, storageDirectory: await directory(t) }, deps); + assert.equal(result.status, approved ? "acquired" : "untestable", JSON.stringify(result)); + if (!approved) assert.equal(result.outcome.reason, + archiveStatus === 404 ? "unavailable-sdk" : archiveStatus === 503 ? "network-failure" : "checksum-provenance", JSON.stringify(result)); + else { + assert.equal(result.identity.requested.archive.provenance.kind, checksum ? "publisher-checksum" : "approved-official-https-measured"); + assert.deepEqual(result.identity.archive.provenance, result.identity.requested.archive.provenance); + for (const [name, edit] of identityMutations) { + const changed = structuredClone(result.identity); + edit(changed); + assert.equal(verifyToolchainIdentity({ policy: policy(), toolchain: result.identity.requested, identity: changed }).status, + "untestable", name); + } + } + assert.deepEqual(result.attempts.filter(a => a.url.endsWith(".sha512")).map(a => a.status), + checksum === "manifest" ? [] : checksum === "sidecar" ? [200] : [404, 404]); + assert.ok(requests.some(r => r.url === url), "archive availability is independent of checksum availability"); + if (approved && archiveStatus === 404) { + assert.deepEqual(requests.filter(r => r.url.endsWith(".zip")).map(r => [r.method, new URL(r.url).hostname]), + [["HEAD", "builds.dotnet.microsoft.com"], ["HEAD", "ci.dot.net"], ["GET", "ci.dot.net"]]); + } + }); +} + const forbiddenDownloads = [ "https://ci.dot.net/public/../../private/sdk.zip", "https://ci.dot.net/public/%2e%2e/private/sdk.zip", @@ -421,8 +486,8 @@ test("existing historical SDK search directories cannot leak a same-version mach assert.deepEqual(await fs.readFile(path.join(root, "global.json")), bytes); }); -for (const available of [true, false]) { - test(`explicit supplemental runtime ${available ? "acquired and separated" : "unavailable is untestable"}`, async t => { +for (const [available, bundled] of [[true, false], [false, false], [true, true]]) { + test(`explicit supplemental runtime ${available ? "acquired and separated" : "unavailable is untestable"}, bundled=${bundled}`, async t => { const { root, bytes, result: initial } = await source(t); const sampleConfig = path.join(root, "sample.runtimeconfig.json"); await fs.writeFile(sampleConfig, JSON.stringify({ @@ -443,8 +508,9 @@ for (const available of [true, false]) { installer: async request => { products.push(request.product); if (request.product === "dotnet" && !available) throw new Error("runtime archive HTTP 404"); + if (bundled) await fs.mkdir(path.join(request.root, "shared", "Microsoft.NETCore.App", "9.0.8"), { recursive: true }); return { - archive: { ...request.archive, actualSha512: hash512(archiveBytes), sha256: digest(archiveBytes), redirectUrl: request.archive.url }, + archive: archiveReceipt(request.archive, request.archivePath, archiveBytes), installer: {}, attempts: [{ url: request.archive.url, version: request.version, status: 200 }], }; }, @@ -457,13 +523,19 @@ for (const available of [true, false]) { return identity; }, }); - assert.deepEqual(products, ["sdk", "dotnet"]); + assert.deepEqual(products, bundled ? ["sdk"] : ["sdk", "dotnet"]); assert.deepEqual(await fs.readFile(path.join(root, "global.json")), bytes); if (available) { assert.equal(result.status, "acquired", JSON.stringify(result)); assert.equal(result.identity.compilerRuntime.version, "10.0.0"); assert.equal(result.identity.requested.runtimeRequirements[0].version, "9.0.8"); assert.equal(result.identity.buildToolchain, undefined, "SDK-only acquisition cannot attest a not-yet-built local compiler"); + for (const product of bundled ? ["sdk"] : ["sdk", "dotnet"]) { + const identity = structuredClone(result.identity); + identity.installations = identity.installations.filter(i => i.product !== product); + assert.equal(verifyToolchainIdentity({ policy: policy(), toolchain: identity.requested, identity }).outcome.reason, + "wrong-identity", `missing ${product} receipt must invalidate reuse`); + } } else { assert.equal(result.status, "untestable"); assert.equal(result.outcome.reason, "unavailable-runtime"); @@ -489,7 +561,7 @@ test("historical public preview fallback preserves checksum provenance and exact })); await fs.mkdir(path.join(request.root, "shared", "Microsoft.NETCore.App", "8.0.0-preview.1.1"), { recursive: true }); return { - archive: { ...request.archive, actualSha512: hash512(archiveBytes), sha256: digest(archiveBytes), redirectUrl: request.archive.url }, + archive: archiveReceipt(request.archive, request.archivePath, archiveBytes), installer: {}, attempts: [], }; }, @@ -502,6 +574,7 @@ test("historical public preview fallback preserves checksum provenance and exact assert.equal(result.status, "acquired", JSON.stringify(result)); assert.equal(calls[0].version, toolchain.sdkVersion); assert.ok(calls[0].archive.url.startsWith("https://ci.dot.net/public/")); + assert.equal(calls[0].archive.provenance.kind, "publisher-checksum"); assert.deepEqual(result.attempts.filter(a => a.url.endsWith(".sha512")).map(a => a.status), [404, 200]); }); @@ -594,7 +667,7 @@ async function acquireSource(t, version, sourceSha, p, { rid = p.rid, edit = () installer: async request => { await fs.mkdir(path.join(request.root, "shared", "Microsoft.NETCore.App", runtimeVersion), { recursive: true }); return { - archive: { ...request.archive, actualSha512: hash512(archiveBytes), sha256: digest(archiveBytes), redirectUrl: request.archive.url }, + archive: archiveReceipt(request.archive, request.archivePath, archiveBytes), installer: {}, attempts: [], }; }, diff --git a/eng/regression-bisect/workspace.cjs b/eng/regression-bisect/workspace.cjs index c2eb35d5060..5579d26474f 100644 --- a/eng/regression-bisect/workspace.cjs +++ b/eng/regression-bisect/workspace.cjs @@ -49,7 +49,9 @@ function failure(w, stage, error) { const code = w?.signal?.aborted ? (w.options.signal?.aborted ? "cancelled" : "timeout") : error.code ?? stage; return { status: "untestable", outcome: untestable(code === "timeout" ? "timeout" : - { build: "build-failure", identity: "wrong-identity", sdk: "unavailable-sdk", runtime: "unavailable-runtime" }[stage] ?? "setup-failure"), + ["checksum-provenance", "network-failure"].includes(code) ? code : + stage === "sdk" && ["command-failure", "installation-failure"].includes(code) ? "setup-failure" : + { build: "build-failure", identity: "wrong-identity", sdk: "unavailable-sdk", runtime: "unavailable-runtime" }[stage] ?? "setup-failure"), error: { stage, code, message: error.message }, }; } @@ -211,9 +213,9 @@ async function helper(w) { const tfm = `net${runtime.split(".")[0]}.0`; const output = path.join(directory, "output"); await fs.writeFile(path.join(directory, "NuGet.Config"), - ''); + ''); await invoke(w, "helper", w.toolchain.hostPath, ["build", path.join(directory, "Evidence.csproj"), - "-c", "Release", "-o", output, `-p:TargetFramework=${tfm}`, "-nr:false", "--nologo"], { cwd: directory }); + "-c", "Release", "-o", output, `-p:EvidenceTargetFramework=${tfm}`, "-nr:false", "--nologo"], { cwd: directory }); w.helper = { executable: w.toolchain.hostPath, dll: path.join(output, "Evidence.dll"), runtimeVersion: runtime, hook: path.join(output, "Repro.StartupHook.dll") }; } @@ -268,8 +270,9 @@ async function inventory(w, rows) { files.push({ role: `dependency-${files.length}`, ...await fileIdentity(location, w.sourceDirectory) }); } } - const inspected = await inspect(w, files.map(f => f.path)); - for (const file of files) { + const selected = files.filter(f => ["fsi", "compiler", "core", "tasks", "corePackage"].includes(f.role)); + const inspected = await inspect(w, selected.map(f => f.path)); + for (const file of selected) { const actual = inspected.find(i => path.resolve(i.path) === file.path); requireValue(actual?.sha256 === file.sha256, "artifact inspection digest mismatch"); if (["fsi", "compiler", "core", "tasks"].includes(file.role)) requireValue(actual.assemblyVersion && actual.fileVersion && @@ -317,7 +320,8 @@ async function buildRevision(w, dependencies = {}) { }); await evidence(w, "acquisition.json", acquisition); if (acquisition.outcome?.reason === "unavailable-runtime") stage = "runtime"; - requireValue(acquisition.status === "acquired", acquisition.error?.message ?? "toolchain acquisition failed"); + if (acquisition.status !== "acquired") throw Object.assign(new Error(acquisition.error?.message ?? "toolchain acquisition failed"), + { code: acquisition.error?.code }); w.toolchain = acquisition.identity; const oldRoot = w.env.DOTNET_ROOT; for (const [key, value] of Object.entries(w.env)) if (value === oldRoot) w.env[key] = w.toolchain.root; @@ -412,7 +416,7 @@ async function finalize(w, result) { const outcome = { ...result, buildInstanceId: w.buildInstanceId, actualSourceSha: w.actualSourceSha, requested: w.options.request, adapter: w.adapter, globalJson: w.globalJson, - toolchain: w.toolchain ? { sdkVersion: w.toolchain.sdkVersion, requested: w.declaration, + toolchain: w.toolchain ? { sdkVersion: w.toolchain.sdkVersion, requested: w.toolchain.requested, archive: w.toolchain.archive, acquisitionDigest: fingerprintInput(w.toolchain) } : w.declaration, product: w.product, commands: w.commands, evidence: w.evidence, durationMs: Date.now() - w.started, }; diff --git a/eng/regression-bisect/workspace.smoke.cjs b/eng/regression-bisect/workspace.smoke.cjs index 9e8419bdefb..a007d3e4b7a 100644 --- a/eng/regression-bisect/workspace.smoke.cjs +++ b/eng/regression-bisect/workspace.smoke.cjs @@ -28,7 +28,8 @@ async function main() { }); await fs.writeFile(path.join(root, "result.json"), JSON.stringify(result, null, 2), { flag: "wx" }); console.log(JSON.stringify({ outcome: result.outcome, buildInstanceId: result.buildInstanceId, - actualSourceSha: result.actualSourceSha, error: result.error, result: path.join(root, "result.json") }, null, 2)); + actualSourceSha: result.actualSourceSha, archiveProvenance: result.toolchain?.archive?.provenance, + error: result.error, result: path.join(root, "result.json") }, null, 2)); process.exitCode = result.outcome.outcome === "pass" ? 0 : 2; } diff --git a/eng/regression-bisect/workspace.test.cjs b/eng/regression-bisect/workspace.test.cjs index e477438548d..64a18a776d9 100644 --- a/eng/regression-bisect/workspace.test.cjs +++ b/eng/regression-bisect/workspace.test.cjs @@ -3,7 +3,8 @@ const assert = require("node:assert/strict"); const fs = require("node:fs/promises"); const path = require("node:path"); const { fixture } = require("./fixtures/reproduction.cjs"); -const { seal } = require("./fixtures/contracts.cjs"); +const { seal, archiveReceipt } = require("./fixtures/contracts.cjs"); +const { createHash } = require("node:crypto"); const { fingerprintInput } = require("./model.cjs"); const { hash, isolatedEnvironment, run } = require("./toolchain-io.cjs"); const { execute } = require("./repro-process.cjs"); @@ -70,7 +71,7 @@ async function controlled(t, os = "windows") { const toolchains = { http: async url => { if (url.endsWith("releases-index.json")) return { status: 200, url, bytes: Buffer.from('{"releases-index":[]}') }; - return { status: 200, url, bytes: Buffer.from("a".repeat(128)) }; + return { status: 200, url, bytes: Buffer.from(createHash("sha512").update("archive").digest("hex")) }; }, installer: async request => { activeSdk = request.version; @@ -88,8 +89,8 @@ async function controlled(t, os = "windows") { await fs.mkdir(path.dirname(file), { recursive: true }); await fs.writeFile(file, text); } - return { attempts: [], archive: { ...request.archive, redirectUrl: request.archive.url, - actualSha512: request.archive.sha512, sha256: hash("archive") } }; + await fs.writeFile(request.archivePath, "archive"); + return { attempts: [], archive: archiveReceipt(request.archive, request.archivePath, "archive") }; }, run: async (exe, args) => { const root = path.dirname(exe), runtime = `${activeSdk.split(".")[0]}.0.0`; @@ -132,6 +133,10 @@ async function controlled(t, os = "windows") { return ok(JSON.stringify(result)); } if (args[0] === "msbuild") return ok(JSON.stringify({ Properties: await properties(settings.cwd, args[1]) })); + if (args[0] === "build" && path.basename(args[1]) === "Evidence.csproj") { + f.helperArgs = args; + return ok(""); + } if (args[0] === "pack") { assert.deepEqual(args, ["pack", path.join(settings.cwd, "src", "FSharp.Core", "FSharp.Core.fsproj"), "--configuration", "Release", "--no-build", "--no-restore", @@ -220,6 +225,41 @@ test("fresh revisions use their exact SHA, SDK, TFM and independent same-version await assert.rejects(fs.access(hookMarker)); }); +test("evidence helper selects its runtime TFM consistently for restore and build", async t => { + const f = await controlled(t); + delete f.options.helper; + const result = await testSourceRevision(f.options, f.deps); + assert.equal(result.outcome.outcome, "pass", JSON.stringify(result)); + assert.ok(f.helperArgs.includes("-p:EvidenceTargetFramework=net10.0")); + assert.ok(!f.helperArgs.some(a => a.startsWith("-p:TargetFramework="))); + const helper = path.dirname(f.helperArgs[1]); + assert.match(await fs.readFile(path.join(helper, "Evidence.csproj"), "utf8"), + /\$\(EvidenceTargetFramework\)<\/TargetFramework>/); + assert.match(await fs.readFile(path.join(helper, "NuGet.Config"), "utf8"), /dnceng\/public\/_packaging\/dotnet-public/); +}); + +test("large product inventories hash every dependency without unbounded inspection arguments", async t => { + const f = await controlled(t); + f.afterBuild = async source => { + for (let i = 0; i < 180; i++) await fs.writeFile( + path.join(source, "artifacts", "bin", "fsc", "Release", "net10.0", `dependency-${i}.dll`), `dependency-${i}`); + }; + const execute = f.deps.execute; + const inspected = []; + f.deps.execute = async (exe, args, settings) => { + if (args.includes("inspect")) { + inspected.push(args.slice(args.indexOf("inspect") + 1)); + assert.equal(inspected.at(-1).length, 5, "only selected assemblies and Core package require metadata inspection"); + } + return execute(exe, args, settings); + }; + const result = await testSourceRevision(f.options, f.deps); + assert.equal(result.outcome.outcome, "pass", JSON.stringify(result)); + assert.ok(inspected.length >= 2, "inventory and reproduction both use bounded inspection"); + for (let i = 0; i < 180; i++) assert.equal( + result.product.files.find(f => path.basename(f.path) === `dependency-${i}.dll`).sha256, hash(`dependency-${i}`)); +}); + for (const [name, outputs, outcome, reason] of [ ["expected failure", a => [a.failure, a.failure], "product-failure", "failure-assertion"], ["unexpected diagnostic", a => [a.failure, { ...a.failure, stderr: "different regression\n" }], "untestable", "assertion-mismatch"], @@ -308,6 +348,12 @@ for (const [name, change, reason] of [ ["missing FSI", f => { f.afterBuild = source => fs.rm(path.join(source, "artifacts/bin/fsi/Release/net10.0/fsi.dll")); }, "wrong-identity"], ["changed global.json", f => { f.afterBuild = source => fs.appendFile(path.join(source, "global.json"), " "); }, "wrong-identity"], ["unavailable SDK", f => { f.deps.toolchains.http = async () => { throw new Error("SDK unavailable"); }; }, "unavailable-sdk"], + ["installer process failure", f => { f.deps.toolchains.installer = async () => { + throw Object.assign(new Error("installer process failed"), { code: "command-failure" }); + }; }, "setup-failure"], + ...["checksum-provenance", "network-failure"].map(code => [code, f => { + f.deps.toolchains.http = async () => { throw Object.assign(new Error(code), { code }); }; + }, code]), ]) test(`${name} is untestable, never the reported product failure`, async t => { const f = await controlled(t); change(f); @@ -318,6 +364,8 @@ for (const [name, change, reason] of [ if (f.launchFail) assert.match(result.error.message, /process supervision failed/); if (f.fail) assert.ok(result.evidence.some(e => e.path.endsWith(".binlog"))); assert.ok(result.evidence.every(e => !path.isAbsolute(e.path) && !e.path.split("/").includes("..") && /^[a-f0-9]{64}$/.test(e.sha256))); + const saved = JSON.parse(await fs.readFile(path.join(f.options.evidenceDirectory, result.buildInstanceId, "observation.json"))); + assert.deepEqual(saved.outcome, result.outcome); }); test("confirmation and all phases always allocate independent build instances", async t => { From 044b6d1e4f7144a6209152f356bc91393e2d3090 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 21:42:22 +0200 Subject: [PATCH 22/99] Require complete toolchain receipts and preserve worker identities Validate serialized archive paths using worker OS semantics and retain acquisition failure stages in workspace evidence. Add runtime reuse and cross-host checkpoint regressions, with fresh historical build and released/source acceptance evidence. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 43 ++++++++++++++++- eng/regression-bisect/archive-provenance.cjs | 6 ++- eng/regression-bisect/fixtures/contracts.cjs | 20 +++++++- .../fixtures/reproduction.cjs | 1 + eng/regression-bisect/model.cjs | 3 +- eng/regression-bisect/model.test.cjs | 29 +++++------ eng/regression-bisect/repro.cjs | 2 +- eng/regression-bisect/repro.test.cjs | 48 ++++++++++++++++++- eng/regression-bisect/state.test.cjs | 21 ++++++++ eng/regression-bisect/toolchains.cjs | 30 ++++++------ eng/regression-bisect/toolchains.test.cjs | 14 +++++- eng/regression-bisect/workspace.cjs | 7 +-- eng/regression-bisect/workspace.test.cjs | 18 +++++++ 13 files changed, 195 insertions(+), 47 deletions(-) diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index ef436586538..e24510219f8 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -257,11 +257,21 @@ receipt or claiming a measured digest is a publisher checksum is rejected. Before reproduction reuse, retained archives must still exist as regular, non-linked files and are rehashed; missing or changed bytes invalidate reuse. Read-only file attributes alone are not proof of immutable identity. -An installation list cannot replace or omit the SDK receipt, and supplemental -runtime receipts are verified separately. Receipt/schema and trusted-policy +The installation list is mandatory, even when all required runtimes are bundled +in the SDK. It cannot replace or omit the SDK receipt; each runtime requirement +must name its supplying archive. Supplemental runtime bytes are rehashed before +reuse, independently of the SDK, and missing receipts fail before any reproduction +process starts. Receipt/schema and trusted-policy changes invalidate old checkpoint evidence rather than upgrading old results into measured or publisher-verified claims. +Serialized observation/checkpoint receipts use the worker cell's OS path rules, +not the coordinator's filesystem. Windows receipts therefore round-trip on POSIX +coordinators without rewriting their digested paths. Relative and drive-relative +paths are rejected. Live acquisition and archive rehashing still use the worker's +actual filesystem; accepting a serialized receipt does not prove local byte +availability. + The shared boundaries are injectable: `fs` (`node:fs/promises`), `clock` (`() => Date`), `http(url, { destination?, method? })`, `run(executable, args, { cwd, env, timeoutMs })`, `installer`, and `capture`. HTTP returns status, @@ -452,6 +462,8 @@ their output matches the expected diagnostic. Cancellation terminates process trees and removes only the UUID-marked, resolved owned scratch child; ordinary results retain their scratch artifacts for diagnosis. Results expose no mutation or publisher operation. Controller provenance/admission remains separate. +Acquisition failures retain their original stage, code and message in workspace +evidence: wrong SDK or architecture is `wrong-identity`, not `unavailable-sdk`. Cancellation also finalizes staged `buildRevision` calls. An interruption during final source verification or observation persistence invalidates a completed reproduction, and repeated finalization reuses the same observation. @@ -564,6 +576,33 @@ node eng\regression-bisect\workspace.smoke.cjs --run 5a8b9adb034f420a5c2c1f57767 node eng\regression-bisect\repro.smoke.cjs 'Q:\rb912-s5-20260919-acceptance\verified-products.json' 'Q:\rb912-s5-20260919-acceptance\repro-both-01' ``` +The receipt-completeness, workspace-error and cross-coordinator path repairs were +accepted in a separate fresh run on 2026-09-19, retained under +`Q:\rb912-s5-i2-20260919`. The same pinned source built cleanly in instance +`8c180e7a-a99c-4889-8216-8c8b71d89a27`; its built FSI and the full released/source +project matrix exited 0. All **43 scenarios** matched expectations again: +23 positive passes, 6 intended product failures and 14 identity rejections. +Both SDKs were newly acquired, not copied from the prior run. The audit rehashed +both archives, product inventories, 45 workspace evidence files, 72 project +binlogs (including negative controls), and the frozen released producer DLL/package. +The original failed result and all records in the prior acceptance manifest +remain unchanged. + +The scoped suite passed **872 tests**. Removing supplemental-runtime rehashing +made the otherwise-valid missing/changed-runtime cases fail; restoring it passed. +Eight observation/checkpoint cases also passed with POSIX receipt-path semantics +injected on Windows; reverting to coordinator-local paths failed all four +Windows cases. This is cross-host serialization coverage, not a Linux build smoke. +The evidence helper's exact-SDK Release build and `dotnet test -c Release` command +exited 0; that helper has no managed test cases. All 12 changed scripts passed +syntax checks, formatting changed no files, and whitespace checks passed. + +| Fresh record (relative to `Q:\rb912-s5-i2-20260919`) | SHA-256 | +| --- | --- | +| `fsharp-workspace-smoke-G26zT9\result.json` | `cf5cd20c162c063123886a6900a1630221c5a098dd17c1e21c772580690c6935` | +| `repro-both\smoke-results.json` | `c86d9388032ca912c9a46181398a0922c8cdf06c8c477008f24be6c65fce1d6d` | +| `acceptance-audit.json` | `54098b840f0c1447010a5469045ee3ce997843dc4840cb14a2b6a94ae0b4be62` | + ## Reproduction execution `repro.cjs` is an execution adapter, not an authorization boundary. Its caller diff --git a/eng/regression-bisect/archive-provenance.cjs b/eng/regression-bisect/archive-provenance.cjs index 9af6cecf8bf..86a8b8fa0f1 100644 --- a/eng/regression-bisect/archive-provenance.cjs +++ b/eng/regression-bisect/archive-provenance.cjs @@ -25,7 +25,7 @@ function validateArchiveRequest({ policy, kind, sourceSha, product, version, pla return approval; } -function validateArchiveReceipt(options, receipt) { +function validateArchiveReceipt(options, receipt, filePath = path) { const { fingerprintInput } = require("./model.cjs"); const approval = validateArchiveRequest(options); const { archive, policy } = options; @@ -43,7 +43,9 @@ function validateArchiveReceipt(options, receipt) { typeof receipt.receivedAt === "string" && Number.isFinite(Date.parse(receipt.receivedAt)) && new Date(receipt.receivedAt).toISOString() === receipt.receivedAt, "invalid archive receipt bounds or time"); const retained = receipt.retained; - requireValue(retained && path.isAbsolute(retained.path) && retained.size === receipt.bytesReceived && + requireValue(retained && typeof retained.path === "string" && !/[\r\n\0]/.test(retained.path) && + filePath.isAbsolute(retained.path) && (filePath !== path.win32 || filePath.parse(retained.path).root.length > 1) && + retained.size === receipt.bytesReceived && retained.sha256 === receipt.sha256 && retained.sha512 === receipt.actualSha512, "retained archive identity mismatch"); return true; diff --git a/eng/regression-bisect/fixtures/contracts.cjs b/eng/regression-bisect/fixtures/contracts.cjs index 56bf59e8a43..97ac54284d6 100644 --- a/eng/regression-bisect/fixtures/contracts.cjs +++ b/eng/regression-bisect/fixtures/contracts.cjs @@ -9,6 +9,24 @@ function archiveReceipt(archive, file, bytes) { bytesReceived: Buffer.byteLength(bytes), receivedAt: "2026-09-19T17:30:00.000Z", retained: { path: file, sha256: digest(bytes), sha512, size: Buffer.byteLength(bytes) } }; } +function buildToolchain(sourceSha, os, kind = "publisher-checksum") { + const approval = policy().toolchains.measuredArchives[0]; + const rid = `${{ windows: "win", linux: "linux", macos: "osx" }[os]}-x64`; + const url = approval.url.replace("win-x64.zip", `${rid}${os === "windows" ? ".zip" : ".tar.gz"}`); + const archive = archiveReceipt({ + name: url.split("/").at(-1), url, rid, architecture: "x64", + provenance: kind === "publisher-checksum" ? { kind } : + { kind, approvalRevision: approval.approvalRevision, approvalDigest: model().fingerprintInput(approval) }, + }, os === "windows" ? "Q:\\retained\\sdk.zip" : "/retained/sdk.tar.gz", "archive"); + if (kind === "publisher-checksum") archive.sha512 = archive.actualSha512; + return { + sourceSha, globalJsonDigest: digest("global"), evidenceDigest: digest("receipt"), + requestedSdk: approval.version, requestedTools: approval.version, requestedCompilerRuntime: "11.0.0", + sdk: { version: approval.version, sha256: archive.sha256, verified: true }, + compilerRuntime: { version: "11.0.0", sha256: digest("runtime"), verified: true }, + archive, verified: true, supported: true, + }; +} const model = () => require("../model.cjs"); const seal = m => { const { approvedInputDigest, ...input } = m; @@ -140,4 +158,4 @@ function statusTransport(context) { }; } -module.exports = { policy, manifest, attempt, observation, admission, state, seal, sha, digest, statusTransport, archiveReceipt }; +module.exports = { policy, manifest, attempt, observation, admission, state, seal, sha, digest, statusTransport, archiveReceipt, buildToolchain }; diff --git a/eng/regression-bisect/fixtures/reproduction.cjs b/eng/regression-bisect/fixtures/reproduction.cjs index e5f2c622f4b..e1683370919 100644 --- a/eng/regression-bisect/fixtures/reproduction.cjs +++ b/eng/regression-bisect/fixtures/reproduction.cjs @@ -57,6 +57,7 @@ async function fixture(t, { kind = "source", executable = false, tfms = ["net10. toolchain.archive = archiveReceipt({ name: "sdk.zip", url, rid: "win-x64", architecture: "x64", provenance: { kind: "publisher-checksum" } }, (await file("sdk.zip", "archive")).path, "archive"); toolchain.archive.sha512 = toolchain.archive.actualSha512; + toolchain.installations = [{ product: "sdk", version: "10.0.100", archive: toolchain.archive }]; toolchain.requested = { kind: "released", platform: { os: "windows", architecture: "x64", rid: "win-x64" }, installationRoot: sdkRoot, sdkVersion: "10.0.100", toolsVersion: "10.0.100", diff --git a/eng/regression-bisect/model.cjs b/eng/regression-bisect/model.cjs index 2852a6a91c5..4f2d46270a3 100644 --- a/eng/regression-bisect/model.cjs +++ b/eng/regression-bisect/model.cjs @@ -1,5 +1,6 @@ const { createHash } = require("node:crypto"); const { readFileSync } = require("node:fs"); +const filePath = require("node:path"); const trustedPolicy = JSON.parse(readFileSync(require.resolve("../../.github/policies/regression-bisect.json"), "utf8")); class ContractError extends Error {} @@ -296,7 +297,7 @@ function validateObservation(observation, manifest, options = { finalConfirmatio validateArchiveReceipt({ policy: trustedPolicy, kind: observation.kind, sourceSha: build.sourceSha, product: "sdk", version: build.requestedSdk, platform: { ...cell.environment, rid: build.archive.rid }, archive: build.archive, - }, build.archive); + }, build.archive, cell.environment.os === "windows" ? filePath.win32 : filePath.posix); } catch (error) { throw new ContractError(`observation.cells.buildToolchain.archive: ${error.message}`); } const rid = `${{ windows: "win", linux: "linux", macos: "osx" }[cell.environment.os]}-${cell.environment.architecture}`; check(build.archive.sha256 === build.sdk.sha256 && build.archive.architecture === cell.environment.architecture && diff --git a/eng/regression-bisect/model.test.cjs b/eng/regression-bisect/model.test.cjs index 04f2c43fff1..045bbee41ac 100644 --- a/eng/regression-bisect/model.test.cjs +++ b/eng/regression-bisect/model.test.cjs @@ -4,31 +4,22 @@ const { isEligibleIssue, validatePolicy, validateManifest, fingerprintInput, fingerprintEpoch, classifyAttempts, validateOutcome, validateObservation, validateState, admitBisection, } = require("./model.cjs"); -const { policy, manifest, attempt, observation, admission, state, seal, sha, digest, archiveReceipt } = require("./fixtures/contracts.cjs"); -const path = require("node:path"); +const { policy, manifest, attempt, observation, admission, state, seal, sha, digest, buildToolchain } = require("./fixtures/contracts.cjs"); const rejects = (fn, pattern = /./) => assert.throws(fn, pattern); -for (const kind of ["publisher-checksum", "approved-official-https-measured"]) { - test(`observation serializes and validates ${kind} without relabelling hashes`, () => { +for (const [os, kind] of [ + ["windows", "publisher-checksum"], ["windows", "approved-official-https-measured"], + ["linux", "publisher-checksum"], ["macos", "publisher-checksum"], +]) { + test(`observation serializes and validates ${os} ${kind} without relabelling hashes`, () => { const approval = policy().toolchains.measuredArchives[0]; const m = manifest(); - m.matrix[0].os = "windows"; + m.matrix[0].os = os; seal(m); const o = observation("source", "pass", m, approval.sourceSha); - const archive = archiveReceipt({ - name: path.basename(new URL(approval.url).pathname), url: approval.url, rid: approval.rid, architecture: "x64", - provenance: kind === "publisher-checksum" ? { kind } : - { kind, approvalRevision: approval.approvalRevision, approvalDigest: fingerprintInput(approval) }, - }, path.resolve("retained-sdk.zip"), "archive"); - if (kind === "publisher-checksum") archive.sha512 = archive.actualSha512; - o.cells[0].buildToolchain = { - sourceSha: approval.sourceSha, globalJsonDigest: digest("global"), evidenceDigest: digest("receipt"), - requestedSdk: approval.version, requestedTools: approval.version, requestedCompilerRuntime: "11.0.0", - sdk: { version: approval.version, sha256: archive.sha256, verified: true }, - compilerRuntime: { version: "11.0.0", sha256: digest("runtime"), verified: true }, - archive, verified: true, supported: true, - }; + o.cells[0].buildToolchain = buildToolchain(approval.sourceSha, os, kind); + const { archive } = o.cells[0].buildToolchain; assert.equal(validateObservation(JSON.parse(JSON.stringify(o)), m), true); for (const change of [ a => { a.provenance.kind = "unknown"; }, @@ -38,6 +29,8 @@ for (const kind of ["publisher-checksum", "approved-official-https-measured"]) { a => { a.url = a.url.replace("26420", "26421"); }, a => { a.redirects.push({ from: a.url, to: `${a.url}?other`, status: 302 }); }, a => { a.architecture = "arm64"; }, + ...["relative.zip", "Q:relative.zip", os === "windows" ? "/retained/sdk.zip" : "Q:\\retained\\sdk.zip"] + .map(file => a => { a.retained.path = file; }), ]) { const changed = structuredClone(o); change(changed.cells[0].buildToolchain.archive); diff --git a/eng/regression-bisect/repro.cjs b/eng/regression-bisect/repro.cjs index 96aa230f34b..2ddbbdca0ea 100644 --- a/eng/regression-bisect/repro.cjs +++ b/eng/regression-bisect/repro.cjs @@ -357,7 +357,7 @@ async function runReproduction(options, dependencies = {}) { const verification = verifyToolchainIdentity({ policy, toolchain: toolchain.requested, identity: toolchain }); requireValue(verification.status === "verified", verification.error?.message ?? "unverified acquisition"); await verifyRetainedArchive(toolchain.archive); - for (const installation of toolchain.installations ?? []) + for (const installation of toolchain.installations) if (installation.product !== "sdk") await verifyRetainedArchive(installation.archive); await checkedFile({ path: toolchain.hostPath, sha256: toolchain.hostSha256 }, toolchain.root); requireValue(["source", "released"].includes(options.product.kind) && options.product.buildInstanceId, "unidentified product build"); diff --git a/eng/regression-bisect/repro.test.cjs b/eng/regression-bisect/repro.test.cjs index 221f536571b..84c91e8f9be 100644 --- a/eng/regression-bisect/repro.test.cjs +++ b/eng/regression-bisect/repro.test.cjs @@ -3,7 +3,8 @@ const assert = require("node:assert/strict"); const fs = require("node:fs/promises"); const path = require("node:path"); const { fixture } = require("./fixtures/reproduction.cjs"); -const { seal } = require("./fixtures/contracts.cjs"); +const { seal, archiveReceipt, policy } = require("./fixtures/contracts.cjs"); +const { verifyToolchainIdentity } = require("./toolchains.cjs"); const { selectFsiCommand, verifyProjectIdentities, evaluateAssertion, runReproduction } = require("./repro.cjs"); const { execute } = require("./repro-process.cjs"); const { isolatedEnvironment, hash } = require("./toolchain-io.cjs"); @@ -42,6 +43,51 @@ for (const [changed, installations] of [[false, undefined], [true, undefined], [ }); } +for (const state of ["intact", "missing", "changed", "absent-list", "missing-receipt"]) { + test(`supplemental runtime archive reuse: ${state}`, async t => { + const f = await runnerFixture(t); + const bytes = "runtime archive"; + const receipt = archiveReceipt({ + name: "runtime.zip", + url: "https://builds.dotnet.microsoft.com/dotnet/Runtime/9.0.8/dotnet-runtime-9.0.8-win-x64.zip", + rid: "win-x64", architecture: "x64", provenance: { kind: "publisher-checksum" }, + }, (await f.file("runtime.zip", bytes)).path, bytes); + receipt.sha512 = receipt.actualSha512; + f.toolchain.installations.push({ product: "dotnet", version: "9.0.8", archive: receipt }); + const config = Buffer.from(JSON.stringify({ + runtimeOptions: { framework: { name: "Microsoft.NETCore.App", version: "9.0.8" } }, + })); + f.toolchain.requested.runtimeRequirements.push({ + name: "Microsoft.NETCore.App", version: "9.0.8", role: "sample", archiveUrl: receipt.url, + config: { ...await f.file("sample.runtimeconfig.json", config), bytesBase64: config.toString("base64") }, + }); + const runtime = await f.file("dotnet/shared/Microsoft.NETCore.App/9.0.8/System.Private.CoreLib.dll"); + f.toolchain.runtimes.push({ + name: "Microsoft.NETCore.App", version: "9.0.8", path: path.dirname(path.dirname(runtime.path)), sha256: runtime.sha256, + }); + assert.equal(verifyToolchainIdentity({ policy: policy(), toolchain: f.toolchain.requested, identity: f.toolchain }).status, "verified"); + if (state === "missing" || state === "absent-list") await fs.rm(receipt.retained.path); + if (state === "changed") await fs.writeFile(receipt.retained.path, "Runtime archive"); + if (state === "absent-list") delete f.toolchain.installations; + if (state === "missing-receipt") f.toolchain.installations.pop(); + let executions = 0; + const result = await runReproduction(f, { execute: (...args) => { executions++; return f.execute(...args); } }); + assert.equal(result.outcome.outcome, state === "intact" ? "pass" : "untestable", JSON.stringify(result)); + if (state === "intact") { + assert.equal(result.cells[0].attempts.length, 2); + assert.ok(executions > 0); + } else { + assert.equal(result.outcome.reason, "wrong-identity"); + assert.match(result.error, { + missing: /ENOENT/, changed: /digest or size/, + "absent-list": /missing installation receipts/, "missing-receipt": /missing required runtime archive receipt/, + }[state]); + assert.equal(executions, 0); + assert.equal(result.workDirectory, undefined); + } + }); +} + for (const [name, mutate, options] of [ ["wrong host", f => { f.toolchain.hostPath = path.join(f.root, "dotnet-other", "dotnet.exe"); }], ["host dotnet fsi", f => { f.product.files.find(x => x.role === "fsi").path = "fsi"; }], diff --git a/eng/regression-bisect/state.test.cjs b/eng/regression-bisect/state.test.cjs index fde13ce06af..746e35c2848 100644 --- a/eng/regression-bisect/state.test.cjs +++ b/eng/regression-bisect/state.test.cjs @@ -281,6 +281,27 @@ test("commit binds exact request, build identity, owner and observation; duplica await assert.rejects(reserve(h, 1, "next"), /generation/); }); +for (const [os, kind] of [ + ["windows", "publisher-checksum"], ["windows", "approved-official-https-measured"], + ["linux", "publisher-checksum"], ["macos", "publisher-checksum"], +]) test(`checkpoint roundtrip preserves ${os} ${kind} worker receipt paths`, async () => { + const h = harness(); + const { context } = h; + context.manifest.matrix[0].os = os; + f.seal(context.manifest); + context.authorization.manifestDigest = model.fingerprintInput(context.manifest); + context.authorization.inputDigest = context.manifest.approvedInputDigest; + context.stableSdk.archives[0].os = os; + context.pinnedMainSha = f.policy().toolchains.measuredArchives[0].sourceSha; + await h.start(); + const pending = await reserve(h, 0, "receipt-roundtrip", request(context.pinnedMainSha)); + const done = completion(h, pending); + done.observation.cells[0].buildToolchain = f.buildToolchain(context.pinnedMainSha, os, kind); + await commitObservation(h.transport, context, done); + const { checkpoint: loaded } = await loadCheckpoint(h.transport, context); + assert.deepEqual(loaded.observations[model.fingerprintInput(done.observation)].data, done.observation); +}); + test("a cancelled owner cannot commit a late worker result", async () => { const h = harness(); await h.start(); const pending = await reserve(h); diff --git a/eng/regression-bisect/toolchains.cjs b/eng/regression-bisect/toolchains.cjs index c78de250d2a..4d5ba408d79 100644 --- a/eng/regression-bisect/toolchains.cjs +++ b/eng/regression-bisect/toolchains.cjs @@ -301,23 +301,21 @@ function verifyToolchainIdentity({ policy, toolchain, identity }) { requireValue(identity.sdks.length === 1 && identity.sdks[0].version === toolchain.sdkVersion && identity.sdks[0].path === path.join(root, "sdk"), "unexpected installed SDK or SDK location"); validateArchiveReceipt({ policy, ...toolchain, version: toolchain.sdkVersion, product: "sdk" }, identity.archive); - if (Object.hasOwn(identity, "installations")) { - requireValue(Array.isArray(identity.installations), "missing installation receipts"); - const sdks = identity.installations.filter(i => i.product === "sdk"); - requireValue(sdks.length === 1 && sdks[0].version === toolchain.sdkVersion && - fingerprintInput(sdks[0].archive) === fingerprintInput(identity.archive), "missing or mismatched SDK installation receipt"); - const runtimes = identity.installations.filter(i => i.product !== "sdk"); - for (const installed of runtimes) { - requireValue(toolchain.runtimeRequirements.some(r => runtimeProducts[r.name] === installed.product && r.version === installed.version), - "unexpected supplemental installation receipt"); - validateArchiveReceipt({ policy, ...toolchain, version: installed.version, product: installed.product, - archive: installed.archive }, installed.archive); - } - for (const required of toolchain.runtimeRequirements) - requireValue(identity.installations.some(i => i.archive.url === required.archiveUrl && - (i.product === "sdk" || i.product === runtimeProducts[required.name] && i.version === required.version)), - "missing required runtime archive receipt"); + requireValue(Array.isArray(identity.installations), "missing installation receipts"); + const sdks = identity.installations.filter(i => i.product === "sdk"); + requireValue(sdks.length === 1 && sdks[0].version === toolchain.sdkVersion && + fingerprintInput(sdks[0].archive) === fingerprintInput(identity.archive), "missing or mismatched SDK installation receipt"); + const runtimes = identity.installations.filter(i => i.product !== "sdk"); + for (const installed of runtimes) { + requireValue(toolchain.runtimeRequirements.some(r => runtimeProducts[r.name] === installed.product && r.version === installed.version), + "unexpected supplemental installation receipt"); + validateArchiveReceipt({ policy, ...toolchain, version: installed.version, product: installed.product, + archive: installed.archive }, installed.archive); } + for (const required of toolchain.runtimeRequirements) + requireValue(identity.installations.some(i => i.archive.url === required.archiveUrl && + (i.product === "sdk" || i.product === runtimeProducts[required.name] && i.version === required.version)), + "missing required runtime archive receipt"); for (const [name, assemblyName] of Object.entries({ compiler: "fsc", fsi: "fsi", core: "FSharp.Core" })) { const file = identity.files[name]; requireValue(file && inside(path.join(root, "sdk", toolchain.sdkVersion, "FSharp"), file.path) && diff --git a/eng/regression-bisect/toolchains.test.cjs b/eng/regression-bisect/toolchains.test.cjs index 3e9483d0183..32047e4a057 100644 --- a/eng/regression-bisect/toolchains.test.cjs +++ b/eng/regression-bisect/toolchains.test.cjs @@ -208,6 +208,7 @@ for (const [declarations, expected] of [ function captured(toolchain, root) { const runtime = toolchain.bundledRuntime?.version ?? "10.0.0"; + const receipt = archiveReceipt(toolchain.archive, path.join(root, "sdk.zip"), archiveBytes); return { sdkVersion: toolchain.sdkVersion, architecture: platform.architecture, rid: platform.rid, hostPath: path.join(root, "dotnet.exe"), root, sdkBasePath: path.join(root, "sdk", toolchain.sdkVersion), @@ -219,11 +220,13 @@ function captured(toolchain, root) { path: path.join(root, "sdk", toolchain.sdkVersion, "FSharp", `${assemblyName}.dll`), sha256: digest(name), assemblyName, assemblyVersion: "10.0.0.0", fileVersion: "10.0.0.0", }])), - archive: archiveReceipt(toolchain.archive, path.join(root, "sdk.zip"), archiveBytes), + archive: receipt, + installations: [{ product: "sdk", version: toolchain.sdkVersion, archive: receipt }], }; } const identityMutations = [ + ["absent installation receipts", i => { delete i.installations; }], ["wrong architecture", i => i.architecture = "arm64"], ["wrong resolved SDK", i => i.sdkVersion = "11.0.100-rc.1"], ["host SDK leakage", i => i.sdks.push({ version: "11.0.100", path: path.resolve("host-sdk") })], @@ -536,6 +539,10 @@ for (const [available, bundled] of [[true, false], [false, false], [true, true]] assert.equal(verifyToolchainIdentity({ policy: policy(), toolchain: identity.requested, identity }).outcome.reason, "wrong-identity", `missing ${product} receipt must invalidate reuse`); } + const identity = structuredClone(result.identity); + delete identity.installations; + assert.equal(verifyToolchainIdentity({ policy: policy(), toolchain: identity.requested, identity }).outcome?.reason, + "wrong-identity", "absent installation list must invalidate reuse"); } else { assert.equal(result.status, "untestable"); assert.equal(result.outcome.reason, "unavailable-runtime"); @@ -747,7 +754,10 @@ for (const [p, hostRid] of [ assert.equal(digest(await fs.readFile(execution.hostTrace.path)), execution.hostTrace.sha256); } assert.deepEqual(JSON.parse(await fs.readFile(result.evidencePath)).identity, result.identity); - a[key].cells[0].buildToolchain = buildToolchain; + const serialized = structuredClone(buildToolchain); + // The injected worker uses local files; serialized paths belong to its declared OS. + serialized.archive.retained.path = p.os === "windows" ? "Q:\\retained\\sdk.zip" : "/retained/sdk.tar.gz"; + a[key].cells[0].buildToolchain = serialized; } a.budget.authenticatedObservationDigests = [a.stable, a.main, a.good].map(fingerprintInput); const admitted = admitBisection(a); diff --git a/eng/regression-bisect/workspace.cjs b/eng/regression-bisect/workspace.cjs index 5579d26474f..88e4830a785 100644 --- a/eng/regression-bisect/workspace.cjs +++ b/eng/regression-bisect/workspace.cjs @@ -319,9 +319,10 @@ async function buildRevision(w, dependencies = {}) { }, }); await evidence(w, "acquisition.json", acquisition); - if (acquisition.outcome?.reason === "unavailable-runtime") stage = "runtime"; - if (acquisition.status !== "acquired") throw Object.assign(new Error(acquisition.error?.message ?? "toolchain acquisition failed"), - { code: acquisition.error?.code }); + if (acquisition.status !== "acquired") { + stage = acquisition.error.stage; + throw Object.assign(new Error(acquisition.error.message), { code: acquisition.error.code }); + } w.toolchain = acquisition.identity; const oldRoot = w.env.DOTNET_ROOT; for (const [key, value] of Object.entries(w.env)) if (value === oldRoot) w.env[key] = w.toolchain.root; diff --git a/eng/regression-bisect/workspace.test.cjs b/eng/regression-bisect/workspace.test.cjs index 64a18a776d9..d64dd72f054 100644 --- a/eng/regression-bisect/workspace.test.cjs +++ b/eng/regression-bisect/workspace.test.cjs @@ -354,6 +354,17 @@ for (const [name, change, reason] of [ ...["checksum-provenance", "network-failure"].map(code => [code, f => { f.deps.toolchains.http = async () => { throw Object.assign(new Error(code), { code }); }; }, code]), + ...[ + ["wrong acquired SDK", args => args[0] === "--version", "9.0.100"], + ["wrong acquired architecture", args => args.some(a => path.basename(a) === "identity.fsx"), "ARCH|arm64"], + ].map(([name, matches, replacement]) => [name, f => { + const run = f.deps.toolchains.run; + f.deps.toolchains.run = async (exe, args, settings) => { + const result = await run(exe, args, settings); + return matches(args) ? { ...result, stdout: replacement === "ARCH|arm64" + ? result.stdout.replace("ARCH|x64", replacement) : replacement } : result; + }; + }, "wrong-identity"]), ]) test(`${name} is untestable, never the reported product failure`, async t => { const f = await controlled(t); change(f); @@ -363,6 +374,13 @@ for (const [name, change, reason] of [ assert.ok(result.error); if (f.launchFail) assert.match(result.error.message, /process supervision failed/); if (f.fail) assert.ok(result.evidence.some(e => e.path.endsWith(".binlog"))); + if (name.startsWith("wrong acquired")) { + assert.equal(result.error.stage, "identity"); + assert.equal(f.reproductions, undefined); + const acquisition = JSON.parse(await fs.readFile(path.join(f.options.evidenceDirectory, result.buildInstanceId, "acquisition.json"))); + assert.deepEqual(result.outcome, acquisition.outcome); + assert.deepEqual(result.error, acquisition.error); + } assert.ok(result.evidence.every(e => !path.isAbsolute(e.path) && !e.path.split("/").includes("..") && /^[a-f0-9]{64}$/.test(e.sha256))); const saved = JSON.parse(await fs.readFile(path.join(f.options.evidenceDirectory, result.buildInstanceId, "observation.json"))); assert.deepEqual(saved.outcome, result.outcome); From 06f9beb7aa4300ba1fed91f160e9fce0b45cfd66 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 22:32:11 +0200 Subject: [PATCH 23/99] Preserve network failures through observation checkpoints Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 35 ++++++++++++++++++++ eng/regression-bisect/fixtures/contracts.cjs | 2 +- eng/regression-bisect/model.cjs | 2 +- eng/regression-bisect/model.test.cjs | 27 +++++++++++---- eng/regression-bisect/repro.cjs | 2 +- eng/regression-bisect/state.test.cjs | 25 ++++++++++++-- 6 files changed, 81 insertions(+), 12 deletions(-) diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index e24510219f8..e8fa5a5a202 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -216,6 +216,14 @@ as released stable. `checksum-provenance` means an available archive lacks approved provenance or fails its integrity checks; it is distinct from `unavailable-sdk`/`unavailable-runtime` (missing archives) and `network-failure`. None is a positive reproduction. +Attempt evidence requires `checks.network`: a failed acquisition request sets it +to `false`, preserving `network-failure` even when checksum, setup and product +identity checks could not complete. It takes precedence over those checks, never +overrides a failure into a pass, and survives observation validation and +authenticated checkpoint reload with the same digest and skip reason. Successful +acquisition sets it to `true`; missing or non-boolean checks are incomplete +evidence, not implicitly successful network requests. Older observations lacking +this required check are rejected rather than silently upgraded. Exact `global.json.tools.runtimes` arrays (`dotnet`, `aspnetcore`, `windowsdesktop`, optionally suffixed with the matching architecture) become separate build-runtime requirements. Unevaluated MSBuild version expressions @@ -603,6 +611,33 @@ syntax checks, formatting changed no files, and whitespace checks passed. | `repro-both\smoke-results.json` | `c86d9388032ca912c9a46181398a0922c8cdf06c8c477008f24be6c65fce1d6d` | | `acceptance-audit.json` | `54098b840f0c1447010a5469045ee3ce997843dc4840cb14a2b6a94ae0b4be62` | +The network-failure observation repair was verified in another fresh run on +2026-09-19 under `Q:\rb912-s5-i3-20260919`. Build instance +`cd911f1f-8f3e-4584-8a81-98bf1339fdf2` cleanly built the same pinned historical +source and passed both built-FSI repetitions. A separate acquisition again +resolved publisher-verified stable SDK `10.0.401` / runtime `10.0.12`. +The released/source smoke exited 0 with **43 expected outcomes**: 23 passes, +6 intended product failures and 14 identity rejections, including real multi-TFM +execution and frozen released-producer consumption. The audit rehashed both +archives, 209 product files, 45 workspace evidence files, 72 project binlogs and +the producer DLL/package, and verified both prior acceptance manifests and the +original failed result were unchanged. + +All **879 helper tests** passed. The network/checksum cases round-trip through +observation validation and authenticated checkpoint commit/reload with absent +product identities, preserving their reason, digest and charged budget. +Relabelling network failures as unavailable SDKs in memory failed both network +regressions. Changed-script syntax and whitespace checks passed. The exact-SDK +Release evidence-helper build/test command exited 0 (no managed test cases). +The initial formatter command lacked the root's SDK; rerunning with the freshly +acquired exact historical SDK succeeded and changed no files. + +| Fresh record (relative to `Q:\rb912-s5-i3-20260919`) | SHA-256 | +| --- | --- | +| `fsharp-workspace-smoke-Np6g88\result.json` | `1d4200a191ca02a5ad9561016fde06468c4d3e0af76168e1b77a5019f69cd4f2` | +| `repro-both\smoke-results.json` | `3450faeb7139f429d7d49eb2457f4e43630a27f16a6beae2e29a2940c6314691` | +| `acceptance-audit.json` | `19881e6269b707bd9d06e1e0df031bcf4ab36e434a8c913a307b2fd1ed1b27ce` | + ## Reproduction execution `repro.cjs` is an execution adapter, not an authorization boundary. Its caller diff --git a/eng/regression-bisect/fixtures/contracts.cjs b/eng/regression-bisect/fixtures/contracts.cjs index 97ac54284d6..10ad04425b6 100644 --- a/eng/regression-bisect/fixtures/contracts.cjs +++ b/eng/regression-bisect/fixtures/contracts.cjs @@ -57,7 +57,7 @@ function manifest() { function attempt(outcome) { return { schemaVersion: 1, - checks: { restore: true, build: true, setup: true, identity: true, sdk: true, runtime: true, checksumProvenance: true }, + checks: { restore: true, build: true, setup: true, identity: true, sdk: true, runtime: true, checksumProvenance: true, network: true }, execution: { status: "completed", complete: true, ...structuredClone(manifest().assertions[outcome === "pass" ? "pass" : "failure"]), diff --git a/eng/regression-bisect/model.cjs b/eng/regression-bisect/model.cjs index 4f2d46270a3..5a7007acdc6 100644 --- a/eng/regression-bisect/model.cjs +++ b/eng/regression-bisect/model.cjs @@ -175,7 +175,7 @@ function validateOutcome(outcome) { const outcome = (value, reason) => ({ schemaVersion: 1, outcome: value, reason }); const untestable = reason => outcome("untestable", reason); const checks = { - checksumProvenance: "checksum-provenance", + network: "network-failure", checksumProvenance: "checksum-provenance", restore: "restore-failure", build: "build-failure", setup: "setup-failure", identity: "wrong-identity", sdk: "unavailable-sdk", runtime: "unavailable-runtime", }; diff --git a/eng/regression-bisect/model.test.cjs b/eng/regression-bisect/model.test.cjs index 045bbee41ac..1bf899e8db0 100644 --- a/eng/regression-bisect/model.test.cjs +++ b/eng/regression-bisect/model.test.cjs @@ -43,23 +43,35 @@ for (const [os, kind] of [ }); } -test("checksum provenance is a blocked outcome, not missing SDK or a passing attempt", () => { +for (const [check, reason] of [["network", "network-failure"], ["checksumProvenance", "checksum-provenance"]]) +test(`${reason} is a blocked outcome, not missing SDK or a passing attempt`, () => { const m = manifest(), a = attempt("pass"); - a.checks.checksumProvenance = false; - const blocked = { schemaVersion: 1, outcome: "untestable", reason: "checksum-provenance" }; + a.checks[check] = false; + const blocked = { schemaVersion: 1, outcome: "untestable", reason }; assert.deepEqual(classifyAttempts([a, a], m), blocked); const o = observation("source", "pass", m); const cell = o.cells[0]; cell.sdk = cell.runtime = null; cell.artifacts = Object.fromEntries(Object.keys(cell.artifacts).map(name => [name, null])); a.checks = Object.fromEntries(Object.keys(a.checks).map(name => [name, false])); - a.execution = { status: "exception", complete: false, exitCode: null, stdout: "", stderr: "checksum provenance absent" }; + a.checks.network = check !== "network"; + a.execution = { status: "exception", complete: false, exitCode: null, stdout: "", stderr: reason }; cell.attempts = [a, a]; cell.outcome = o.outcome = blocked; assert.deepEqual(classifyAttempts(cell.attempts, m), blocked); assert.equal(validateObservation(JSON.parse(JSON.stringify(o)), m), true); - assert.equal(validateOutcome({ schemaVersion: 1, outcome: "untestable", reason: "network-failure" }), true); - rejects(() => validateOutcome({ schemaVersion: 1, outcome: "pass", reason: "checksum-provenance" })); + assert.equal(validateOutcome(blocked), true); + for (const value of ["pass", "product-failure"]) { + rejects(() => validateOutcome({ ...blocked, outcome: value })); + const changed = structuredClone(o); + changed.cells[0].outcome = changed.outcome = { + schemaVersion: 1, outcome: value, reason: value === "pass" ? "positive-assertion" : "failure-assertion", + }; + rejects(() => validateObservation(changed, m), /absent identities/); + } + const changed = structuredClone(o); + changed.cells[0].attempts.forEach(a => { a.checks[check] = true; }); + rejects(() => validateObservation(changed, m), /does not match repeated evidence/); }); for (const [name, patch, expected] of [ @@ -190,6 +202,9 @@ for (const [name, results, edit, expected, reason] of [ ["nonzero is not assertion", ["product-failure", "product-failure"], a => a.execution.stderr = "unrelated", "untestable", "assertion-mismatch"], ["incomplete output", ["pass", "pass"], a => a.execution.complete = false, "untestable", "incomplete-evidence"], ["missing evidence", ["pass", "pass"], a => delete a.execution, "untestable", "incomplete-evidence"], + ["missing network check", ["pass", "pass"], a => delete a.checks.network, "untestable", "incomplete-evidence"], + ...[null, "true", 1].map(value => [`invalid network check ${value}`, ["pass", "pass"], + a => a.checks.network = value, "untestable", "incomplete-evidence"]), ["one repetition", ["pass"], () => {}, "untestable", "incomplete-evidence"], ]) { test(`classification: ${name}`, () => { diff --git a/eng/regression-bisect/repro.cjs b/eng/regression-bisect/repro.cjs index 2ddbbdca0ea..8dcbf14d797 100644 --- a/eng/regression-bisect/repro.cjs +++ b/eng/regression-bisect/repro.cjs @@ -77,7 +77,7 @@ async function selectFsiCommand({ product, toolchain, cell }) { const compileAssertion = manifest => /(?:^|\n)[^\n]+\(\d+,\d+(?:,\d+,\d+)?\): error FS\d{4}: [^\n]+/.test(manifest.assertions.failure.stderr); const attemptFor = ({ status, complete, exitCode, stdout, stderr }) => ({ - schemaVersion: 1, checks: { restore: true, build: true, setup: true, identity: true, sdk: true, runtime: true, checksumProvenance: true }, + schemaVersion: 1, checks: { restore: true, build: true, setup: true, identity: true, sdk: true, runtime: true, checksumProvenance: true, network: true }, execution: { status, complete, exitCode, stdout, stderr }, }); diff --git a/eng/regression-bisect/state.test.cjs b/eng/regression-bisect/state.test.cjs index 746e35c2848..5117d6a8990 100644 --- a/eng/regression-bisect/state.test.cjs +++ b/eng/regression-bisect/state.test.cjs @@ -637,13 +637,32 @@ test("observation size overflow leaves the charged reservation durable and block assert.deepEqual(cp.observations, {}); }); -test("untestable observations retain ordered tested and skipped reasons and full evidence", async () => { +for (const reason of ["timeout", "network-failure", "checksum-provenance"]) +test(`${reason} observations retain tested and skipped reasons through checkpoint reload`, async () => { const h = harness(); await h.start(); const p = await reserve(h); const done = completion(h, p, "untestable"); + if (reason !== "timeout") { + const cell = done.observation.cells[0]; + cell.sdk = cell.runtime = null; + cell.artifacts = Object.fromEntries(Object.keys(cell.artifacts).map(name => [name, null])); + for (const attempt of cell.attempts) { + attempt.checks = Object.fromEntries(Object.keys(attempt.checks).map(name => [name, false])); + attempt.checks.network = reason !== "network-failure"; + attempt.execution = { status: "exception", complete: false, exitCode: null, stdout: "", stderr: reason }; + } + cell.outcome = done.observation.outcome = { schemaVersion: 1, outcome: "untestable", reason }; + } const cp = await commitObservation(h.transport, h.context, done); - assert.equal(cp.state.skips[0].reason, "timeout"); - assert.equal(cp.state.skips[0].observationDigest, cp.state.testedRevisions[0].observationDigest); + const digest = model.fingerprintInput(done.observation); + assert.equal(cp.state.skips[0].reason, reason); + assert.equal(cp.state.skips[0].observationDigest, digest); + assert.equal(cp.state.testedRevisions[0].observationDigest, digest); + const loaded = (await loadCheckpoint(h.transport, h.context)).checkpoint; + assert.deepEqual(loaded, cp); + assert.deepEqual(loaded.observations[digest].data, done.observation); + assert.equal(loaded.pending, null); + assert.deepEqual(loaded.state.budget.consumed, { sourceBuilds: 1, workerUnits: 1, reservedMinutes: 110 }); }); test("changed approval, reproduction, producer and orchestration invalidate dependent pending work", async t => { From c400adbe1352ce28f822c8e1e4cd3112e1173b05 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 19 Sep 2026 23:30:32 +0200 Subject: [PATCH 24/99] Consolidate measured archive digest rejection coverage Retain checksum-provenance, SHA256 diagnostic and no-process assertions in the existing parameterized wrong-digest case. Record fresh clean-workspace and released/source acceptance. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 15 ++++++++++++ .../toolchain-installer.test.cjs | 23 ++++--------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index e8fa5a5a202..505a2a128bc 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -638,6 +638,21 @@ acquired exact historical SDK succeeded and changed no files. | `repro-both\smoke-results.json` | `3450faeb7139f429d7d49eb2457f4e43630a27f16a6beae2e29a2940c6314691` | | `acceptance-audit.json` | `19881e6269b707bd9d06e1e0df031bcf4ab36e434a8c913a307b2fd1ed1b27ce` | +The installer tests were consolidated and revalidated on 2026-09-19: the existing +wrong-SHA256 table row now also requires `checksum-provenance` and the SHA256 +diagnostic, retaining its no-process-execution assertion. Removing the duplicate +case reduced the suite to **878 passing tests** without removing a scenario. +In-memory mutations of the error code and message each failed that row. +Fresh acceptance is retained under `Q:\rb912-s5-i4-20260919`: clean build +`03aaca4c-f4cc-4ffd-8da6-90ff66775f9c` and both built-FSI executions passed; +all 43 released/source scenarios matched their expected outcomes (23 passes, +6 intended product failures, 14 identity rejections). The exact-SDK Release +evidence-helper build/test commands, syntax, formatting and whitespace checks +passed. The audit rehashed both archives, 209 product files, 45 workspace evidence +files and 72 project binlogs; all three prior acceptance manifests and the original +failed result remain unchanged. `acceptance-audit.json` has SHA-256 +`cae5825442f29472f95a4490f983926e52b23a01cbfe5a0631017f82b50959b4`. + ## Reproduction execution `repro.cjs` is an execution adapter, not an authorization boundary. Its caller diff --git a/eng/regression-bisect/toolchain-installer.test.cjs b/eng/regression-bisect/toolchain-installer.test.cjs index 4261809ab92..faf3b30645b 100644 --- a/eng/regression-bisect/toolchain-installer.test.cjs +++ b/eng/regression-bisect/toolchain-installer.test.cjs @@ -181,21 +181,6 @@ async function measuredFixture(t) { return result; } -test('measured byte mismatch reports checksum provenance, not archive absence', async t => { - const { options, deps } = await measuredFixture(t); - const transport = deps.http; - deps.http = async (...args) => { - const response = await transport(...args); - if (args[1].destination) response.sha256 = '0'.repeat(64); - return response; - }; - await assert.rejects(installArchive(options, deps), error => { - assert.match(error.message, /HTTP sha256 digest/); - assert.equal(error.code, 'checksum-provenance'); - return true; - }); -}); - for (const conflicting of [false, true]) { test(`Windows installer canonicalizes environment aliases: conflicting=${conflicting}`, async t => { const { options, deps, state } = await fixture(t); @@ -272,10 +257,12 @@ for (const [name, mutate] of [ }); } -for (const [name, mutate] of [ +for (const [name, mutate, expectedError] of [ ['missing sha256', r => { delete r.sha256; }], ['missing sha512', r => { delete r.sha512; }], - ['wrong sha256', r => { r.sha256 = '0'.repeat(64); }], + ['wrong sha256', r => { r.sha256 = '0'.repeat(64); }, { + code: 'checksum-provenance', message: /HTTP sha256 digest/, + }], ['wrong sha512', r => { r.sha512 = '0'.repeat(128); }], ['missing size', r => { delete r.size; }], ['truncated bytes', r => { r.size++; }], @@ -302,7 +289,7 @@ for (const [name, mutate] of [ if (args[1].destination) mutate(response); return response; }; - await assert.rejects(installArchive(options, deps)); + await assert.rejects(installArchive(options, deps), expectedError); assert.equal(state.calls.length, 0); }); } From ec2a1b9b88d485df91731375ad534df1b0cbb5d7 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sun, 20 Sep 2026 00:19:44 +0200 Subject: [PATCH 25/99] Preserve checksum provenance for rejected archive redirects Classify synchronous URL and redirect validation failures without conflating them with unavailable archives or network failures. Cover native HTTP and injected availability responses, and record fresh clean-workspace and released/source smoke acceptance. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 23 +++++- .../toolchain-installer.test.cjs | 4 +- eng/regression-bisect/toolchain-io.cjs | 27 ++++--- eng/regression-bisect/toolchains.cjs | 4 +- eng/regression-bisect/toolchains.test.cjs | 74 +++++++++++++++++++ 5 files changed, 117 insertions(+), 15 deletions(-) diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index 505a2a128bc..724b4fd3e26 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -212,8 +212,13 @@ attempts }`, not reproduction pass/failure. Missing archives, insufficient checksum provenance, network failures, unsupported installer flags or declarations, SDK/tool declaration disagreement, changed source files, and unfulfilled runtime requirements fail closed. Historical previews are allowed, but never selected -as released stable. `checksum-provenance` means an available archive lacks -approved provenance or fails its integrity checks; it is distinct from +as released stable. `checksum-provenance` means an archive lacks +approved provenance or fails its integrity checks. It also covers rejected +download URLs or redirect chains, even when an availability HEAD probe cannot +establish whether the archive exists. Native HTTP and injected response +validation preserve this reason; they do not report a forbidden redirect as a +missing SDK. Network operations remain outside that validation boundary. +It is distinct from `unavailable-sdk`/`unavailable-runtime` (missing archives) and `network-failure`. None is a positive reproduction. Attempt evidence requires `checks.network`: a failed acquisition request sets it @@ -653,6 +658,20 @@ files and 72 project binlogs; all three prior acceptance manifests and the origi failed result remain unchanged. `acceptance-audit.json` has SHA-256 `cae5825442f29472f95a4490f983926e52b23a01cbfe5a0631017f82b50959b4`. +The redirect-provenance repair was revalidated under `Q:\rb912-s5-i5-20260919`. +Before the fix, 32 native/injected HTTP provenance assertions failed while the +three archive-absence/network controls passed; all now pass. The full suite has +**898 passing tests**, with zero failures or skips. Fresh clean build +`a03fc535-eaeb-4389-87cd-e7300655dc8a` and both built-FSI executions passed for the +same pinned historical source. All 43 released/source smoke scenarios matched +expectations: 23 passes, 6 intended product failures and 14 identity rejections. +The audit rehashed both archives, 209 product files, 45 workspace evidence files +and 72 project binlogs, and confirmed all four prior acceptance manifests and +the original failed result remain unchanged. Exact-SDK Release evidence-helper +build/test commands, syntax, formatting and whitespace checks passed. +`acceptance-audit.json` has SHA-256 +`c426ad8a5fbeb65aa0750d00088eaba77051a57c4be112ceb94e47112a9ae8ab`. + ## Reproduction execution `repro.cjs` is an execution adapter, not an authorization boundary. Its caller diff --git a/eng/regression-bisect/toolchain-installer.test.cjs b/eng/regression-bisect/toolchain-installer.test.cjs index faf3b30645b..478b7e3bfb8 100644 --- a/eng/regression-bisect/toolchain-installer.test.cjs +++ b/eng/regression-bisect/toolchain-installer.test.cjs @@ -632,7 +632,7 @@ for (const url of [ const fetch = t.mock.method(globalThis, 'fetch', () => assert.fail('must reject before network')); await assert.rejects(createHttp(policy())(url, { constraints: { allowedUrls: [approval.url], maxBytes: 100, timeoutMs: 1000 }, - }), /unapproved|exact approved/); + }), { code: 'checksum-provenance', message: /unapproved|exact approved/ }); assert.equal(fetch.mock.callCount(), 0); }); } @@ -646,7 +646,7 @@ for (const redirect of [ const fetch = t.mock.method(globalThis, 'fetch', async () => new Response(null, { status: 302, headers: { location: redirect } })); await assert.rejects(createHttp(policy())(approval.url, { constraints: { allowedUrls: [approval.url], maxBytes: 100, timeoutMs: 1000 }, - }), /unapproved|exact approved/); + }), { code: 'checksum-provenance', message: /unapproved|exact approved/ }); assert.equal(fetch.mock.callCount(), 1); }); } diff --git a/eng/regression-bisect/toolchain-io.cjs b/eng/regression-bisect/toolchain-io.cjs index cedb57fc152..9ad6deed41c 100644 --- a/eng/regression-bisect/toolchain-io.cjs +++ b/eng/regression-bisect/toolchain-io.cjs @@ -43,13 +43,19 @@ function validateHttpResponse(url, response, policy, constraints) { return response; } +function validateDownloadProvenance(validate) { + // Keep network and filesystem operations outside this synchronous validation boundary. + try { return validate(); } + catch (error) { throw Object.assign(error, { code: "checksum-provenance" }); } +} + function createHttp(policy, fileSystem = fs, signal) { const networkFailure = error => { throw Object.assign(new Error(error.message, { cause: error }), { name: error.name, code: "network-failure" }); }; return async (url, { destination, method = "GET", constraints } = {}) => { const redirects = []; - let current = downloadUrl(url, policy, constraints); + let current = validateDownloadProvenance(() => downloadUrl(url, policy, constraints)); requireValue(["GET", "HEAD"].includes(method), "unsupported download method"); const deadline = AbortSignal.timeout(constraints?.timeoutMs ?? 300000); const downloadSignal = signal ? AbortSignal.any([signal, deadline]) : deadline; @@ -61,12 +67,15 @@ function createHttp(policy, fileSystem = fs, signal) { if (downloadSignal.aborted) networkFailure(downloadSignal.reason); if ([301, 302, 303, 307, 308].includes(response.status)) { await response.body?.cancel(); - const location = response.headers.get("location"); - requireValue(location?.trim(), "redirect response is missing Location"); - requireValue(!/[%\\\s]/.test(location) && !location.split(/[/?#]/).some(segment => segment === "." || segment === ".."), - `unapproved public download URL: ${location}`); - const next = downloadUrl(new URL(location, current).href, policy, constraints); - if (/^https?:/i.test(location)) requireValue(location === next, `unapproved public download URL: ${location}`); + const next = validateDownloadProvenance(() => { + const location = response.headers.get("location"); + requireValue(location?.trim(), "redirect response is missing Location"); + requireValue(!/[%\\\s]/.test(location) && !location.split(/[/?#]/).some(segment => segment === "." || segment === ".."), + `unapproved public download URL: ${location}`); + const next = downloadUrl(new URL(location, current).href, policy, constraints); + if (/^https?:/i.test(location)) requireValue(location === next, `unapproved public download URL: ${location}`); + return next; + }); redirects.push({ from: current, to: next, status: response.status }); current = next; continue; @@ -119,7 +128,7 @@ function createHttp(policy, fileSystem = fs, signal) { sha256: sha256.digest("hex"), sha512: sha512.digest("hex"), size, }; } - throw new Error(`too many redirects: ${url}`); + throw Object.assign(new Error(`too many redirects: ${url}`), { code: "checksum-provenance" }); }; } @@ -304,4 +313,4 @@ async function capture(request, deps) { }; } -module.exports = { hash, requireValue, inside, publicUrl, validateHttpResponse, createHttp, run, isolatedEnvironment, fileIdentity, capture }; +module.exports = { hash, requireValue, inside, publicUrl, validateHttpResponse, validateDownloadProvenance, createHttp, run, isolatedEnvironment, fileIdentity, capture }; diff --git a/eng/regression-bisect/toolchains.cjs b/eng/regression-bisect/toolchains.cjs index 4d5ba408d79..4a8e1a23986 100644 --- a/eng/regression-bisect/toolchains.cjs +++ b/eng/regression-bisect/toolchains.cjs @@ -1,7 +1,7 @@ const fs = require("node:fs/promises"); const path = require("node:path"); const { validatePolicy, fingerprintInput } = require("./model.cjs"); -const { hash, requireValue, inside, publicUrl, createHttp, validateHttpResponse, run, isolatedEnvironment, capture } = require("./toolchain-io.cjs"); +const { hash, requireValue, inside, publicUrl, createHttp, validateHttpResponse, validateDownloadProvenance, run, isolatedEnvironment, capture } = require("./toolchain-io.cjs"); const { validateArchiveReceipt } = require("./archive-provenance.cjs"); const indexUrl = "https://builds.dotnet.microsoft.com/dotnet/release-metadata/releases-index.json"; @@ -245,7 +245,7 @@ async function exactArchive(toolchain, product, version, options, deps, attempts attempts.push(attempt); const response = await deps.http(url, { method: "HEAD", ...(constraints ? { constraints } : {}) }); attempt.status = response.status; - validateHttpResponse(url, response, options.policy, constraints); + validateDownloadProvenance(() => validateHttpResponse(url, response, options.policy, constraints)); if (response.status === 404) continue; if (response.status === 200 && approval) return { archive: { name, url, rid: toolchain.platform.rid, architecture: toolchain.platform.architecture, diff --git a/eng/regression-bisect/toolchains.test.cjs b/eng/regression-bisect/toolchains.test.cjs index 32047e4a057..e6ad392d90e 100644 --- a/eng/regression-bisect/toolchains.test.cjs +++ b/eng/regression-bisect/toolchains.test.cjs @@ -408,6 +408,80 @@ for (const location of forbiddenDownloads.filter(url => url.startsWith("https:") test(`public feed rejects ${location}`, () => assert.throws(() => publicUrl(location, policy()), /unapproved/)); } +for (const [name, response, reason, message] of [ + ...[`${feed}/other-sdk.zip`, ...forbiddenDownloads].map(location => [ + `redirect to ${location}`, () => new Response(null, { status: 302, headers: { location } }), + "checksum-provenance", /unapproved|exact approved/, + ]), + ["missing Location", () => new Response(null, { status: 302 }), "checksum-provenance", /missing Location/], + ["malformed Location", () => new Response(null, { status: 302, headers: { location: "https://[" } }), + "checksum-provenance", /Invalid URL/], + ["redirect loop", url => new Response(null, { status: 302, headers: { location: url } }), + "checksum-provenance", /too many redirects/], + ["archive absent", () => new Response(null, { status: 404 }), "unavailable-sdk", /archive unavailable/], + ["service unavailable", () => new Response(null, { status: 503 }), "network-failure", /HTTP 503/], + ["connection failure", () => { throw new TypeError("fetch failed"); }, "network-failure", /fetch failed/], +]) { + test(`historical availability preserves reason: ${name}`, async t => { + const approval = policy().toolchains.measuredArchives[0]; + const { result: { toolchain } } = await source(t, approval.version); + toolchain.sourceSha = approval.sourceSha; + const docs = metadata().docs; + const requests = []; + t.mock.method(globalThis, "fetch", async (url, settings) => { + requests.push({ url, method: settings.method }); + if (docs.has(url)) return new Response(JSON.stringify(docs.get(url))); + if (url.endsWith(".sha512")) return new Response(null, { status: 404 }); + assert.ok(policy().toolchains.measuredArchives.some(a => a.url === url), "no unapproved redirect request"); + assert.equal(settings.method, "HEAD", "no archive bytes before provenance approval"); + return response(url); + }); + const result = await acquireToolchain({ policy: policy(), toolchain, storageDirectory: await directory(t) }, { + installer: () => assert.fail("blocked probe must not install"), + capture: () => assert.fail("blocked probe must not execute a host SDK"), + }); + assert.equal(result.status, "untestable"); + assert.equal(result.outcome.reason, reason, JSON.stringify(result)); + assert.equal(result.error.code, reason === "unavailable-sdk" ? "archive-unavailable" : reason); + assert.equal(result.error.stage, "sdk"); + assert.match(result.error.message, message); + assert.deepEqual(result.attempts.filter(a => a.url.endsWith(".sha512")).map(a => a.status), [404, 404]); + assert.deepEqual(requests.filter(r => r.method === "HEAD").map(r => r.url), + name === "archive absent" ? policy().toolchains.measuredArchives.map(a => a.url) + : Array(name === "redirect loop" ? 6 : 1).fill(approval.url)); + assert.ok(result.attempts.every(a => a.status !== "pending")); + }); +} + +for (const [name, change] of [ + ["unapproved redirect", r => { r.redirects = [{ from: r.url, to: `${feed}/other-sdk.zip`, status: 302 }]; r.url = `${feed}/other-sdk.zip`; }], + ["missing redirects", r => { delete r.redirects; }], + ["disconnected redirect", r => { r.redirects = [{ from: `${feed}/other-sdk.zip`, to: r.url, status: 302 }]; }], + ["unrecorded final URL", r => { r.url = `${feed}/other-sdk.zip`; }], +]) { + test(`historical availability validates injected provenance: ${name}`, async t => { + const approval = policy().toolchains.measuredArchives[0]; + const { result: { toolchain } } = await source(t, approval.version); + toolchain.sourceSha = approval.sourceSha; + const deps = metadata(); + const result = await acquireToolchain({ policy: policy(), toolchain, storageDirectory: await directory(t) }, { + http: async (url, settings) => { + if (url.endsWith(".sha512")) return { status: 404, url, redirects: [] }; + if (settings?.method !== "HEAD") return deps.http(url); + const response = { status: 200, url, redirects: [] }; + change(response); + return response; + }, + installer: () => assert.fail("invalid response provenance must not install"), + capture: () => assert.fail("invalid response provenance must not execute"), + }); + assert.equal(result.outcome.reason, "checksum-provenance", JSON.stringify(result)); + assert.equal(result.error.code, "checksum-provenance"); + assert.equal(result.error.stage, "sdk"); + assert.deepEqual(result.attempts.map(a => a.status), [200, 200, 404, 404, 200]); + }); +} + for (const status of [301, 302, 303, 307, 308]) { for (const location of forbiddenDownloads) { test(`HTTP ${status} rejects redirect before requesting ${location}`, async t => { From 7b14e7b59cf203f3f8f1b0d9a66d790a45d1172d Mon Sep 17 00:00:00 2001 From: Copilot Date: Sun, 20 Sep 2026 01:26:22 +0200 Subject: [PATCH 26/99] Add endpoint-first resumable regression bisection engine Authenticate endpoint evidence before any Git bisect operation, preserve finite issue reservations, and reconstruct typed decisions in isolated history workspaces. Stop at fresh-confirmation intent or explicit uncertainty. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 151 ++++- eng/regression-bisect/engine.cjs | 326 +++++++++++ eng/regression-bisect/engine.test.cjs | 535 ++++++++++++++++++ eng/regression-bisect/fixtures/checkpoint.cjs | 77 +++ eng/regression-bisect/model.cjs | 18 +- eng/regression-bisect/state.cjs | 57 +- eng/regression-bisect/state.test.cjs | 92 +-- eng/regression-bisect/workspace.cjs | 6 +- eng/regression-bisect/workspace.test.cjs | 21 + 9 files changed, 1196 insertions(+), 87 deletions(-) create mode 100644 eng/regression-bisect/engine.cjs create mode 100644 eng/regression-bisect/engine.test.cjs create mode 100644 eng/regression-bisect/fixtures/checkpoint.cjs diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index 724b4fd3e26..dc2d3d3c16f 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -2,9 +2,9 @@ This directory defines version 1 of the deterministic input, evidence, state, and admission contracts, complete issue intake, authenticated issue-comment -checkpoints, and isolated exact toolchain acquisition. Acquisition runs the -official installer and fixed product-identity probes, not reproductions, source -builds, or bisection. It does not activate a workflow. +checkpoints, and isolated exact toolchain acquisition. The endpoint-first engine composes +those contracts with the existing reproduction and clean source-build runners. +It does not activate a workflow or attribute a culprit. GitHub access is injected by the trusted controller; tests use synthetic records and an in-memory transport, never live issue mutations. Records under `fixtures` are not reproduction approvals. @@ -179,11 +179,144 @@ this relationship from the pinned source graph, never issue claims or timestamps Stable pass stops investigation; main pass stops as release-specific/already fixed; untestable evidence blocks it. -Only the orchestration engine may eventually spawn `git bisect`. It must call -this gate immediately before the first operation and again after any -evidence-invalidating change, using the current authenticated context and -charged reservation. This contract suite does not claim to prove an engine's -subprocess behavior; no bisection engine is implemented here. +Only `engine.cjs` spawns `git bisect`. It calls the admission gate immediately +before constructing Git state, including every resumed reconstruction. Endpoint +evidence is already durably authenticated at that point; the selected source +worker is durably reserved before execution. + +## Endpoint-first controller seam + +`advanceInvestigation({ now }, dependencies)` re-reads the complete issue through +`readIssueSnapshot` (including `findActiveInvestigations`), re-resolves official +latest-stable metadata, and returns **at most one** charged `work` reservation or +a `stopped`, `blocked`, or `confirmation` report intent. No engine result is +`complete`, a published report, or an attributed culprit. Controllers must hold +the existing whole-issue concurrency lock across state transitions. + +Required trusted dependencies: + +* `context` and `transport` use the authenticated state API described below. + An existing initialized allowance is mandatory. Missing, empty, forged or + rolled-back state is blocked, never initialized by this engine. Approval and + policy/workflow revisions come from the pinned trusted default-branch loader. +* `approval` supplies intake's `readSource`, `authenticateRecord` and + `readTrustedCheckpoints`. These callbacks verify pinned raw reproduction and + producer assets and authentic workflow publications, not issue-supplied flags. + `intakeTransport` may supply separate read-only GitHub access. +* `platform` identifies the target worker. `environments` maps **every** matrix + cell ID to `{ os, architecture, imageDigest }`. Its canonical fingerprint is + part of the authenticated epoch. Changed images invalidate evidence without + resetting charges. +* `history` supplies absolute `gitExecutable`, existing `scratchDirectory`, + trusted full `mirrorDirectory`, and `mainRef: "refs/heads/main"`. The engine + fetches into its own disposable repository with no checkout, empty hooks, + isolated configuration, disabled credentials/replacement objects, and local + transport only. It never runs Git in the caller's checkout or mirror. + +The initialized context includes the exact released SDK archive SHA-256 +identities already established by trusted acquisition. Every advance resolves +official metadata at `now` and checks its version/index digest against that +context. Changed metadata or a new release blocks stale execution: the controller +must refresh the authenticated stable identity through the existing acquisition +boundary and invalidate the epoch, **not** create another allowance or substitute +an installed SDK. The worker reacquires and verifies the matching archive under +its charged reservation. Main is fetched and pinned only after a verified stable +failure; the saved SHA precedes its build. A fresh process can recover that pin +from authenticated state without remembering a prior return value. + +The order is stable failure, pinned-main failure, older-source pass, admission, +then Git selection. Stable pass stops as not reproduced; main pass stops as +fixed/release-specific. Untestable stable/main endpoints block. The deterministic +older search considers approved source-SHA hints that ordinary ancestry queries +verify, then oldest-first candidates in a bounded 4,096-commit main window. +SDK/version/URL claims are not converted to source observations or trusted tag +mappings. Every selected source still requires a clean build and repeated sample. +Search work is bounded by that window and the unchanged issue allowance. + +Admission requires comparable sample SDK/runtime, environment, producer, matrix +and exact assertions, but not equal revision **build** SDKs. Source observations +used by the engine must retain `buildToolchain`. Different build toolchains are +returned as transition evidence, never erased to make endpoints comparable. + +`executeInvestigationWork(work, dependencies)` executes the returned permission. +It calls `beginWork` before invoking any runner, making a lost acknowledgement +or duplicate worker invocation non-executable. The state ledger keeps the +reservation owner, attempt and execution generation. A cancelled owning run is +retired only through `reserveWork`/`invalidateStaleEvidence`; retrying costs another +full reservation and never more than two attempts per item. + +Worker dependencies include `worker` options for the existing runners, +`inventoryReleased(identity, buildInstanceId)`, and +`collectObservation({ result, reservation, manifest, context })`. These last two +are **trusted collector** boundaries: inventory the acquired SDK's compiler, +Core/package, tasks, targets and runtime bytes; authenticate captured worker +provenance and required artifacts; return `{ observation, evidence }` in the +existing state format. A worker setting `verified: true` is not a collector. +The engine checks that successful collected assertions are the original runner's +matrix repetitions and that source SHA/build IDs match the reserved work. Work +results carry only public epoch inputs in `binding`, never controller keys. +Execution takes authentication context from trusted dependencies, not the work +record. Never send those dependencies to an untrusted worker. + +Released execution uses `acquireToolchain`, `verifyToolchainIdentity` and +`runReproduction`. Source execution uses `testSourceRevision`, which reads each +revision's declarations, acquires its own build toolchain, and creates fresh +Bootstrap/Proto, bin/obj and package state. `worker.reproductionToolchain` is +required separately for engine source work; optional `sampleRuntimes` freezes +explicit sample runtimes. The workspace runner now accepts a reserved +`buildInstanceId`; reusing its directory fails rather than reusing outputs. +Unsupported historical runtime/sample combinations remain untestable. There is +no engine-specific FSI command, compiler runner, SDK installer, retargeting or +`global.json` rewrite. + +The interval adds strict `decisions: [{ sha, outcome, observationDigest }]` and +`remaining: [sha]` fields. Decisions are saved with observations and the next +advance recomputes graph bounds from the authenticated ledger. The legacy +`gitLog` is never executed. A lost selection worktree is reconstructed using +`bisect start --no-checkout` and typed `good`, `bad`, or `skip` arguments only; +there is no stored replay string, `bisect run`, or interpretation of generic +nonzero build exits as bad. Each selection pass has a five-minute wall-time cap +and each Git subprocess a one-minute cap. All-skipped results retain possible +commits and skip reasons. + +Observed descendant passes after failures, branches outside main ancestry, +merges in the asserted interval, and explicit revert/reintroduction/cherry-pick +messages stop automatic attribution with history/interval uncertainty. This +bounded inspection does not prove arbitrary histories monotonic. Merges require +explicit parent analysis rather than first-parent blame. + +The policy remains 24 source builds, 40 workers, 720 reserved minutes, two work +attempts and two assertion repetitions, with 90-minute builds, 110-minute +workers and 300-second reproductions. **Six reservations consume 660 minutes**; +the seventh does not fit even when each fixture finishes immediately. Candidate +selection preserves room for candidate-plus-parent confirmation where feasible. +Otherwise it returns the unconfirmed interval and consumed budget. A singleton +interval returns `confirmation` with `candidate`, all relevant `parents`, and +`requiresFreshConfirmation: true`; a separate consumer must reserve and perform +fresh confirmation before any attribution. + +`engine.test.cjs` uses real temporary Git histories and inexpensive injected +revision evaluators, not product acceptance. It records every actual bisect +subprocess and exit code, including zero counts for all denied endpoint tuples +and malformed/stale evidence. Process/GitHub/clock dependencies are injectable +at the existing runner boundaries; production defaults call the original modules. +The scoped suite covers execution claims, cumulative cancellation/retry budgets, +environment/input/main/stable epochs, lost worktrees, skips, toolchain transitions +and uncertain histories. + +Fresh Windows x64 product acceptance is retained at +`Q:\rb912-engine-acceptance-20260920`. The original workspace runner clean-built +`5a8b9adb034f420a5c2c1f577670c5eeeb454b7e` in build instance +`d6c3ce9a-a9a9-4f70-ae93-3a99a68c9a36` using its approved measured +`11.0.100-rc.1.26420.103` archive, with unchanged `global.json` and two passing +FSI assertions. Released SDK `10.0.401` and the fresh source products then matched +all 43 existing reproduction-smoke expectations (23 passes, six intended product +failures, 14 identity rejections). The scoped evidence-helper Release build/test +commands exited 0; that helper project contains no managed test cases. +`final-acceptance-report.json` has SHA-256 +`ccf46ba63cb50dc9ddc89b824aebb0ada1cf6c96d0c4b2a22310039c299f4864`. +This validates the original product runners, not a real regression's admitted +interval, an activated workflow, or fresh culprit attribution. ## Exact toolchain acquisition @@ -1076,8 +1209,10 @@ lock. All state-transition calls reread the authoritative comment. | --- | --- | | `loadCheckpoint(transport, context)` | Returns `{ checkpoint, bodyDigest, commentId, owner, executionAllowed, publicationAllowed, blockers, warnings }`. Authenticates freshness and can finish an interrupted receipt write under the issue lock. Invalid authentication/schema/freshness throws; stale approvals/inputs or required evidence loss explicitly block reuse. | | `reserveWork(transport, context, { generation, attemptId, request })` | Durably charges and saves the exact request before returning a reservation with `execute: true`. Duplicate requests return `execute: false`; they **must not launch another worker**. Conflicting attempt IDs, stale generations, active pending owners, exhausted counters, and deadlines fail closed. | +| `beginWork(transport, context, { generation, attemptId })` | Durably claims execution for the owning reservation. Returns `{ execute, generation }`; duplicates cannot execute again. Commit using this execution generation. A claim does not add a second charge or refund cancelled work. | | `commitObservation(transport, context, { generation, attemptId, observation, evidence })` | Checks ownership, request/build/provenance and the shared observation contract, then commits all decisive evidence before clearing pending work. Exact duplicate completion is idempotent. | | `invalidateStaleEvidence(transport, context, { generation })` | Invalidates changed input/approval/producer/orchestration dependencies without erasing history or charges. Main/stable changes start a new endpoint epoch while retaining exact-input historical source observations. Closed/unlabeled issues stop publication; existing result identities remain auditable. | +| `recordSearchProgress(transport, context, { generation, phase, interval, goodDigest? })` | Persists engine phase/interval and an optional authenticated passing-source endpoint. Typed decisions must match durable observations. Cannot alter a pending reservation, finish attribution, erase usage or refund work. | | `serializeWorkflowRecord({ kind, status, payload }, context)` | Signs a `checkpoint`, `proposal`, or `result` envelope. Low-level serialization is not approval or a storage transition. | | `authenticateWorkflowRecord(transport, context, comment)` | Returns true only for a matching signed record and fetched run provenance, not its freshness. Forged records return false; required API failures throw. Bind the expected record ID before using this as intake's exclusion callback, and use `loadCheckpoint` in the mandatory trusted-checkpoint reader to detect rollback before freezing. | diff --git a/eng/regression-bisect/engine.cjs b/eng/regression-bisect/engine.cjs new file mode 100644 index 00000000000..3439bc9671f --- /dev/null +++ b/eng/regression-bisect/engine.cjs @@ -0,0 +1,326 @@ +const fs = require("node:fs/promises"); +const path = require("node:path"); +const { + isEligibleIssue, validateManifest, validateObservation, classifyAttempts, admitBisection, + fingerprintInput: fingerprint, fingerprintEpoch, validateState, +} = require("./model.cjs"); +const intake = require("./intake.cjs"); +const { loadCheckpoint, reserveWork, beginWork, commitObservation, invalidateStaleEvidence, recordSearchProgress } = require("./state.cjs"); +const toolchains = require("./toolchains.cjs"); +const { runReproduction } = require("./repro.cjs"); +const { testSourceRevision } = require("./workspace.cjs"); +const { execute } = require("./repro-process.cjs"); +const { isolatedEnvironment, requireValue } = require("./toolchain-io.cjs"); + +const equal = (a, b) => fingerprint(a) === fingerprint(b); +const fullSha = s => /^[a-f0-9]{40}$/.test(s); +const maxHistory = 4096; +const report = (status, reason, extra = {}) => ({ status, reason, requiresFreshConfirmation: true, ...extra }); + +async function historyWorkspace(options, run, signal) { + requireValue(path.isAbsolute(options.gitExecutable) && options.mainRef === "refs/heads/main", + "trusted absolute Git executable and main ref required"); + const parent = await fs.realpath(options.scratchDirectory); + const mirror = await fs.realpath(options.mirrorDirectory); + const root = await fs.mkdtemp(path.join(parent, "selection-")); + const platform = { os: { win32: "windows", linux: "linux", darwin: "macos" }[process.platform], architecture: process.arch }; + const env = { ...isolatedEnvironment(root, platform), + GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: path.join(root, "gitconfig"), GIT_TERMINAL_PROMPT: "0", + GCM_INTERACTIVE: "Never", GIT_ALLOW_PROTOCOL: "file", GIT_NO_REPLACE_OBJECTS: "1" }; + env.PATH = `${path.dirname(options.gitExecutable)}${path.delimiter}${env.PATH}`; + const hooks = path.join(root, "hooks"); + await fs.mkdir(hooks); + await fs.mkdir(env.HOME); await fs.mkdir(env.TEMP); + await fs.writeFile(env.GIT_CONFIG_GLOBAL, "", { flag: "wx" }); + const commands = []; + const git = async (args, allowed = [0]) => { + const result = await run(options.gitExecutable, [ + "-c", `core.hooksPath=${hooks}`, "-c", "credential.helper=", "-c", "core.fsmonitor=false", + "-c", "protocol.allow=never", "-c", "protocol.file.allow=always", ...args, + ], { cwd: root, env, signal, timeoutMs: 60000, maxBytes: 4 * 1024 * 1024 }); + commands.push({ args, exitCode: result.exitCode }); + requireValue(result.complete && result.status === "completed" && allowed.includes(result.exitCode), + `history ${args[0]} failed (${result.exitCode}): ${result.stderr}`); + return result.stdout.trim(); + }; + const dispose = async () => { + requireValue(path.dirname(root) === parent && path.basename(root).startsWith("selection-") && + await fs.realpath(root) === root && !(await fs.lstat(root)).isSymbolicLink(), "unowned history workspace"); + await fs.rm(root, { recursive: true }); + }; + try { + await git(["init", "--quiet", "--template="]); + await git(["fetch", "--quiet", "--no-tags", "--no-recurse-submodules", mirror, options.mainRef]); + requireValue(await git(["rev-parse", "--is-shallow-repository"]) === "false", "shallow history is untestable"); + const main = await git(["rev-parse", "FETCH_HEAD"]); + requireValue(fullSha(main), "main did not resolve to a full SHA"); + await git(["update-ref", "--no-deref", "HEAD", main]); + const ancestor = async (a, b) => (await git(["merge-base", a, b], [0, 1])) === a; + return { git, main, ancestor, commands, dispose }; + } catch (error) { await dispose(); throw error; } +} + +function authenticated(cp, id, environments) { + const record = cp.observations[id]; + requireValue(record?.valid && cp.state.budget.authenticatedObservationDigests.includes(id), "missing authenticated observation"); + const o = record.data; + validateObservation(o, cp.state.manifest); + requireValue(fingerprint(o) === id && + o.provenance.workflowRevision === cp.state.budget.workflowRevision && + o.provenance.policyRevision === cp.state.budget.policyRevision, "observation provenance mismatch"); + for (const cell of o.cells) { + requireValue(equal(cell.outcome, classifyAttempts(cell.attempts, cp.state.manifest)), "inconsistent assertion evidence"); + requireValue(equal(cell.environment, environments[cell.id]), "stale worker environment"); + requireValue(o.kind !== "source" || o.outcome.outcome === "untestable" || cell.buildToolchain, + "source build toolchain evidence missing"); + } + return o; +} + +function capacity(cp, units, now) { + const { consumed, policy } = cp.state.budget; + return consumed.sourceBuilds + units <= policy.limits.sourceBuilds && + consumed.workerUnits + units <= policy.limits.workerUnits && + consumed.reservedMinutes + units * policy.limits.workerTimeoutMinutes <= policy.limits.reservedMinutes && + Date.parse(now) + units * policy.limits.workerTimeoutMinutes * 60000 <= Date.parse(cp.deadline); +} + +async function selectSource(cp, context, deps, history) { + const s = cp.state; + const get = id => authenticated(cp, id, deps.environments); + const stable = get(s.endpointEvidence.stable), main = get(s.endpointEvidence.main); + const revisions = s.testedRevisions.map(r => ({ ...r, data: get(r.observationDigest) })); + const all = (await history.git(["rev-list", "--topo-order", `--max-count=${maxHistory + 1}`, history.main])).split("\n"); + const older = all.slice(1, maxHistory); + for (const r of revisions) if ((all.length <= maxHistory && !all.includes(r.sourceSha)) || !(await history.ancestor(r.sourceSha, history.main))) + return report("blocked", "observed source is outside main ancestry", { historyEvidence: revisions.map(r => r.sourceSha) }); + for (const bad of revisions.filter(r => r.data.outcome.outcome === "product-failure")) + for (const good of revisions.filter(r => r.data.outcome.outcome === "pass")) + if (await history.ancestor(bad.sourceSha, good.sourceSha)) + return report("blocked", "nonmonotonic observed descendant pass after failure", + { historyEvidence: { failure: bad.sourceSha, descendantPass: good.sourceSha }, interval: s.interval }); + const passing = revisions.find(r => r.sourceSha !== history.main && r.data.outcome.outcome === "pass"); + if (!passing) { + const candidates = [...new Set([...s.manifest.searchHints.sourceShas, ...older.toReversed()])] + .filter(sha => older.includes(sha) && !revisions.some(r => r.sourceSha === sha)) + .slice(0, s.budget.policy.limits.sourceBuilds); + if (!candidates.length) return report("stopped", "no verified starting interval", { skips: s.skips }); + if (!capacity(cp, 3, context.now)) return report("blocked", "no verified starting interval; confirmation budget exhausted", + { budget: s.budget.consumed, skips: s.skips }); + return { request: { kind: "source", target: { sourceSha: candidates[0] }, endpoint: "good", phase: "good-search" } }; + } + const ancestry = { goodSha: passing.sourceSha, badSha: history.main, pinnedMainSha: history.main, isAncestor: true }; + const good = passing.data; + const intervalShas = (await history.git(["rev-list", "--topo-order", `--max-count=${maxHistory + 1}`, + history.main, `^${passing.sourceSha}`])).split("\n"); + requireValue(intervalShas.length <= maxHistory, "history bound exceeded; interval uncertainty"); + const initialInterval = { goodSha: passing.sourceSha, badSha: history.main, gitLog: [], decisions: [], remaining: intervalShas }; + const parents = await history.git(["rev-list", "--parents", history.main, `^${passing.sourceSha}`]); + if (parents.split("\n").some(line => line.split(" ").length > 2)) + return report("blocked", "merge/side-branch interval requires explicit parent analysis", { interval: initialInterval }); + const messages = await history.git(["log", "--format=%B%x00", history.main, `^${passing.sourceSha}`]); + if (/\b(?:revert(?:ed|ing)?|reintroduc\w*|cherry[ -]pick\w*)\b/i.test(messages)) + return report("blocked", "explicit revert/reintroduction/backport history; monotonicity uncertain", { interval: initialInterval }); + const decisions = cp.records.filter(r => r.status === "committed" && r.epoch === s.epoch && r.request.phase === "bisect") + .map(r => { + const o = get(r.observationDigest); + requireValue(intervalShas.includes(o.target.sourceSha), "decision outside admitted interval"); + for (const c of o.cells) { + const m = main.cells.find(m => m.id === c.id); + requireValue(o.outcome.outcome === "untestable" || equal(c.sdk, m.sdk) && equal(c.runtime, m.runtime), + "candidate reproduction toolchain changed"); + } + return { sha: o.target.sourceSha, outcome: o.outcome.outcome, observationDigest: r.observationDigest }; + }); + const admission = admitBisection({ manifest: s.manifest, stable, main, good, ancestry, budget: s.budget }); + if (admission.decision !== "admit") return report(admission.decision === "stop" ? "stopped" : "blocked", admission.reason); + // No bisect command, including reset, is permitted before this gate. + await history.git(["bisect", "start", "--no-checkout", history.main, passing.sourceSha]); + let ambiguous = false; + for (const d of decisions) { + const output = await history.git(["bisect", { pass: "good", "product-failure": "bad", untestable: "skip" }[d.outcome], d.sha], [0, 2]); + ambiguous ||= output.includes("only 'skip'ped commits left"); + } + const badSha = await history.git(["rev-parse", "refs/bisect/bad"]); + const goods = (await history.git(["for-each-ref", "--format=%(objectname)", "refs/bisect/good-*"])).split("\n"); + requireValue(goods.every(fullSha), "missing Git good boundary"); + const remaining = (await history.git(["rev-list", "--topo-order", badSha, ...goods.map(sha => `^${sha}`)])).split("\n"); + let goodSha = passing.sourceSha; + for (const sha of goods) if (await history.ancestor(goodSha, sha)) goodSha = sha; + const interval = { goodSha, badSha, gitLog: [], decisions, remaining }; + if (ambiguous) return report("blocked", "all-skipped ambiguous interval", { interval, skips: s.skips }); + if (remaining.length === 1) { + const relevantParents = (await history.git(["rev-list", "--parents", "-n", "1", badSha])).split(" ").slice(1); + if (!capacity(cp, 1 + relevantParents.length, context.now)) + return report("blocked", "unconfirmed interval; insufficient fresh confirmation budget", { interval, budget: s.budget.consumed }); + return report("confirmation", "candidate requires fresh candidate and parent assertions", { + candidate: badSha, parents: relevantParents, interval, toolchainTransitions: admission.toolchainTransitions ?? [], + }); + } + if (!capacity(cp, 3, context.now)) return report("blocked", "unconfirmed interval; preserving confirmation budget", + { interval, skips: s.skips, budget: s.budget.consumed }); + const sha = await history.git(["rev-parse", "BISECT_HEAD"]); + requireValue(fullSha(sha) && remaining.includes(sha) && !decisions.some(d => d.sha === sha), "invalid repeated Git selection"); + return { interval, goodDigest: passing.observationDigest, + request: { kind: "source", target: { sourceSha: sha }, endpoint: null, phase: "bisect" } }; +} + +async function advanceInvestigation({ now }, deps) { + let history, cp, context; + const signal = AbortSignal.any([AbortSignal.timeout(300000), ...(deps.signal ? [deps.signal] : [])]); + try { + context = { ...deps.context, now, environmentFingerprint: fingerprint(deps.environments) }; + validateManifest(context.manifest); + requireValue(["readSource", "authenticateRecord", "readTrustedCheckpoints"].every(k => typeof deps.approval?.[k] === "function"), + "pinned approval assets and authenticated intake readers required"); + const snapshot = await (deps.intake?.readIssueSnapshot ?? intake.readIssueSnapshot)({ + ...deps.approval, transport: deps.intakeTransport ?? deps.transport, repository: context.repository, + issueNumber: context.issueNumber, reviewedManifest: context.manifest, now, + }); + if (snapshot.status !== "complete") return report(snapshot.status === "ineligible" ? "stopped" : "blocked", snapshot.reason); + const current = snapshot.snapshot; + requireValue(current.investigations?.status === "complete" && current.input.reproduction && + equal(current.input.reproduction.manifest, context.manifest) && + current.discussionFingerprint === context.manifest.discussionFingerprint, "incomplete or stale approved input"); + context.issue = current.input.discussion.issue; + context.issueFingerprint = current.discussionFingerprint; + requireValue(isEligibleIssue(context.issue), "issue is no longer eligible"); + const active = current.investigations.active.filter(a => !(a.basis === "trusted-checkpoint-active-run" && + a.commentId === context.commentId && a.owner.runId === context.owner.runId && a.owner.attempt === context.owner.attempt)); + if (active.length) return report("blocked", "active linked investigation", { active }); + let loaded = await loadCheckpoint(deps.transport, context); + cp = loaded.checkpoint; + if (context.pinnedMainSha !== cp.state.pinnedMainSha) { + context.pinnedMainSha = cp.state.pinnedMainSha; + loaded = await loadCheckpoint(deps.transport, context); + } + if (!loaded.executionAllowed) { + await invalidateStaleEvidence(deps.transport, context, { generation: cp.generation }); + loaded = await loadCheckpoint(deps.transport, context); + cp = loaded.checkpoint; + if (!loaded.executionAllowed) return report("blocked", loaded.blockers.join("; ")); + } + validateState(cp.state); + requireValue(cp.state.epoch === fingerprintEpoch(context.manifest, cp.state.budget, context.pinnedMainSha), "stale epoch"); + const resolved = await (deps.toolchains?.resolveLatestStableSdk ?? toolchains.resolveLatestStableSdk)( + { policy: context.policy, platform: deps.platform, signal }, { clock: () => new Date(now), ...deps.toolchainDependencies }); + requireValue(resolved.status === "resolved", `latest stable resolution blocked: ${resolved.error?.message}`); + requireValue(resolved.toolchain.sdkVersion === context.stableSdk.version && + resolved.toolchain.metadata.some(m => m.url === context.stableSdk.releaseIndexUrl && m.sha256 === context.stableSdk.releaseIndexDigest), + "latest stable identity changed; authenticated endpoint context must be refreshed"); + if (cp.pending) { + const owner = cp.pending.owner; + const run = await deps.transport.getRun(owner.runId, owner.attempt); + if (run.status !== "completed") return report("blocked", "work already reserved by an active owner", + { pending: { id: cp.pending.id, owner, deadline: cp.pending.deadline } }); + } + let selected; + const stableId = cp.state.endpointEvidence.stable; + if (!stableId) selected = { request: { kind: "released", target: { sdk: context.stableSdk }, endpoint: "stable", phase: "stable" } }; + else { + const stable = authenticated(cp, stableId, deps.environments); + requireValue(stable.kind === "released" && equal(stable.target.sdk, context.stableSdk), "wrong stable endpoint identity"); + if (stable.outcome.outcome !== "product-failure") selected = report( + stable.outcome.outcome === "pass" ? "stopped" : "blocked", + stable.outcome.outcome === "pass" ? "not reproduced on latest stable" : `stable untestable: ${stable.outcome.reason}`); + else { + history = await historyWorkspace(deps.history, deps.execute ?? execute, signal); + if (history.main !== cp.state.pinnedMainSha) { + context.pinnedMainSha = history.main; + cp = await invalidateStaleEvidence(deps.transport, context, { generation: cp.generation }); + } + const mainId = cp.state.endpointEvidence.main; + if (!mainId) selected = { request: { kind: "source", target: { sourceSha: history.main }, endpoint: "main", phase: "main" } }; + else { + const main = authenticated(cp, mainId, deps.environments); + requireValue(main.kind === "source" && main.target.sourceSha === history.main, "wrong pinned-main endpoint identity"); + if (main.outcome.outcome !== "product-failure") selected = report( + main.outcome.outcome === "pass" ? "stopped" : "blocked", + main.outcome.outcome === "pass" ? "release-specific or fixed on main" : `main untestable: ${main.outcome.reason}`); + else selected = await selectSource(cp, context, deps, history); + } + } + } + if ((selected.interval || selected.status) && cp.pending === null) { + cp = await recordSearchProgress(deps.transport, context, { + generation: cp.generation, phase: selected.status ?? "bisect", + interval: selected.interval ?? cp.state.interval, ...(selected.goodDigest ? { goodDigest: selected.goodDigest } : {}), + }); + } + if (selected.status) return { ...selected, generation: cp.generation, commands: history?.commands ?? [] }; + signal.throwIfAborted(); + const attemptId = fingerprint({ owner: context.owner, sequence: cp.sequence + 1, epoch: cp.state.epoch }).slice(0, 32); + const reservation = await reserveWork(deps.transport, context, { generation: cp.generation, attemptId, request: selected.request }); + return { status: "work", reservation, binding: { + issue: context.issue, issueFingerprint: context.issueFingerprint, pinnedMainSha: context.pinnedMainSha, + environmentFingerprint: context.environmentFingerprint, + }, resolved: resolved.toolchain, commands: history?.commands ?? [] }; + } catch (error) { + return report("blocked", error.message, { ...(cp ? { budget: cp.state.budget.consumed } : {}), commands: history?.commands ?? [] }); + } finally { if (history) await history.dispose(); } +} + +async function executeInvestigationWork(work, deps) { + requireValue(work.status === "work" && work.reservation.execute, "work has no new execution permission"); + const context = { ...deps.context, now: (deps.clock ?? (() => new Date()))().toISOString(), + issue: work.binding.issue, issueFingerprint: work.binding.issueFingerprint, + pinnedMainSha: work.binding.pinnedMainSha, environmentFingerprint: fingerprint(deps.environments) }; + const loaded = await loadCheckpoint(deps.transport, context); + const r = loaded.checkpoint.pending; + requireValue(loaded.executionAllowed && r?.id === work.reservation.id && + r.generation === work.reservation.generation && equal(r.request, work.reservation.request), + "work reservation no longer current"); + requireValue(typeof deps.collectObservation === "function", "trusted observation collector required"); + requireValue(r.request.kind !== "source" || deps.worker.reproductionToolchain, + "frozen reproduction toolchain required separately from revision build SDK"); + const started = await beginWork(deps.transport, context, { generation: r.generation, attemptId: r.id }); + if (!started.execute) return report("blocked", "work execution already claimed"); + const manifest = loaded.checkpoint.state.manifest; + const signal = AbortSignal.any([AbortSignal.timeout(Math.max(1, Date.parse(r.deadline) - Date.parse(context.now))), + ...(deps.signal ? [deps.signal] : [])]); + let result; + if (r.request.kind === "source") { + result = await (deps.testSourceRevision ?? testSourceRevision)({ + ...deps.worker, manifest, signal, buildInstanceId: r.id, + request: { schemaVersion: 1, repository: context.repository, sourceSha: r.request.target.sourceSha, + phase: r.request.phase === "good-search" ? "older-search" : r.request.phase, + manifestDigest: fingerprint(manifest), approvalDigest: manifest.approvedInputDigest, + platform: deps.platform, allocation: { buildTimeoutMs: context.policy.limits.buildTimeoutMinutes * 60000, + workerTimeoutMs: context.policy.limits.workerTimeoutMinutes * 60000 } }, + }, deps.workspaceDependencies); + } else { + const acquired = await (deps.acquireToolchain ?? toolchains.acquireToolchain)({ + policy: context.policy, toolchain: work.resolved, storageDirectory: deps.worker.storageDirectory, signal, + }, deps.toolchainDependencies); + if (acquired.status !== "acquired") result = acquired; + else { + const verified = (deps.verifyToolchainIdentity ?? toolchains.verifyToolchainIdentity)({ + policy: context.policy, toolchain: acquired.identity.requested, identity: acquired.identity, + }); + requireValue(verified.status === "verified", "stable acquisition identity rejected"); + requireValue(acquired.identity.archive.sha256 === context.stableSdk.archives.find(a => + a.os === deps.platform.os && a.architecture === deps.platform.architecture)?.sha256, "stable archive identity changed"); + const product = await deps.inventoryReleased(acquired.identity, r.id); + result = await (deps.runReproduction ?? runReproduction)({ + ...deps.worker, manifest, product, toolchain: acquired.identity, + }, { execute: (exe, args, settings) => (deps.workerExecute ?? execute)(exe, args, { ...settings, signal }) }); + } + } + signal.throwIfAborted(); + const collected = await deps.collectObservation({ result, reservation: r, manifest, context }); + validateObservation(collected.observation, manifest); + requireValue(equal(collected.observation.outcome, result.outcome), "collector changed worker outcome"); + if (result.outcome.outcome !== "untestable") { + const reproduction = r.request.kind === "source" ? result.reproduction : result; + requireValue(r.request.kind !== "source" || result.status === "tested" && + result.actualSourceSha === r.request.target.sourceSha && result.buildInstanceId === r.id, "source result identity mismatch"); + requireValue(reproduction?.cells.length === manifest.matrix.length && + collected.observation.cells.every(c => equal(c.attempts, reproduction.cells.find(r => r.id === c.id)?.attempts)), + "collector omitted original repeated assertions"); + } + return commitObservation(deps.transport, { ...context, now: (deps.clock ?? (() => new Date()))().toISOString() }, { + generation: started.generation, attemptId: r.id, observation: collected.observation, evidence: collected.evidence, + }); +} + +module.exports = { advanceInvestigation, executeInvestigationWork }; diff --git a/eng/regression-bisect/engine.test.cjs b/eng/regression-bisect/engine.test.cjs new file mode 100644 index 00000000000..786d97fbfbf --- /dev/null +++ b/eng/regression-bisect/engine.test.cjs @@ -0,0 +1,535 @@ +const { test } = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs/promises"); +const path = require("node:path"); +const os = require("node:os"); +const { execFileSync, spawnSync } = require("node:child_process"); +const f = require("./fixtures/contracts.cjs"); +const { harness } = require("./fixtures/checkpoint.cjs"); +const model = require("./model.cjs"); +const state = require("./state.cjs"); +const { advanceInvestigation, executeInvestigationWork } = require("./engine.cjs"); + +const gitExecutable = process.platform === "win32" + ? execFileSync("where.exe", ["git.exe"], { encoding: "utf8" }).trim().split(/\r?\n/)[0] + : execFileSync("which", ["git"], { encoding: "utf8" }).trim(); + +async function fixture(t, outcomes = ["product-failure", "product-failure", "pass"], messages = ["base", "one", "two", "main"]) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "engine-test-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const mirror = path.join(root, "mirror"), scratch = path.join(root, "scratch"); + await fs.mkdir(mirror); await fs.mkdir(scratch); + const git = (...args) => execFileSync(gitExecutable, ["-c", "core.hooksPath=", "-c", "commit.gpgSign=false", ...args], { + cwd: mirror, encoding: "utf8", env: { ...process.env, GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: path.join(root, "empty"), GIT_AUTHOR_NAME: "Fixture", GIT_AUTHOR_EMAIL: "fixture@example.invalid", + GIT_COMMITTER_NAME: "Fixture", GIT_COMMITTER_EMAIL: "fixture@example.invalid", + GIT_AUTHOR_DATE: "2026-09-01T00:00:00Z", GIT_COMMITTER_DATE: "2026-09-01T00:00:00Z" }, + }).trim(); + await fs.writeFile(path.join(root, "empty"), ""); + git("init", "--quiet", "--initial-branch=main"); + const shas = []; + for (const message of messages) { git("commit", "--quiet", "--allow-empty", "-m", message); shas.push(git("rev-parse", "HEAD")); } + const h = harness(); + h.context.pinnedMainSha = shas.at(-1); + h.context.manifest.searchHints.sourceShas = [shas[0]]; + f.seal(h.context.manifest); + h.context.authorization.manifestDigest = model.fingerprintInput(h.context.manifest); + h.context.authorization.inputDigest = h.context.manifest.approvedInputDigest; + const environments = Object.fromEntries(h.context.manifest.matrix.map(c => [c.id, + { os: c.os, architecture: c.architecture, imageDigest: f.digest("runner image") }])); + h.context.environmentFingerprint = model.fingerprintInput(environments); + await h.start(); + const trace = [], calls = []; + t.after(() => t.diagnostic(JSON.stringify({ bisectCount: calls.length, + calls: calls.map(args => ({ args: [...args], exitCode: args.exitCode })) }))); + const snapshot = { status: "complete", snapshot: { + discussionFingerprint: h.context.issueFingerprint, + input: { discussion: { issue: h.context.issue }, reproduction: { manifest: h.context.manifest } }, + investigations: { status: "complete", active: [] }, + } }; + const deps = { + transport: h.transport, context: h.context, environments, + intake: { readIssueSnapshot: async () => { trace.push("snapshot"); return snapshot; } }, + approval: { readSource: async () => Buffer.from('printfn "ok"'), + authenticateRecord: async () => false, readTrustedCheckpoints: async () => [] }, + platform: { os: "linux", architecture: "x64", rid: "linux-x64" }, + history: { gitExecutable, mirrorDirectory: mirror, scratchDirectory: scratch, mainRef: "refs/heads/main" }, + toolchains: { resolveLatestStableSdk: async () => { + trace.push("latest-stable"); + return { status: "resolved", toolchain: { sdkVersion: h.context.stableSdk.version, + metadata: [{ url: h.context.stableSdk.releaseIndexUrl, sha256: h.context.stableSdk.releaseIndexDigest }] } }; + } }, + execute: async (exe, args, settings) => { + let call; + if (args.includes("bisect")) { + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + assert.ok(Object.values(cp.state.endpointEvidence).every(Boolean), "bisect preceded durable endpoints"); + call = args.slice(args.indexOf("bisect")); + calls.push(call); + } + const result = spawnSync(exe, args, { ...settings, encoding: "utf8", timeout: settings.timeoutMs, maxBuffer: settings.maxBytes }); + if (result.error) throw result.error; + if (call) call.exitCode = result.status; + return { status: "completed", complete: !result.signal, exitCode: result.status, stdout: result.stdout, stderr: result.stderr }; + }, + }; + async function advance() { + return advanceInvestigation({ now: h.context.now }, deps); + } + function observationFor(work, value) { + const r = work.reservation; + trace.push(r.request.phase); + const which = r.request.endpoint; + const outcome = value ?? outcomes[which === "stable" ? 0 : which === "main" ? 1 : 2]; + const observation = f.observation(r.request.kind, outcome, h.context.manifest, r.request.target.sourceSha); + observation.buildInstanceId = r.id; + observation.provenance.runId = r.owner.runId; + for (const cell of observation.cells) { + for (const artifact of Object.values(cell.artifacts)) artifact.buildInstanceId = r.id; + if (observation.kind === "source") cell.buildToolchain = f.buildToolchain(observation.target.sourceSha, cell.environment.os); + } + return observation; + } + async function finish(work, value) { + const r = work.reservation; + const observation = observationFor(work, value); + await state.commitObservation(h.transport, { ...h.context, ...work.binding }, { + generation: r.generation, attemptId: r.id, observation, evidence: [], + }); + return observation; + } + return Object.assign(h, { deps, snapshot, trace, calls, shas, git, advance, finish, observationFor, scratch }); +} + +async function endpoints(h) { + for (let i = 0; i < 3; i++) { + const work = await h.advance(); + assert.equal(work.status, "work", JSON.stringify(work)); + await h.finish(work); + } +} + +test("all 27 endpoint tuples deny every actual bisect subprocess except failure/failure/pass", async t => { + for (const stable of ["pass", "product-failure", "untestable"]) + for (const main of ["pass", "product-failure", "untestable"]) + for (const good of ["pass", "product-failure", "untestable"]) + await t.test(`${stable}/${main}/${good}`, async t => { + const h = await fixture(t, [stable, main, good]); + let result; + for (let i = 0; i < 7; i++) { + result = await h.advance(); + if (result.status !== "work" || result.reservation.request.phase === "bisect") break; + await h.finish(result); + } + const admitted = stable === "product-failure" && main === "product-failure" && good === "pass"; + assert.equal(h.calls.length > 0, admitted, JSON.stringify(result)); + if (!admitted) assert.equal(h.calls.length, 0); + else assert.equal(result.status, "work", JSON.stringify(result)); + assert.notEqual(result.status, "complete"); + }); +}); + +test("malformed, stale, unauthenticated and incomplete evidence never spawns bisect", async t => { + const variants = { + missing: cp => { cp.state.endpointEvidence.good = f.digest("missing"); }, + fingerprint: cp => { Object.values(cp.observations)[0].data.inputDigest = f.digest("wrong"); }, + authentication: (cp, h) => { h.context.keys.fixture = Buffer.from("wrong-authentication-key"); }, + provenance: cp => { Object.values(cp.observations)[0].data.provenance.policyRevision = f.sha("wrong"); }, + matrix: cp => { Object.values(cp.observations)[0].data.cells = []; }, + artifact: cp => { Object.values(cp.observations)[0].data.cells[0].artifacts.core.sha256 = "wrong"; }, + assertion: cp => { Object.values(cp.observations)[0].data.cells[0].attempts[0].execution.stderr = "different diagnostic"; }, + restore: cp => { Object.values(cp.observations)[0].data.cells[0].attempts[0].checks.restore = false; }, + runtime: cp => { Object.values(cp.observations)[0].data.cells[0].runtime = null; }, + sdk: cp => { Object.values(cp.observations)[0].data.cells[0].sdk = null; }, + buildToolchain: cp => { Object.values(cp.observations).find(o => o.data.kind === "source").data.cells[0].buildToolchain.sourceSha = f.sha("wrong"); }, + approval: (cp, h) => { h.context.authorization.approvalRevision = f.sha("untrusted"); }, + discussion: (cp, h) => { h.snapshot.snapshot.discussionFingerprint = f.digest("new correction"); }, + incompleteSnapshot: (cp, h) => { h.snapshot.status = "blocked"; h.snapshot.reason = "rate limited; incomplete comments"; }, + linkedInvestigation: (cp, h) => { h.snapshot.snapshot.investigations.active.push({ basis: "timeline-cross-reference" }); }, + latestChanged: (cp, h) => { h.deps.toolchains.resolveLatestStableSdk = async () => ({ status: "resolved", + toolchain: { sdkVersion: "10.0.999", metadata: [] } }); }, + expiredArtifact: (cp, h) => { + Object.values(cp.observations)[0].evidence = [{ required: true, sha256: f.digest("durable evidence"), url: "https://example.invalid/evidence" }]; + h.transport.readEvidence = async () => null; + }, + }; + for (const [name, mutate] of Object.entries(variants)) await t.test(name, async t => { + const h = await fixture(t); + await endpoints(h); + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + mutate(cp, h); + if (name !== "authentication") h.checkpoint(cp); + const result = await h.advance(); + assert.equal(result.status, "blocked", JSON.stringify(result)); + assert.equal(h.calls.length, 0); + }); +}); + +test("Git skip and two-run flakes retain all possible commits, never a guessed culprit", async t => { + for (const flaky of [false, true]) await t.test(flaky ? "flaky" : "unbuildable", async t => { + const h = await fixture(t, undefined, ["base", "middle", "main"]); + await endpoints(h); + const work = await h.advance(); + assert.equal(work.status, "work", JSON.stringify(work)); + const o = h.observationFor(work, "untestable"); + for (const c of o.cells) { + c.attempts = flaky ? [f.attempt("pass"), f.attempt("product-failure")] : + [f.attempt("pass"), f.attempt("pass")].map(a => ({ ...a, checks: { ...a.checks, build: false } })); + c.outcome = model.classifyAttempts(c.attempts, h.context.manifest); + } + o.outcome = o.cells[0].outcome; + await state.commitObservation(h.transport, h.context, { generation: work.reservation.generation, + attemptId: work.reservation.id, observation: o, evidence: [] }); + const result = await h.advance(); + assert.equal(result.status, "blocked", JSON.stringify(result)); + assert.match(result.reason, /all-skipped/); + assert.deepEqual(new Set(result.interval.remaining), new Set(h.shas.slice(1))); + assert.equal(result.skips[0].reason, flaky ? "flaky" : "build-failure"); + assert.ok(h.calls.some(args => args[1] === "skip")); + assert.equal(result.candidate, undefined); + }); +}); + +test("explicit reverts, reintroductions and cherry-picked backports block before bisect", async t => { + for (const message of ['Revert "regression"', "Reintroduce changed semantics", "Fix (cherry picked from commit abcdef)"]) + await t.test(message, async t => { + const h = await fixture(t, undefined, ["base", message, "main"]); + await endpoints(h); + const result = await h.advance(); + assert.equal(result.status, "blocked"); + assert.match(result.reason, /history|monotonicity/); + assert.equal(result.interval.remaining.length, 2); + assert.equal(h.calls.length, 0); + }); +}); + +test("merge and side-branch evidence cannot be assigned to a first parent", async t => { + const h = await fixture(t); + h.git("checkout", "--quiet", "-b", "side", h.shas[0]); + h.git("commit", "--quiet", "--allow-empty", "-m", "side change"); + const side = h.git("rev-parse", "HEAD"); + h.git("checkout", "--quiet", "main"); + h.git("merge", "--quiet", "--no-ff", "side", "-m", "merge side"); + h.context.pinnedMainSha = h.git("rev-parse", "HEAD"); + await endpoints(h); + const result = await h.advance(); + assert.equal(result.status, "blocked"); + assert.match(result.reason, /merge\/side-branch/); + assert.ok(result.interval.remaining.includes(side)); + assert.equal(h.calls.length, 0); +}); + +test("cancelled reservations are charged, bounded and resumed through the state lifecycle", async t => { + const h = await fixture(t); + const first = await h.advance(); + assert.equal((await h.advance()).status, "blocked"); + h.nextRun(); + const second = await h.advance(); + assert.equal(second.status, "work", JSON.stringify(second)); + assert.equal(second.reservation.attempt, 2); + assert.notEqual(second.reservation.id, first.reservation.id); + h.nextRun(); + const result = await h.advance(); + assert.match(result.reason, /attempt budget/); + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + assert.equal(cp.state.budget.consumed.reservedMinutes, 220); + assert.equal(cp.records[0].status, "cancelled"); + assert.equal(h.calls.length, 0); +}); + +test("lost selection worktree and resumed process replay only authenticated typed decisions", async t => { + const h = await fixture(t); + await endpoints(h); + const work = await h.advance(); + await h.finish(work, "pass"); + assert.deepEqual(await fs.readdir(h.scratch), []); + h.nextRun(); + const result = await h.advance(); + assert.equal(result.status, "confirmation", JSON.stringify(result)); + assert.ok(h.calls.filter(args => args[1] === "start").length >= 2); + assert.ok(h.calls.some(args => args[1] === "good" && args[2] === work.reservation.request.target.sourceSha)); + assert.ok(h.calls.every(args => !["replay", "run", "reset"].includes(args[1]))); +}); + +test("fresh environment and input epochs invalidate evidence without replenishing allowance", async t => { + for (const change of ["environment", "reproduction", "discussion", "approval", "main", "stable"]) await t.test(change, async t => { + const h = await fixture(t); + await h.finish(await h.advance()); + if (change === "environment") h.deps.environments["linux-x64"].imageDigest = f.digest("new image"); + else if (change === "main") { + h.git("commit", "--quiet", "--allow-empty", "-m", "advance main"); + } else if (change === "stable") { + h.context.stableSdk.releaseIndexDigest = f.digest("new official metadata"); + } else { + if (change === "reproduction") h.context.manifest.assertions.pass.stdout = "new\n"; + if (change === "discussion") { + h.context.manifest.discussionFingerprint = f.digest("corrected discussion"); + h.context.issueFingerprint = h.context.manifest.discussionFingerprint; + h.snapshot.snapshot.discussionFingerprint = h.context.issueFingerprint; + } + if (change === "approval") h.context.policyRevision = f.sha("new approval"); + f.seal(h.context.manifest); + Object.assign(h.context.authorization, { manifestDigest: model.fingerprintInput(h.context.manifest), + inputDigest: h.context.manifest.approvedInputDigest, approvalRevision: h.context.policyRevision }); + } + const work = await h.advance(); + assert.equal(work.status, "work", JSON.stringify(work)); + assert.equal(work.reservation.request.phase, change === "main" ? "main" : "stable"); + const cp = (await state.loadCheckpoint(h.transport, { ...h.context, ...work.binding })).checkpoint; + assert.equal(cp.state.budget.consumed.workerUnits, 2); + assert.equal(cp.state.budget.consumed.reservedMinutes, 220); + assert.equal(cp.history.length, 1); + assert.equal(h.calls.length, 0); + }); +}); + +test("expiry, missing state and allowance exhaustion never create a fresh allowance", async t => { + for (const failure of ["expiry", "missing", "budget"]) await t.test(failure, async t => { + const h = await fixture(t); + if (failure === "expiry") h.context.now = "2026-10-01T00:00:00Z"; + if (failure === "missing") h.comment.body = ""; + if (failure === "budget") { + for (let i = 0; i < 6; i++) { + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + await state.reserveWork(h.transport, h.context, { generation: cp.generation, attemptId: `spent-${i}`, + request: { kind: "source", target: { sourceSha: f.sha(`unused-${i}`) }, phase: "good-search", endpoint: null } }); + h.nextRun(); + } + } + const result = await h.advance(); + assert.equal(result.status, "blocked", JSON.stringify(result)); + assert.match(result.reason, /deadline|authentication|budget/); + assert.equal(h.calls.length, 0); + if (failure === "budget") + assert.equal((await state.loadCheckpoint(h.transport, h.context)).checkpoint.state.budget.consumed.reservedMinutes, 660); + }); +}); + +test("linear history saves endpoints, selects Git midpoint and returns fresh confirmation only", async t => { + const h = await fixture(t); + for (const phase of ["stable", "main", "good-search"]) { + const work = await h.advance(); + assert.equal(work.status, "work", JSON.stringify(work)); + assert.equal(work.reservation.request.phase, phase); + assert.deepEqual(Object.keys(work).sort(), ["binding", "commands", "reservation", "resolved", "status"]); + assert.equal(h.calls.length, 0); + await h.finish(work); + } + const work = await h.advance(); + assert.equal(work.status, "work", JSON.stringify(work)); + assert.equal(work.reservation.request.target.sourceSha, h.shas[2]); + await h.finish(work, "pass"); + const result = await h.advance(); + assert.equal(result.status, "confirmation"); + assert.equal(result.candidate, h.shas[3]); + assert.deepEqual(result.parents, [h.shas[2]]); + assert.equal(result.requiresFreshConfirmation, true); + assert.deepEqual(result.interval.remaining, [h.shas[3]]); + assert.ok(h.calls.length > 0); + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + assert.equal(cp.state.phase, "confirmation"); + assert.deepEqual(cp.state.budget.consumed, { sourceBuilds: 3, workerUnits: 4, reservedMinutes: 440 }); + assert.deepEqual(cp.state.interval.decisions, [{ sha: h.shas[2], outcome: "pass", + observationDigest: cp.state.testedRevisions.at(-1).observationDigest }]); +}); + +test("a deterministic midpoint failure narrows bad without turning a build exit into a culprit", async t => { + const h = await fixture(t, undefined, ["base", "introduces regression", "main"]); + await endpoints(h); + const work = await h.advance(); + assert.equal(work.reservation.request.target.sourceSha, h.shas[1]); + await h.finish(work, "product-failure"); + const result = await h.advance(); + assert.equal(result.status, "confirmation", JSON.stringify(result)); + assert.equal(result.candidate, h.shas[1]); + assert.deepEqual(result.parents, [h.shas[0]]); + assert.deepEqual(result.interval.remaining, [h.shas[1]]); + assert.ok(h.calls.some(args => args[1] === "bad" && args[2] === h.shas[1])); +}); + +test("observed descendant passes and unrelated branch claims stop before admission", async t => { + for (const mismatch of [false, true]) await t.test(mismatch ? "ancestry" : "nonmonotonic", async t => { + const h = await fixture(t); + await endpoints(h); + let sha = h.shas[1]; + if (mismatch) { + h.git("checkout", "--quiet", "--orphan", "unrelated"); + h.git("commit", "--quiet", "--allow-empty", "-m", "other root"); + sha = h.git("rev-parse", "HEAD"); + h.git("checkout", "--quiet", "main"); + } + for (const [sourceSha, outcome] of [[sha, "product-failure"], ...(!mismatch ? [[h.shas[2], "pass"]] : [])]) { + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + const reservation = await state.reserveWork(h.transport, h.context, { + generation: cp.generation, attemptId: `additional-${sourceSha}`, + request: { kind: "source", target: { sourceSha }, endpoint: null, phase: "good-search" }, + }); + await h.finish({ reservation }, outcome); + } + const result = await h.advance(); + assert.equal(result.status, "blocked"); + assert.match(result.reason, mismatch ? /ancestry/ : /nonmonotonic/); + assert.equal(h.calls.length, 0); + }); +}); + +test("source toolchain transitions remain evidence without changing the frozen sample", async t => { + const h = await fixture(t, undefined, ["base", "main"]); + for (const phase of ["stable", "main", "good-search"]) { + const work = await h.advance(); + const observation = h.observationFor(work); + if (phase === "good-search") for (const c of observation.cells) { + const previousSdk = c.buildToolchain.requestedSdk; + c.buildToolchain.requestedSdk = c.buildToolchain.requestedTools = c.buildToolchain.sdk.version = "9.0.100"; + c.buildToolchain.archive.url = c.buildToolchain.archive.url.replaceAll(previousSdk, "9.0.100"); + c.buildToolchain.archive.redirectUrl = c.buildToolchain.archive.url; + c.buildToolchain.archive.name = c.buildToolchain.archive.url.split("/").at(-1); + c.buildToolchain.compilerRuntime = { version: "10.0.0", sha256: f.digest("older compiler runtime"), verified: true }; + c.buildToolchain.requestedCompilerRuntime = "10.0.0"; + } + await state.commitObservation(h.transport, h.context, { generation: work.reservation.generation, + attemptId: work.reservation.id, observation, evidence: [] }); + } + const result = await h.advance(); + assert.equal(result.status, "confirmation", JSON.stringify(result)); + assert.equal(result.toolchainTransitions.length, 1); + assert.equal(result.toolchainTransitions[0].good.requestedSdk, "9.0.100"); + assert.equal(result.toolchainTransitions[0].main.requestedSdk, "11.0.100-rc.1.26420.103"); + assert.equal(result.toolchainTransitions[0].good.requestedCompilerRuntime, "10.0.0"); + assert.equal(result.toolchainTransitions[0].main.requestedCompilerRuntime, "11.0.0"); +}); + +test("shallow history is untestable rather than a guessed older endpoint", async t => { + const h = await fixture(t); + await h.finish(await h.advance()); + await fs.writeFile(path.join(h.deps.history.mirrorDirectory, ".git", "shallow"), `${h.shas.at(-1)}\n`); + const result = await h.advance(); + assert.equal(result.status, "blocked"); + assert.match(result.reason, /shallow|history/); + assert.equal(h.calls.length, 0); +}); + +test("typed decisions and environment epochs have strict schemas", async t => { + const h = await fixture(t); + await endpoints(h); + const work = await h.advance(); + await h.finish(work, "pass"); + await h.advance(); + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + for (const mutate of [ + s => { s.interval.decisions[0].replay = "git bisect bad HEAD"; }, + s => { s.interval.decisions[0].sha = "HEAD"; }, + s => { s.interval.decisions[0].outcome = "bad"; }, + s => { s.interval.decisions.push(s.interval.decisions[0]); }, + s => { s.interval.decisions[0].observationDigest = f.digest("untrusted"); }, + s => { s.interval.remaining = []; }, + s => { s.budget.environmentFingerprint = f.digest("other environment"); }, + ]) { + const s = structuredClone(cp.state); + mutate(s); + assert.throws(() => model.validateState(s)); + } + cp.state.interval.decisions[0].outcome = "product-failure"; + h.checkpoint(cp); + await assert.rejects(state.loadCheckpoint(h.transport, h.context), /decision.*authenticated/); +}); + +test("default intake freezes complete corrected discussion and immutable bytes before any work", async t => { + const h = await fixture(t); + const issue = { ...h.context.issue, id: 123, title: "regression", body: "reporter claim is not a command" }; + const comment = { id: 22, body: "corrected sample", user: { id: 1 } }; + const transport = { request: async route => ({ data: route.endsWith("/comments") ? [comment] : + route.endsWith("/timeline") ? [] : issue, headers: { link: "" } }) }; + const { readIssueSnapshot } = require("./intake.cjs"); + const current = await readIssueSnapshot({ repository: "dotnet/fsharp", issueNumber: 123, transport, now: h.context.now }); + assert.equal(current.status, "complete"); + h.context.manifest.discussionFingerprint = current.snapshot.discussionFingerprint; + f.seal(h.context.manifest); + Object.assign(h.context.authorization, { manifestDigest: model.fingerprintInput(h.context.manifest), + inputDigest: h.context.manifest.approvedInputDigest }); + delete h.deps.intake; + h.deps.intakeTransport = transport; + h.deps.approval.readTrustedCheckpoints = async () => []; + h.deps.approval.authenticateRecord = async () => false; + let reads = 0; + h.deps.approval.readSource = async source => { + reads++; + assert.equal(source.sha256, f.digest('printfn "ok"')); + return Buffer.from('printfn "ok"'); + }; + const work = await h.advance(); + assert.equal(work.status, "work", JSON.stringify(work)); + assert.equal(work.reservation.request.phase, "stable"); + assert.equal(reads, 1); + assert.equal(h.calls.length, 0); +}); + +test("worker seam preserves original FSI/project inputs and charges before invoking existing runners", async t => { + for (const kind of ["fsi", "project"]) await t.test(kind, async t => { + const h = await fixture(t); + if (kind === "project") { + const m = h.context.manifest; + m.kind = kind; + m.sources[0].path = "fixtures/repro.fsproj"; + m.matrix[0].project = m.sources[0].path; + m.matrix[0].command = ["run-project", m.sources[0].path]; + f.seal(m); + Object.assign(h.context.authorization, { manifestDigest: model.fingerprintInput(m), inputDigest: m.approvedInputDigest }); + } + const clock = () => new Date(h.context.now); + let raw, acquisitionCalls = 0, sourceCalls = 0, reproductionCalls = 0; + const worker = { sourceDirectory: "immutable-reviewed-bytes", storageDirectory: "isolated-storage", + reproductionToolchain: { invariant: true }, sampleRuntimes: { "net10.0": { name: "Microsoft.NETCore.App", version: "10.0.0" } } }; + const dependencies = { ...h.deps, worker, clock, + acquireToolchain: async () => { + acquisitionCalls++; + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + assert.equal(cp.state.budget.consumed.workerUnits, 1); + assert.equal(cp.state.budget.pendingWork.status, "running"); + return { status: "acquired", identity: { requested: {}, archive: { sha256: h.context.stableSdk.archives[0].sha256 } } }; + }, + verifyToolchainIdentity: () => ({ status: "verified" }), + inventoryReleased: async (identity, id) => ({ kind: "released", buildInstanceId: id }), + runReproduction: async options => { + reproductionCalls++; + assert.deepEqual(options.manifest, h.context.manifest); + assert.deepEqual(options.manifest.producer, h.context.manifest.producer); + assert.equal(options.sourceDirectory, worker.sourceDirectory); + return raw = { outcome: f.observation("released", "product-failure", h.context.manifest).outcome, + cells: f.observation("released", "product-failure", h.context.manifest).cells }; + }, + testSourceRevision: async options => { + sourceCalls++; + assert.deepEqual(options.manifest.matrix, h.context.manifest.matrix); + assert.equal(options.sourceDirectory, worker.sourceDirectory); + assert.equal(options.reproductionToolchain, worker.reproductionToolchain); + assert.deepEqual(options.sampleRuntimes, worker.sampleRuntimes); + assert.equal(options.request.sourceSha, h.shas.at(-1)); + assert.equal(options.request.allocation.buildTimeoutMs, 90 * 60000); + assert.equal(options.request.allocation.workerTimeoutMs, 110 * 60000); + raw = { status: "tested", actualSourceSha: options.request.sourceSha, buildInstanceId: options.buildInstanceId, + outcome: f.observation("source", "product-failure", h.context.manifest).outcome, + reproduction: { cells: f.observation("source", "product-failure", h.context.manifest).cells } }; + return raw; + }, + collectObservation: async ({ reservation, result }) => { + assert.equal(result, raw); + return { observation: h.observationFor({ reservation }, "product-failure"), evidence: [] }; + }, + }; + for (const phase of ["stable", "main"]) { + const work = await h.advance(); + assert.equal(work.status, "work", JSON.stringify(work)); + assert.equal(work.reservation.request.phase, phase); + await assert.rejects(executeInvestigationWork({ ...work, reservation: { ...work.reservation, + request: { ...work.reservation.request, target: { sourceSha: f.sha("unreserved target") } } } }, dependencies), + /no longer current/); + await executeInvestigationWork(work, dependencies); + await assert.rejects(executeInvestigationWork(work, dependencies), /no longer current/); + } + assert.equal(acquisitionCalls, 1); + assert.equal(reproductionCalls, 1); + assert.equal(sourceCalls, 1); + assert.equal(h.calls.length, 0); + }); +}); diff --git a/eng/regression-bisect/fixtures/checkpoint.cjs b/eng/regression-bisect/fixtures/checkpoint.cjs new file mode 100644 index 00000000000..31ef85875f1 --- /dev/null +++ b/eng/regression-bisect/fixtures/checkpoint.cjs @@ -0,0 +1,77 @@ +const assert = require("node:assert/strict"); +const f = require("./contracts.cjs"); +const model = require("../model.cjs"); +const { initializeCheckpoint, serializeWorkflowRecord } = require("../state.cjs"); + +function harness() { + const a = f.admission(); + const context = { + repository: "dotnet/fsharp", issueNumber: 123, commentId: 456, + actor: { id: 77, login: "github-actions[bot]", type: "Bot", appId: 88 }, + workflowPath: ".github/workflows/regression-bisect.yml", + workflowRef: "refs/heads/main", trustedWorkflowRevisions: [f.sha("workflow")], + checkpointAnchorSha: f.sha("workflow"), bootstrapDigest: model.fingerprintInput("unused placeholder"), + keys: { fixture: Buffer.from("synthetic-test-key-not-a-production-secret") }, keyId: "fixture", + owner: { runId: "12345", attempt: 1 }, now: "2026-09-18T00:00:00Z", + issue: { number: 123, state: "open", created_at: "2026-09-01T00:00:00Z", labels: ["Regression"] }, + issueFingerprint: a.manifest.discussionFingerprint, + manifest: a.manifest, authorization: a.budget.authorization, policy: a.budget.policy, + policyRevision: a.budget.policyRevision, workflowRevision: a.budget.workflowRevision, + pinnedMainSha: a.ancestry.pinnedMainSha, stableSdk: a.budget.stableSdk, + }; + const runs = new Map(); + const run = owner => ({ + id: Number(owner.runId), run_attempt: owner.attempt, path: context.workflowPath, + head_sha: context.workflowRevision, head_branch: "main", + repository: { full_name: context.repository }, head_repository: { full_name: context.repository }, + status: "in_progress", conclusion: null, + }); + runs.set("12345/1", run(context.owner)); + let comment = { + id: 456, user: { id: 77, login: "github-actions[bot]", type: "Bot" }, + performed_via_github_app: { id: 88 }, + issue_url: "https://api.github.com/repos/dotnet/fsharp/issues/123", + body: "unused placeholder", + }; + let tail = Promise.resolve(); + const freshness = f.statusTransport(context); + const transport = { + request: freshness.request, + getComment: async id => { assert.equal(id, 456); return structuredClone(comment); }, + updateComment: async (id, body) => { assert.equal(id, 456); comment.body = body; return structuredClone(comment); }, + getRun: async (id, attempt) => { + const value = runs.get(`${id}/${attempt}`); + if (!value) throw new Error("run unavailable"); + return structuredClone(value); + }, + readEvidence: async () => Buffer.from("durable evidence"), + }; + return { + context, transport, runs, statuses: freshness.statuses, + get comment() { return comment; }, + set comment(c) { comment = c; }, + checkpoint(cp) { + cp.previousDigest = f.digest(comment.body); + cp.generation++; + comment.body = serializeWorkflowRecord({ kind: "checkpoint", status: "fixture transition", payload: cp }, context); + }, + lock(action) { + const next = tail.then(action); + tail = next.catch(() => {}); + return next; + }, + async start() { + return initializeCheckpoint(transport, context, { + allowanceId: "maintainer-approved-investigation-1", deadline: "2026-10-01T00:00:00Z", + }); + }, + nextRun() { + runs.get(`${context.owner.runId}/${context.owner.attempt}`).status = "completed"; + runs.get(`${context.owner.runId}/${context.owner.attempt}`).conclusion = "cancelled"; + context.owner = { runId: String(Number(context.owner.runId) + 1), attempt: 1 }; + runs.set(`${context.owner.runId}/1`, run(context.owner)); + }, + }; +} + +module.exports = { harness }; diff --git a/eng/regression-bisect/model.cjs b/eng/regression-bisect/model.cjs index 5a7007acdc6..7296d54034a 100644 --- a/eng/regression-bisect/model.cjs +++ b/eng/regression-bisect/model.cjs @@ -348,6 +348,7 @@ function validateBudget(budget, manifest) { stableSdk, authenticatedObservationDigests: list(digest), consumed: counters, attempts: list(shape({ workItemId: text, count: positive })), nextWorkItemId: text, pendingWork: nullable(reservation), + ...(budget && Object.hasOwn(budget, "environmentFingerprint") ? { environmentFingerprint: digest } : {}), }, "budget"); const auth = budget.authorization; check(auth.repository === manifest.repository && auth.issueNumber === manifest.issueNumber && @@ -383,6 +384,7 @@ function fingerprintEpoch(manifest, budget, pinnedMainSha) { schemaVersion: 1, manifestDigest: fingerprintInput(manifest), policyRevision: budget.policyRevision, workflowRevision: budget.workflowRevision, pinnedMainSha, stableSdk: budget.stableSdk, + ...(budget.environmentFingerprint ? { environmentFingerprint: budget.environmentFingerprint } : {}), }); } @@ -452,7 +454,14 @@ function validateState(state) { endpointEvidence: shape({ stable: nullable(digest), main: nullable(digest), good: nullable(digest) }), testedRevisions: list(shape({ sourceSha, observationDigest: digest })), skips: list(shape({ sourceSha, reason: oneOf(untestableReasons), observationDigest: digest })), - interval: nullable(shape({ goodSha: sourceSha, badSha: sourceSha, gitLog: list(text, 0, 1000) })), + interval: nullable((v, p) => object(v, { + goodSha: sourceSha, badSha: sourceSha, gitLog: list(text, 0, 1000), + ...(Object.hasOwn(v, "decisions") ? { + decisions: list(shape({ sha: sourceSha, outcome: oneOf(["pass", "product-failure", "untestable"]), + observationDigest: digest }), 0, trustedPolicy.limits.sourceBuilds), + remaining: list(sourceSha, 1, 4096), + } : {}), + }, p)), publication: shape({ digest: nullable(digest), commentId: nullable(positive) }), }, "state"); validateBudget(state.budget, state.manifest); @@ -463,8 +472,13 @@ function validateState(state) { "state", "identity or epoch mismatch"); const budget = state.budget; const evidence = [...Object.values(state.endpointEvidence).filter(x => x !== null), - ...state.testedRevisions.map(r => r.observationDigest), ...state.skips.map(r => r.observationDigest)]; + ...state.testedRevisions.map(r => r.observationDigest), ...state.skips.map(r => r.observationDigest), + ...(state.interval?.decisions ?? []).map(r => r.observationDigest)]; check(evidence.every(d => budget.authenticatedObservationDigests.includes(d)), "state", "unauthenticated evidence"); + if (state.interval?.decisions) { + unique(state.interval.decisions.map(d => d.sha), "state.interval.decisions"); + unique(state.interval.remaining, "state.interval.remaining"); + } return true; } catch (error) { if (!(error instanceof ContractError)) throw error; diff --git a/eng/regression-bisect/state.cjs b/eng/regression-bisect/state.cjs index 5bf199706b4..6c91d50a7ef 100644 --- a/eng/regression-bisect/state.cjs +++ b/eng/regression-bisect/state.cjs @@ -166,7 +166,8 @@ function validateCheckpoint(cp) { const totals = { sourceBuilds: 0, workerUnits: 0, reservedMinutes: 0 }; const ids = new Set(); for (const [index, r] of cp.records.entries()) { - fields(r, "id workItemId request requestDigest owner sequence attempt generation deadline inputDigest epoch reserved status observationDigest", "work record"); + fields(r, "id workItemId request requestDigest owner sequence attempt generation deadline inputDigest epoch reserved status observationDigest" + + (Object.hasOwn(r, "executionGeneration") ? " executionGeneration" : ""), "work record"); requireState(typeof r.id === "string" && r.id.length > 0 && r.id.length <= 128 && !ids.has(r.id) && r.sequence === index + 1 && positive(r.attempt) && positive(r.generation) && timestamp(r.deadline) && digest(r.inputDigest) && digest(r.epoch) && @@ -180,6 +181,8 @@ function validateCheckpoint(cp) { ids.add(r.id); for (const name of counterNames) totals[name] += r.reserved[name]; requireState(r.status === "committed" ? digest(r.observationDigest) : r.observationDigest === null, "invalid observation identity"); + if (Object.hasOwn(r, "executionGeneration")) requireState(positive(r.executionGeneration) && + r.executionGeneration > r.generation && r.executionGeneration <= cp.generation, "invalid execution generation"); } requireState(equal(totals, cp.state.budget.consumed), "consumed budgets must equal the durable reservation ledger"); const counts = new Map(); @@ -209,6 +212,11 @@ function validateCheckpoint(cp) { "missing durable observation"); requireState(equal(Object.entries(cp.observations).filter(([, o]) => o.valid).map(([id]) => id), cp.state.budget.authenticatedObservationDigests), "authenticated observation table mismatch"); + for (const decision of cp.state.interval?.decisions ?? []) { + const o = cp.observations[decision.observationDigest]; + requireState(o?.valid && o.data.kind === "source" && o.data.target.sourceSha === decision.sha && + o.data.outcome.outcome === decision.outcome, "search decision does not match authenticated observation"); + } } function reproductionDigest(manifest) { @@ -228,6 +236,7 @@ function currentState(cp, context) { const budget = { ...cp.state.budget, policy: context.policy, policyRevision: context.policyRevision, workflowRevision: context.workflowRevision, authorization: context.authorization, stableSdk: context.stableSdk, + ...(context.environmentFingerprint ? { environmentFingerprint: context.environmentFingerprint } : {}), }; state = { ...cp.state, manifest: context.manifest, budget, @@ -403,6 +412,7 @@ async function initializeCheckpoint(transport, context, { allowanceId, deadline allowanceId, authorization: context.authorization, stableSdk: context.stableSdk, authenticatedObservationDigests: [], consumed: { sourceBuilds: 0, workerUnits: 0, reservedMinutes: 0 }, attempts: [], nextWorkItemId: "none", pendingWork: null, + ...(context.environmentFingerprint ? { environmentFingerprint: context.environmentFingerprint } : {}), }; const state = { schemaVersion: 1, repository: context.repository, issueNumber: context.issueNumber, @@ -502,7 +512,7 @@ async function commitObservation(transport, context, { generation, attemptId, ob const observationDigest = fingerprint(observation); validateEvidence(evidence); if (record.status === "committed") { - requireState(generation === record.generation, "stale completion generation"); + requireState(generation === (record.executionGeneration ?? record.generation), "stale completion generation"); requireState(record.observationDigest === observationDigest && equal(cp.observations[observationDigest].evidence, evidence), "observation identity conflict"); return cp; @@ -531,6 +541,8 @@ async function commitObservation(transport, context, { generation, attemptId, ob const tested = { sourceSha: observation.target.sourceSha, observationDigest }; cp.state.testedRevisions.push(tested); if (observation.outcome.outcome === "untestable") cp.state.skips.push({ ...tested, reason: observation.outcome.reason }); + if (record.request.phase === "bisect" && cp.state.interval?.decisions) + cp.state.interval.decisions.push({ sha: tested.sourceSha, outcome: observation.outcome.outcome, observationDigest }); } cp.pending = null; cp.state.budget.pendingWork = null; @@ -538,6 +550,22 @@ async function commitObservation(transport, context, { generation, attemptId, ob return saveCheckpoint(transport, context, cp, loaded.bodyDigest); } +async function beginWork(transport, context, { generation, attemptId }) { + const loaded = await loadCheckpoint(transport, context); + const cp = loaded.checkpoint; + requireState(loaded.executionAllowed, `execution blocked: ${loaded.blockers.join("; ")}`); + requireState(cp.pending?.id === attemptId && ownerEqual(cp.pending.owner, context.owner), "work owner mismatch"); + if (cp.pending.executionGeneration !== undefined) return { execute: false, generation: cp.pending.executionGeneration }; + checkGeneration(cp, generation); + requireState(Date.parse(context.now) < Date.parse(cp.pending.deadline), "reservation deadline exhausted"); + cp.generation++; + cp.pending.executionGeneration = cp.generation; + cp.records.find(r => r.id === attemptId).executionGeneration = cp.generation; + cp.state.budget.pendingWork.status = "running"; + await saveCheckpoint(transport, context, cp, loaded.bodyDigest); + return { execute: true, generation: cp.generation }; +} + async function invalidateStaleEvidence(transport, context, { generation }) { const loaded = await loadCheckpoint(transport, context); const cp = loaded.checkpoint; @@ -553,7 +581,8 @@ async function invalidateStaleEvidence(transport, context, { generation }) { if (current.stale || current.state === null) { const sameInputs = current.state !== null && old.approvalFingerprint === current.state.approvalFingerprint && old.budget.workflowRevision === current.state.budget.workflowRevision && - old.budget.policyRevision === current.state.budget.policyRevision; + old.budget.policyRevision === current.state.budget.policyRevision && + old.budget.environmentFingerprint === current.state.budget.environmentFingerprint; cp.history.push({ epoch: old.epoch, inputDigest: cp.inputDigest, approvalDigest: old.approvalFingerprint, policyDigest: cp.policyDigest, workflowRevision: old.budget.workflowRevision, @@ -585,7 +614,27 @@ async function invalidateStaleEvidence(transport, context, { generation }) { return saveCheckpoint(transport, context, cp, loaded.bodyDigest); } +async function recordSearchProgress(transport, context, { generation, phase, interval, goodDigest }) { + const loaded = await loadCheckpoint(transport, context); + requireState(loaded.executionAllowed, `search blocked: ${loaded.blockers.join("; ")}`); + const cp = loaded.checkpoint; + checkGeneration(cp, generation); + requireState(cp.pending === null, "cannot change search with pending work"); + requireState(["admission", "bisect", "confirmation", "blocked", "stopped"].includes(phase), "invalid search phase"); + if (goodDigest !== undefined) { + const good = cp.observations[goodDigest]; + requireState(good?.valid && good.data.kind === "source" && good.data.outcome.outcome === "pass", + "search requires authenticated passing source"); + cp.state.endpointEvidence.good = goodDigest; + } + if (cp.state.phase === phase && equal(cp.state.interval, interval) && goodDigest === undefined) return cp; + cp.state.phase = phase; + cp.state.interval = structuredClone(interval); + cp.generation++; + return saveCheckpoint(transport, context, cp, loaded.bodyDigest); +} + module.exports = { initializeCheckpoint, serializeWorkflowRecord, authenticateWorkflowRecord, - loadCheckpoint, reserveWork, commitObservation, invalidateStaleEvidence, + loadCheckpoint, reserveWork, beginWork, commitObservation, invalidateStaleEvidence, recordSearchProgress, }; diff --git a/eng/regression-bisect/state.test.cjs b/eng/regression-bisect/state.test.cjs index 5117d6a8990..e690845d02c 100644 --- a/eng/regression-bisect/state.test.cjs +++ b/eng/regression-bisect/state.test.cjs @@ -6,81 +6,31 @@ const f = require("./fixtures/contracts.cjs"); const model = require("./model.cjs"); const { initializeCheckpoint, serializeWorkflowRecord, authenticateWorkflowRecord, - loadCheckpoint, reserveWork, commitObservation, invalidateStaleEvidence, + loadCheckpoint, reserveWork, beginWork, commitObservation, invalidateStaleEvidence, } = require("./state.cjs"); const clone = structuredClone; -function harness() { - const a = f.admission(); - const context = { - repository: "dotnet/fsharp", issueNumber: 123, commentId: 456, - actor: { id: 77, login: "github-actions[bot]", type: "Bot", appId: 88 }, - workflowPath: ".github/workflows/regression-bisect.yml", - workflowRef: "refs/heads/main", trustedWorkflowRevisions: [f.sha("workflow")], - checkpointAnchorSha: f.sha("workflow"), bootstrapDigest: model.fingerprintInput("unused placeholder"), - keys: { fixture: Buffer.from("synthetic-test-key-not-a-production-secret") }, keyId: "fixture", - owner: { runId: "12345", attempt: 1 }, now: "2026-09-18T00:00:00Z", - issue: { number: 123, state: "open", created_at: "2026-09-01T00:00:00Z", labels: ["Regression"] }, - issueFingerprint: a.manifest.discussionFingerprint, - manifest: a.manifest, authorization: a.budget.authorization, policy: a.budget.policy, - policyRevision: a.budget.policyRevision, workflowRevision: a.budget.workflowRevision, - pinnedMainSha: a.ancestry.pinnedMainSha, stableSdk: a.budget.stableSdk, - }; - const runs = new Map(); - const run = owner => ({ - id: Number(owner.runId), run_attempt: owner.attempt, path: context.workflowPath, - head_sha: context.workflowRevision, head_branch: "main", - repository: { full_name: context.repository }, head_repository: { full_name: context.repository }, - status: "in_progress", conclusion: null, +const { harness } = require("./fixtures/checkpoint.cjs"); + +test("execution claims survive lost acknowledgements and cannot execute a reservation twice", async () => { + const h = harness(); await h.start(); + const pending = await reserve(h); + const first = await beginWork(h.transport, h.context, { + generation: pending.generation, attemptId: pending.id, }); - runs.set("12345/1", run(context.owner)); - let comment = { - id: 456, user: { id: 77, login: "github-actions[bot]", type: "Bot" }, - performed_via_github_app: { id: 88 }, - issue_url: "https://api.github.com/repos/dotnet/fsharp/issues/123", - body: "unused placeholder", - }; - let tail = Promise.resolve(); - const freshness = f.statusTransport(context); - const transport = { - request: freshness.request, - getComment: async id => { assert.equal(id, 456); return clone(comment); }, - updateComment: async (id, body) => { assert.equal(id, 456); comment.body = body; return clone(comment); }, - getRun: async (id, attempt) => { - const value = runs.get(`${id}/${attempt}`); - if (!value) throw new Error("run unavailable"); - return clone(value); - }, - readEvidence: async () => Buffer.from("durable evidence"), - }; - return { - context, transport, runs, statuses: freshness.statuses, - get comment() { return comment; }, - set comment(c) { comment = c; }, - checkpoint(cp) { - cp.previousDigest = f.digest(comment.body); - cp.generation++; - comment.body = serializeWorkflowRecord({ kind: "checkpoint", status: "fixture transition", payload: cp }, context); - }, - // This stands for the controller's whole issue concurrency group, not comment CAS. - lock(action) { - const next = tail.then(action); - tail = next.catch(() => {}); - return next; - }, - async start() { - return initializeCheckpoint(transport, context, { - allowanceId: "maintainer-approved-investigation-1", deadline: "2026-10-01T00:00:00Z", - }); - }, - nextRun() { - runs.get(`${context.owner.runId}/${context.owner.attempt}`).status = "completed"; - runs.get(`${context.owner.runId}/${context.owner.attempt}`).conclusion = "cancelled"; - context.owner = { runId: String(Number(context.owner.runId) + 1), attempt: 1 }; - runs.set(`${context.owner.runId}/1`, run(context.owner)); - }, - }; -} + const duplicate = await beginWork(h.transport, h.context, { + generation: pending.generation, attemptId: pending.id, + }); + assert.equal(first.execute, true); + assert.equal(duplicate.execute, false); + assert.equal(first.generation, duplicate.generation); + const done = { ...completion(h, pending), generation: first.generation }; + await commitObservation(h.transport, h.context, done); + await commitObservation(h.transport, h.context, done); + const cp = (await loadCheckpoint(h.transport, h.context)).checkpoint; + assert.equal(cp.state.budget.consumed.workerUnits, 1); + assert.equal(cp.records[0].executionGeneration, first.generation); +}); const request = (sourceSha = f.sha("main"), endpoint = "main") => ({ kind: "source", target: { sourceSha }, endpoint, phase: "main", diff --git a/eng/regression-bisect/workspace.cjs b/eng/regression-bisect/workspace.cjs index 88e4830a785..ef3ddec3db9 100644 --- a/eng/regression-bisect/workspace.cjs +++ b/eng/regression-bisect/workspace.cjs @@ -100,7 +100,8 @@ async function prepareRevisionWorkspace(options, dependencies = {}) { const mirror = await fs.realpath(options.mirrorDirectory); requireValue(parent !== mirror && !inside(mirror, parent) && !inside(parent, mirror) && parent !== evidenceParent && !inside(parent, evidenceParent), "scratch, mirror and evidence must be separate"); - const buildInstanceId = randomUUID(); + const buildInstanceId = options.buildInstanceId ?? randomUUID(); + requireValue(/^[A-Za-z0-9-]{1,128}$/.test(buildInstanceId), "invalid reserved build instance"); const root = path.join(parent, `revision-${buildInstanceId}`); await fs.mkdir(root); const signal = AbortSignal.any([AbortSignal.timeout(options.request.allocation.workerTimeoutMs), @@ -451,7 +452,8 @@ async function testSourceRevision(options, dependencies = {}) { try { const reproduction = await runReproduction({ manifest: options.manifest, sourceDirectory: w.sampleDirectory, workDirectory: path.join(w.root, "executions"), - product: w.product, toolchain: w.toolchain, helper: w.helper, + product: w.product, toolchain: options.reproductionToolchain ?? w.toolchain, helper: w.helper, + sampleRuntimes: options.sampleRuntimes, }, { execute: (exe, args, settings) => w.execute(exe, args, { ...settings, signal: w.signal, timeoutMs: Math.max(1, Math.min(settings.timeoutMs, w.deadline - Date.now())), diff --git a/eng/regression-bisect/workspace.test.cjs b/eng/regression-bisect/workspace.test.cjs index d64dd72f054..229406a17a2 100644 --- a/eng/regression-bisect/workspace.test.cjs +++ b/eng/regression-bisect/workspace.test.cjs @@ -238,6 +238,27 @@ test("evidence helper selects its runtime TFM consistently for restore and build assert.match(await fs.readFile(path.join(helper, "NuGet.Config"), "utf8"), /dnceng\/public\/_packaging\/dotnet-public/); }); +test("reserved source identity and a separate sample toolchain never retarget historical products", async t => { + const f = await controlled(t); + const buildInstanceId = "reserved-source-build-1"; + const result = await testSourceRevision({ + ...f.options, buildInstanceId, + request: { ...f.options.request, sourceSha: f.shas[1] }, + reproductionToolchain: f.toolchain, + }, f.deps); + assert.equal(result.buildInstanceId, buildInstanceId); + assert.equal(result.actualSourceSha, f.shas[1]); + assert.equal(result.toolchain.sdkVersion, "9.0.100"); + assert.equal(f.toolchain.sdkVersion, "10.0.100"); + assert.equal(result.product.targetFramework, "net9.0"); + assert.equal(result.outcome.outcome, "untestable"); + assert.match(result.reproduction.error, /runtime|identity|artifact/); + assert.equal(result.globalJson.before, result.globalJson.after); + const duplicate = await prepareRevisionWorkspace({ ...f.options, buildInstanceId }, f.deps); + assert.equal(duplicate.status, "untestable"); + assert.match(duplicate.error.message, /EEXIST/); +}); + test("large product inventories hash every dependency without unbounded inspection arguments", async t => { const f = await controlled(t); f.afterBuild = async source => { From 38693a684e8aa20380ac6b4e2e6e534e04814f8c Mon Sep 17 00:00:00 2001 From: Copilot Date: Sun, 20 Sep 2026 02:34:05 +0200 Subject: [PATCH 27/99] Fix bisection checkpoint size and confirmation accounting Persist compact intervals and reconstruct Git decisions from all comparable authenticated source evidence, including older-search failures and skips. Separate admission from the last completed item's attempt count and require fresh candidate/parent attempt capacity before confirmation. Cover large real-Git intervals, retries, cumulative/time boundaries, and worker-result integrity rejection. Keep policy limits and confirmation-only attribution unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 50 +++++--- eng/regression-bisect/engine.cjs | 29 +++-- eng/regression-bisect/engine.test.cjs | 177 +++++++++++++++++++++++++- eng/regression-bisect/model.cjs | 2 - 4 files changed, 224 insertions(+), 34 deletions(-) diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index dc2d3d3c16f..ab3a1878fbf 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -228,7 +228,7 @@ The order is stable failure, pinned-main failure, older-source pass, admission, then Git selection. Stable pass stops as not reproduced; main pass stops as fixed/release-specific. Untestable stable/main endpoints block. The deterministic older search considers approved source-SHA hints that ordinary ancestry queries -verify, then oldest-first candidates in a bounded 4,096-commit main window. +verify, then oldest-first candidates in a bounded 4,096-ancestor main window. SDK/version/URL claims are not converted to source observations or trusted tag mappings. Every selected source still requires a clean build and repeated sample. Search work is bounded by that window and the unchanged issue allowance. @@ -269,10 +269,14 @@ Unsupported historical runtime/sample combinations remain untestable. There is no engine-specific FSI command, compiler runner, SDK installer, retargeting or `global.json` rewrite. -The interval adds strict `decisions: [{ sha, outcome, observationDigest }]` and -`remaining: [sha]` fields. Decisions are saved with observations and the next -advance recomputes graph bounds from the authenticated ledger. The legacy -`gitLog` is never executed. A lost selection worktree is reconstructed using +The persisted interval contains good/bad SHAs, the legacy `gitLog`, and strict +`decisions: [{ sha, outcome, observationDigest }]`. It does not store the +potentially thousands of remaining SHAs in the 48 KiB checkpoint. Report intents +include `interval.remaining: [sha]`, recomputed from Git on each advance. +Reconstruction uses all valid comparable source observations in the interval, +including failures and skips from older-endpoint search, not just bisect workers. +Decisions are saved with observations; the legacy `gitLog` is never executed. +A lost selection worktree is reconstructed using `bisect start --no-checkout` and typed `good`, `bad`, or `skip` arguments only; there is no stored replay string, `bisect run`, or interpretation of generic nonzero build exits as bad. Each selection pass has a five-minute wall-time cap @@ -290,10 +294,16 @@ attempts and two assertion repetitions, with 90-minute builds, 110-minute workers and 300-second reproductions. **Six reservations consume 660 minutes**; the seventh does not fit even when each fixture finishes immediately. Candidate selection preserves room for candidate-plus-parent confirmation where feasible. -Otherwise it returns the unconfirmed interval and consumed budget. A singleton -interval returns `confirmation` with `candidate`, all relevant `parents`, and -`requiresFreshConfirmation: true`; a separate consumer must reserve and perform -fresh confirmation before any attribution. +Admission has no next selected item yet, so it must not mistake the ledger's +last completed item for that item after a successful retry. Actual work +reservations still enforce the two-attempt limit. Confirmation checks both +aggregate allowance/deadline and the remaining attempts for each concrete +candidate and parent, including their cancelled reservations under the same +approved input. Otherwise it returns the unconfirmed interval and consumed +budget. A singleton interval returns `confirmation` with `candidate`, all +relevant `parents`, and `requiresFreshConfirmation: true` only when those fresh +reservations fit; a separate consumer must reserve and perform them before any +attribution. `engine.test.cjs` uses real temporary Git histories and inexpensive injected revision evaluators, not product acceptance. It records every actual bisect @@ -302,19 +312,27 @@ and malformed/stale evidence. Process/GitHub/clock dependencies are injectable at the existing runner boundaries; production defaults call the original modules. The scoped suite covers execution claims, cumulative cancellation/retry budgets, environment/input/main/stable epochs, lost worktrees, skips, toolchain transitions -and uncertain histories. +and uncertain histories. Real 1,000- and 4,096-commit intervals exercise compact +checkpoint round-trips. Boundary cases check concrete confirmation attempts, +the last two worker units, exact deadline capacity, and rejection of worker +outcome/source/build/matrix/repetition mismatches without committing evidence. Fresh Windows x64 product acceptance is retained at -`Q:\rb912-engine-acceptance-20260920`. The original workspace runner clean-built +`Q:\rb912-engine-repair-20260920`. The original workspace runner clean-built `5a8b9adb034f420a5c2c1f577670c5eeeb454b7e` in build instance -`d6c3ce9a-a9a9-4f70-ae93-3a99a68c9a36` using its approved measured +`cda0551b-12a9-47bf-a4e7-372ba380a371` using its approved measured `11.0.100-rc.1.26420.103` archive, with unchanged `global.json` and two passing -FSI assertions. Released SDK `10.0.401` and the fresh source products then matched -all 43 existing reproduction-smoke expectations (23 passes, six intended product -failures, 14 identity rejections). The scoped evidence-helper Release build/test +FSI assertions. Released SDK `10.0.401` was reverified against current official +metadata and its retained archive. It and the fresh source products then matched +all 43 existing reproduction-smoke expectations (23 passes, six intended +product failures, 14 identity rejections). The scoped evidence-helper Release build/test commands exited 0; that helper project contains no managed test cases. +The targeted engine/model/state suite passed 498 tests and the full scoped suite +passed 1,007, including 75 zero-bisect cases and 37 actual bisect subprocesses. +Negative tests detected all eight in-memory removals of confirmation and +worker-result integrity guards; repository files were not mutated for those probes. `final-acceptance-report.json` has SHA-256 -`ccf46ba63cb50dc9ddc89b824aebb0ada1cf6c96d0c4b2a22310039c299f4864`. +`f4acb60b0a90bef48fc7df9f119be8cccbc76e664e54f057ec42798c179ef189`. This validates the original product runners, not a real regression's admitted interval, an activated workflow, or fresh culprit attribution. diff --git a/eng/regression-bisect/engine.cjs b/eng/regression-bisect/engine.cjs index 3439bc9671f..9f2051cc6d8 100644 --- a/eng/regression-bisect/engine.cjs +++ b/eng/regression-bisect/engine.cjs @@ -77,12 +77,15 @@ function authenticated(cp, id, environments) { return o; } -function capacity(cp, units, now) { +function capacity(cp, units, now, sourceShas = []) { const { consumed, policy } = cp.state.budget; return consumed.sourceBuilds + units <= policy.limits.sourceBuilds && consumed.workerUnits + units <= policy.limits.workerUnits && consumed.reservedMinutes + units * policy.limits.workerTimeoutMinutes <= policy.limits.reservedMinutes && - Date.parse(now) + units * policy.limits.workerTimeoutMinutes * 60000 <= Date.parse(cp.deadline); + Date.parse(now) + units * policy.limits.workerTimeoutMinutes * 60000 <= Date.parse(cp.deadline) && + sourceShas.every(sha => cp.records.filter(r => r.inputDigest === cp.inputDigest && + r.request.kind === "source" && r.request.target.sourceSha === sha).length + + sourceShas.filter(s => s === sha).length <= policy.limits.attemptsPerItem); } async function selectSource(cp, context, deps, history) { @@ -91,7 +94,7 @@ async function selectSource(cp, context, deps, history) { const stable = get(s.endpointEvidence.stable), main = get(s.endpointEvidence.main); const revisions = s.testedRevisions.map(r => ({ ...r, data: get(r.observationDigest) })); const all = (await history.git(["rev-list", "--topo-order", `--max-count=${maxHistory + 1}`, history.main])).split("\n"); - const older = all.slice(1, maxHistory); + const older = all.slice(1, maxHistory + 1); for (const r of revisions) if ((all.length <= maxHistory && !all.includes(r.sourceSha)) || !(await history.ancestor(r.sourceSha, history.main))) return report("blocked", "observed source is outside main ancestry", { historyEvidence: revisions.map(r => r.sourceSha) }); for (const bad of revisions.filter(r => r.data.outcome.outcome === "product-failure")) @@ -105,7 +108,7 @@ async function selectSource(cp, context, deps, history) { .filter(sha => older.includes(sha) && !revisions.some(r => r.sourceSha === sha)) .slice(0, s.budget.policy.limits.sourceBuilds); if (!candidates.length) return report("stopped", "no verified starting interval", { skips: s.skips }); - if (!capacity(cp, 3, context.now)) return report("blocked", "no verified starting interval; confirmation budget exhausted", + if (!capacity(cp, 3, context.now, [candidates[0]])) return report("blocked", "no verified starting interval; confirmation budget exhausted", { budget: s.budget.consumed, skips: s.skips }); return { request: { kind: "source", target: { sourceSha: candidates[0] }, endpoint: "good", phase: "good-search" } }; } @@ -121,10 +124,10 @@ async function selectSource(cp, context, deps, history) { const messages = await history.git(["log", "--format=%B%x00", history.main, `^${passing.sourceSha}`]); if (/\b(?:revert(?:ed|ing)?|reintroduc\w*|cherry[ -]pick\w*)\b/i.test(messages)) return report("blocked", "explicit revert/reintroduction/backport history; monotonicity uncertain", { interval: initialInterval }); - const decisions = cp.records.filter(r => r.status === "committed" && r.epoch === s.epoch && r.request.phase === "bisect") + const decisions = [...new Map(revisions.filter(r => r.sourceSha !== history.main && intervalShas.includes(r.sourceSha)) + .map(r => [r.sourceSha, r])).values()] .map(r => { - const o = get(r.observationDigest); - requireValue(intervalShas.includes(o.target.sourceSha), "decision outside admitted interval"); + const o = r.data; for (const c of o.cells) { const m = main.cells.find(m => m.id === c.id); requireValue(o.outcome.outcome === "untestable" || equal(c.sdk, m.sdk) && equal(c.runtime, m.runtime), @@ -132,7 +135,9 @@ async function selectSource(cp, context, deps, history) { } return { sha: o.target.sourceSha, outcome: o.outcome.outcome, observationDigest: r.observationDigest }; }); - const admission = admitBisection({ manifest: s.manifest, stable, main, good, ancestry, budget: s.budget }); + // Git has not selected a next item; the ledger's last completed item is not that item. + const admission = admitBisection({ manifest: s.manifest, stable, main, good, ancestry, + budget: { ...s.budget, nextWorkItemId: s.budget.pendingWork?.workItemId ?? "none" } }); if (admission.decision !== "admit") return report(admission.decision === "stop" ? "stopped" : "blocked", admission.reason); // No bisect command, including reset, is permitted before this gate. await history.git(["bisect", "start", "--no-checkout", history.main, passing.sourceSha]); @@ -151,7 +156,7 @@ async function selectSource(cp, context, deps, history) { if (ambiguous) return report("blocked", "all-skipped ambiguous interval", { interval, skips: s.skips }); if (remaining.length === 1) { const relevantParents = (await history.git(["rev-list", "--parents", "-n", "1", badSha])).split(" ").slice(1); - if (!capacity(cp, 1 + relevantParents.length, context.now)) + if (!capacity(cp, 1 + relevantParents.length, context.now, [badSha, ...relevantParents])) return report("blocked", "unconfirmed interval; insufficient fresh confirmation budget", { interval, budget: s.budget.consumed }); return report("confirmation", "candidate requires fresh candidate and parent assertions", { candidate: badSha, parents: relevantParents, interval, toolchainTransitions: admission.toolchainTransitions ?? [], @@ -161,6 +166,8 @@ async function selectSource(cp, context, deps, history) { { interval, skips: s.skips, budget: s.budget.consumed }); const sha = await history.git(["rev-parse", "BISECT_HEAD"]); requireValue(fullSha(sha) && remaining.includes(sha) && !decisions.some(d => d.sha === sha), "invalid repeated Git selection"); + if (!capacity(cp, 3, context.now, [sha])) return report("blocked", "unconfirmed interval; candidate attempt budget exhausted", + { interval, skips: s.skips, budget: s.budget.consumed }); return { interval, goodDigest: passing.observationDigest, request: { kind: "source", target: { sourceSha: sha }, endpoint: null, phase: "bisect" } }; } @@ -242,9 +249,11 @@ async function advanceInvestigation({ now }, deps) { } } if ((selected.interval || selected.status) && cp.pending === null) { + const { remaining, ...interval } = selected.interval ?? cp.state.interval ?? {}; cp = await recordSearchProgress(deps.transport, context, { generation: cp.generation, phase: selected.status ?? "bisect", - interval: selected.interval ?? cp.state.interval, ...(selected.goodDigest ? { goodDigest: selected.goodDigest } : {}), + interval: selected.interval || cp.state.interval ? interval : null, + ...(selected.goodDigest ? { goodDigest: selected.goodDigest } : {}), }); } if (selected.status) return { ...selected, generation: cp.generation, commands: history?.commands ?? [] }; diff --git a/eng/regression-bisect/engine.test.cjs b/eng/regression-bisect/engine.test.cjs index 786d97fbfbf..488c4dc5453 100644 --- a/eng/regression-bisect/engine.test.cjs +++ b/eng/regression-bisect/engine.test.cjs @@ -14,24 +14,27 @@ const gitExecutable = process.platform === "win32" ? execFileSync("where.exe", ["git.exe"], { encoding: "utf8" }).trim().split(/\r?\n/)[0] : execFileSync("which", ["git"], { encoding: "utf8" }).trim(); -async function fixture(t, outcomes = ["product-failure", "product-failure", "pass"], messages = ["base", "one", "two", "main"]) { +async function fixture(t, outcomes = ["product-failure", "product-failure", "pass"], messages = ["base", "one", "two", "main"], hints = [0]) { const root = await fs.mkdtemp(path.join(os.tmpdir(), "engine-test-")); t.after(() => fs.rm(root, { recursive: true, force: true })); const mirror = path.join(root, "mirror"), scratch = path.join(root, "scratch"); await fs.mkdir(mirror); await fs.mkdir(scratch); - const git = (...args) => execFileSync(gitExecutable, ["-c", "core.hooksPath=", "-c", "commit.gpgSign=false", ...args], { - cwd: mirror, encoding: "utf8", env: { ...process.env, GIT_CONFIG_NOSYSTEM: "1", + const runGit = (args, input) => execFileSync(gitExecutable, ["-c", "core.hooksPath=", "-c", "commit.gpgSign=false", ...args], { + cwd: mirror, input, encoding: "utf8", env: { ...process.env, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: path.join(root, "empty"), GIT_AUTHOR_NAME: "Fixture", GIT_AUTHOR_EMAIL: "fixture@example.invalid", GIT_COMMITTER_NAME: "Fixture", GIT_COMMITTER_EMAIL: "fixture@example.invalid", GIT_AUTHOR_DATE: "2026-09-01T00:00:00Z", GIT_COMMITTER_DATE: "2026-09-01T00:00:00Z" }, }).trim(); + const git = (...args) => runGit(args); await fs.writeFile(path.join(root, "empty"), ""); git("init", "--quiet", "--initial-branch=main"); - const shas = []; - for (const message of messages) { git("commit", "--quiet", "--allow-empty", "-m", message); shas.push(git("rev-parse", "HEAD")); } + runGit(["fast-import", "--quiet"], messages.map((message, i) => + `commit refs/heads/main\nmark :${i + 1}\ncommitter Fixture 1788220800 +0000\n` + + `data ${Buffer.byteLength(message)}\n${message}\n${i ? `from :${i}\n` : ""}\n`).join("")); + const shas = git("rev-list", "--reverse", "HEAD").split("\n"); const h = harness(); h.context.pinnedMainSha = shas.at(-1); - h.context.manifest.searchHints.sourceShas = [shas[0]]; + h.context.manifest.searchHints.sourceShas = hints.map(i => shas[i]); f.seal(h.context.manifest); h.context.authorization.manifestDigest = model.fingerprintInput(h.context.manifest); h.context.authorization.inputDigest = h.context.manifest.approvedInputDigest; @@ -347,6 +350,124 @@ test("a deterministic midpoint failure narrows bad without turning a build exit assert.ok(h.calls.some(args => args[1] === "bad" && args[2] === h.shas[1])); }); +test("supported-size intervals round-trip compact checkpoints and reconstruct remaining commits", async t => { + for (const size of [1000, 4096]) await t.test(`${size} commits`, async t => { + const h = await fixture(t, undefined, Array.from({ length: size + 1 }, (_, i) => `change ${i}`)); + await endpoints(h); + const work = await h.advance(); + assert.equal(work.status, "work", JSON.stringify(work)); + const admitted = (await state.loadCheckpoint(h.transport, h.context)).checkpoint.state.interval; + assert.equal(admitted.goodSha, h.shas[0]); + assert.equal(admitted.badSha, h.shas[size]); + await h.finish(work, "pass"); + h.nextRun(); + const result = await h.advance(); + assert.equal(result.status, "blocked", JSON.stringify(result)); + assert.match(result.reason, /preserving confirmation budget/); + const index = h.shas.indexOf(work.reservation.request.target.sourceSha); + assert.deepEqual(result.interval.remaining, h.shas.slice(index + 1).toReversed()); + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + assert.deepEqual(Object.keys(cp.state.interval).sort(), ["badSha", "decisions", "gitLog", "goodSha"]); + assert.equal(cp.state.interval.decisions.length, 1); + assert.ok(Buffer.byteLength(h.comment.body) < 48 * 1024); + assert.deepEqual(await fs.readdir(h.scratch), []); + }); +}); + +test("retried endpoints do not masquerade as the next item, but confirmation needs fresh attempts", async t => { + for (const retry of ["stable", "main", "good-search"]) await t.test(retry, async t => { + const h = await fixture(t, undefined, ["base", "main"]); + for (const phase of ["stable", "main", "good-search"]) { + let work = await h.advance(); + assert.equal(work.status, "work", JSON.stringify(work)); + if (phase === retry) { + h.nextRun(); + work = await h.advance(); + assert.equal(work.reservation.attempt, 2); + } + await h.finish(work); + } + const result = await h.advance(); + assert.equal(result.status, retry === "stable" ? "confirmation" : "blocked", JSON.stringify(result)); + assert.ok(h.calls.length > 0, "a completed retry is not the next admission work item"); + if (retry !== "stable") assert.match(result.reason, /fresh confirmation budget/); + assert.deepEqual(result.interval.remaining, [h.shas[1]]); + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + assert.equal(cp.state.budget.consumed.reservedMinutes, 440); + }); +}); + +test("older-search failures and skips are replayed only when comparable after resume", async t => { + for (const outcome of ["product-failure", "untestable", "incomparable"]) await t.test(outcome, async t => { + const h = await fixture(t, undefined, ["base", "introduced", "main"], [1, 0]); + for (const value of ["product-failure", "product-failure", outcome, "pass"]) { + const work = await h.advance(); + assert.equal(work.status, "work", JSON.stringify(work)); + assert.equal(h.calls.length, 0); + if (value === "incomparable") { + const observation = h.observationFor(work, "product-failure"); + observation.cells[0].runtime.version = "10.0.1"; + await state.commitObservation(h.transport, h.context, { generation: work.reservation.generation, + attemptId: work.reservation.id, observation, evidence: [] }); + } else await h.finish(work, value); + } + h.nextRun(); + const result = await h.advance(); + assert.equal(result.status, outcome === "product-failure" ? "confirmation" : "blocked", JSON.stringify(result)); + if (outcome === "incomparable") { + assert.match(result.reason, /reproduction toolchain changed/); + assert.equal(h.calls.length, 0); + return; + } + assert.ok(h.calls.some(args => args[1] === (outcome === "product-failure" ? "bad" : "skip") && args[2] === h.shas[1])); + if (outcome === "product-failure") { + assert.equal(result.candidate, h.shas[1]); + assert.deepEqual(result.parents, [h.shas[0]]); + assert.deepEqual(result.interval.remaining, [h.shas[1]]); + } else { + assert.match(result.reason, /all-skipped/); + assert.deepEqual(result.interval.remaining, h.shas.slice(1).toReversed()); + } + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + assert.equal(cp.state.interval.decisions[0].outcome, outcome); + assert.equal(cp.state.budget.consumed.workerUnits, 4); + }); +}); + +test("confirmation checks aggregate and wall-time boundaries and can actually reserve both targets", async t => { + for (const [name, extra, minutes, allowed] of [ + ["normal", 0, 220, true], ["last two units", 1, 220, true], + ["only one unit", 2, 220, false], ["one millisecond short", 0, 220 - 1 / 60000, false], + ]) await t.test(name, async t => { + const h = await fixture(t, undefined, ["base", "main"]); + await endpoints(h); + for (let i = 0; i < extra; i++) { + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + await state.reserveWork(h.transport, h.context, { generation: cp.generation, attemptId: `cancelled-${i}`, + request: { kind: "source", target: { sourceSha: f.sha(`cancelled-${i}`) }, endpoint: null, phase: "bisect" } }); + h.nextRun(); + } + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + cp.deadline = new Date(Date.parse(h.context.now) + minutes * 60000).toISOString(); + h.checkpoint(cp); + const result = await h.advance(); + assert.equal(result.status, allowed ? "confirmation" : "blocked", JSON.stringify(result)); + if (!allowed) assert.match(result.reason, /fresh confirmation budget/); + else for (const sourceSha of [result.candidate, ...result.parents]) { + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + const reservation = await state.reserveWork(h.transport, h.context, { + generation: cp.generation, attemptId: `confirm-${sourceSha}`, + request: { kind: "source", target: { sourceSha }, endpoint: null, phase: "confirmation" }, + }); + assert.equal(reservation.attempt, 2); + await h.finish({ reservation }, sourceSha === result.candidate ? "product-failure" : "pass"); + } + const end = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + assert.equal(end.state.budget.consumed.workerUnits, 3 + extra + (allowed ? 2 : 0)); + assert.equal(result.requiresFreshConfirmation, true); + }); +}); + test("observed descendant passes and unrelated branch claims stop before admission", async t => { for (const mismatch of [false, true]) await t.test(mismatch ? "ancestry" : "nonmonotonic", async t => { const h = await fixture(t); @@ -533,3 +654,47 @@ test("worker seam preserves original FSI/project inputs and charges before invok assert.equal(h.calls.length, 0); }); }); + +test("worker-result integrity mismatches cannot commit collector-shaped success", async t => { + const variants = { + outcome: r => { r.outcome = f.observation("released", "pass").outcome; }, + missingMatrix: (r, cells) => { cells.length = 0; }, + missingRepetition: (r, cells) => { cells[0].attempts.pop(); }, + alteredAssertion: (r, cells) => { cells[0].attempts[0].execution.stderr = "unrelated failure"; }, + sourceStatus: r => { r.status = "untestable"; }, + sourceSha: r => { r.actualSourceSha = f.sha("other source"); }, + sourceBuild: r => { r.buildInstanceId = "unreserved-build"; }, + }; + for (const kind of ["released", "source"]) + for (const [name, mutate] of Object.entries(variants)) { + if (kind === "released" && name.startsWith("source")) continue; + await t.test(`${kind}/${name}`, async t => { + const h = await fixture(t); + if (kind === "source") await h.finish(await h.advance()); + const work = await h.advance(); + const observation = h.observationFor(work, "product-failure"); + const cells = structuredClone(observation.cells); + const result = kind === "released" ? { outcome: observation.outcome, cells } : + { status: "tested", outcome: observation.outcome, actualSourceSha: work.reservation.request.target.sourceSha, + buildInstanceId: work.reservation.id, reproduction: { cells } }; + mutate(result, cells); + const before = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + await assert.rejects(executeInvestigationWork(work, { + ...h.deps, clock: () => new Date(h.context.now), + worker: { reproductionToolchain: {}, storageDirectory: "unused" }, + acquireToolchain: async () => ({ status: "acquired", identity: { + requested: {}, archive: { sha256: h.context.stableSdk.archives[0].sha256 }, + } }), + verifyToolchainIdentity: () => ({ status: "verified" }), inventoryReleased: async () => ({}), + runReproduction: async () => result, testSourceRevision: async () => result, + collectObservation: async () => ({ observation, evidence: [] }), + }), /collector changed worker outcome|source result identity mismatch|collector omitted original repeated assertions/); + const after = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + assert.deepEqual(after.observations, before.observations); + assert.deepEqual(after.state.endpointEvidence, before.state.endpointEvidence); + assert.deepEqual(after.state.budget.consumed, before.state.budget.consumed); + assert.equal(after.state.budget.pendingWork.status, "running"); + assert.equal(h.calls.length, 0); + }); + } +}); diff --git a/eng/regression-bisect/model.cjs b/eng/regression-bisect/model.cjs index 7296d54034a..aa6e890e26a 100644 --- a/eng/regression-bisect/model.cjs +++ b/eng/regression-bisect/model.cjs @@ -459,7 +459,6 @@ function validateState(state) { ...(Object.hasOwn(v, "decisions") ? { decisions: list(shape({ sha: sourceSha, outcome: oneOf(["pass", "product-failure", "untestable"]), observationDigest: digest }), 0, trustedPolicy.limits.sourceBuilds), - remaining: list(sourceSha, 1, 4096), } : {}), }, p)), publication: shape({ digest: nullable(digest), commentId: nullable(positive) }), @@ -477,7 +476,6 @@ function validateState(state) { check(evidence.every(d => budget.authenticatedObservationDigests.includes(d)), "state", "unauthenticated evidence"); if (state.interval?.decisions) { unique(state.interval.decisions.map(d => d.sha), "state.interval.decisions"); - unique(state.interval.remaining, "state.interval.remaining"); } return true; } catch (error) { From a97ead560872fd1517ec278847c44374fa2ac444 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sun, 20 Sep 2026 03:46:52 +0200 Subject: [PATCH 28/99] Revalidate bisection workers at execution boundaries Reject stale input and endpoint identities before claiming reserved work, enforce its absolute deadline across asynchronous setup, and separate sample SDK acquisition provenance from tested source identity. Add real-runner historical runtime coverage and count denied bisect processes independently of engine error handling. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 62 +++++++--- eng/regression-bisect/engine.cjs | 110 ++++++++++++------ eng/regression-bisect/engine.test.cjs | 108 ++++++++++++++++- .../fixtures/reproduction.cjs | 50 +++++++- eng/regression-bisect/repro.cjs | 4 +- eng/regression-bisect/repro.test.cjs | 64 ++++------ eng/regression-bisect/workspace.test.cjs | 80 ++++++++++++- 7 files changed, 376 insertions(+), 102 deletions(-) diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index ab3a1878fbf..c2c11e4ecd5 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -238,12 +238,24 @@ and exact assertions, but not equal revision **build** SDKs. Source observations used by the engine must retain `buildToolchain`. Different build toolchains are returned as transition evidence, never erased to make endpoints comparable. -`executeInvestigationWork(work, dependencies)` executes the returned permission. -It calls `beginWork` before invoking any runner, making a lost acknowledgement -or duplicate worker invocation non-executable. The state ledger keeps the -reservation owner, attempt and execution generation. A cancelled owning run is -retired only through `reserveWork`/`invalidateStaleEvidence`; retrying costs another -full reservation and never more than two attempts per item. +`executeInvestigationWork(work, dependencies)` revalidates the returned permission. +It needs the same current trusted context, intake/approval readers, official +metadata resolver and (for source work) history mirror as selection. It re-reads +complete discussion, immutable assets, eligibility and active investigations, +checks the authenticated epoch and binding, re-resolves latest stable, and +checks current main against the saved pin before claiming execution. The saved +`binding` cannot override current issue data; the returned `resolved` record is +not used instead of execution-time release metadata. + +It then calls `beginWork` before invoking any runner, making a lost acknowledgement +or duplicate worker invocation non-executable. Deadline checks surround +asynchronous claiming, acquisition, inventory and collection; the abort signal +uses the reservation's remaining time, not a fresh worker timeout. An expired +claim cannot start a runner even if its checkpoint write completed successfully. +The state ledger keeps the reservation owner, attempt and execution generation. +Rejected, expired and cancelled work is not refunded. A cancelled owning run is +retired only through `reserveWork`/`invalidateStaleEvidence`; retrying costs +another full reservation and never more than two attempts per item. Worker dependencies include `worker` options for the existing runners, `inventoryReleased(identity, buildInstanceId)`, and @@ -263,8 +275,14 @@ Released execution uses `acquireToolchain`, `verifyToolchainIdentity` and revision's declarations, acquires its own build toolchain, and creates fresh Bootstrap/Proto, bin/obj and package state. `worker.reproductionToolchain` is required separately for engine source work; optional `sampleRuntimes` freezes -explicit sample runtimes. The workspace runner now accepts a reserved -`buildInstanceId`; reusing its directory fails rather than reusing outputs. +explicit sample runtimes. A source-derived sample SDK retains its acquisition +`requested.sourceSha` as provenance, independently of the tested product SHA. +The workspace/engine bind the product to the reserved source; the reproduction +runner still verifies its local artifacts, host, archive and actual compiler, +sample-runtime and Core loads. Compatible historical projects may therefore +run a net9 compiler with the frozen SDK 10/net10 sample without retargeting. +The workspace runner accepts a reserved `buildInstanceId`; reusing its directory +fails rather than reusing outputs. Unsupported historical runtime/sample combinations remain untestable. There is no engine-specific FSI command, compiler runner, SDK installer, retargeting or `global.json` rewrite. @@ -307,32 +325,40 @@ attribution. `engine.test.cjs` uses real temporary Git histories and inexpensive injected revision evaluators, not product acceptance. It records every actual bisect -subprocess and exit code, including zero counts for all denied endpoint tuples -and malformed/stale evidence. Process/GitHub/clock dependencies are injectable -at the existing runner boundaries; production defaults call the original modules. +subprocess and exit code before checking saved endpoint evidence, so an engine +catch cannot hide a forbidden invocation. This includes zero counts for all +denied endpoint tuples and malformed/stale evidence. Process/GitHub/clock +dependencies are injectable at the existing runner boundaries; production +defaults call the original modules. The scoped suite covers execution claims, cumulative cancellation/retry budgets, environment/input/main/stable epochs, lost worktrees, skips, toolchain transitions and uncertain histories. Real 1,000- and 4,096-commit intervals exercise compact checkpoint round-trips. Boundary cases check concrete confirmation attempts, the last two worker units, exact deadline capacity, and rejection of worker outcome/source/build/matrix/repetition mismatches without committing evidence. +Reservation-to-execution edits and clock advances during asynchronous setup +must start no stale or expired worker. Shared project evidence fixtures exercise +the real workspace/reproduction seam with distinct historical compiler and +frozen sample runtimes, for both released and source-derived sample acquisitions. Fresh Windows x64 product acceptance is retained at -`Q:\rb912-engine-repair-20260920`. The original workspace runner clean-built +`Q:\rb912-engine-hardening-20260920`. The original workspace runner clean-built `5a8b9adb034f420a5c2c1f577670c5eeeb454b7e` in build instance -`cda0551b-12a9-47bf-a4e7-372ba380a371` using its approved measured +`fbe0059e-2dfd-4ce2-b160-58a65f6092b9` using its approved measured `11.0.100-rc.1.26420.103` archive, with unchanged `global.json` and two passing FSI assertions. Released SDK `10.0.401` was reverified against current official metadata and its retained archive. It and the fresh source products then matched all 43 existing reproduction-smoke expectations (23 passes, six intended product failures, 14 identity rejections). The scoped evidence-helper Release build/test commands exited 0; that helper project contains no managed test cases. -The targeted engine/model/state suite passed 498 tests and the full scoped suite -passed 1,007, including 75 zero-bisect cases and 37 actual bisect subprocesses. -Negative tests detected all eight in-memory removals of confirmation and -worker-result integrity guards; repository files were not mutated for those probes. +The targeted engine/model/state suite passed 525 tests and the full scoped suite +passed 1,044, including 100 zero-bisect cases and 37 actual bisect subprocesses. +Seven in-memory mutations were detected: bypassed current intake, released SDK +and main freshness, removed asynchronous deadline checks, omitted sample-runtime +forwarding, recoupled acquisition/product source provenance, and a forbidden +pre-admission bisect reset. Repository files were not mutated for those probes. `final-acceptance-report.json` has SHA-256 -`f4acb60b0a90bef48fc7df9f119be8cccbc76e664e54f057ec42798c179ef189`. +`02ef9214a9835070a846ab1cc13678a8ebe0617cdc01e3f717cba60628042528`. This validates the original product runners, not a real regression's admitted interval, an activated workflow, or fresh culprit attribution. diff --git a/eng/regression-bisect/engine.cjs b/eng/regression-bisect/engine.cjs index 9f2051cc6d8..9d3227729be 100644 --- a/eng/regression-bisect/engine.cjs +++ b/eng/regression-bisect/engine.cjs @@ -172,29 +172,46 @@ async function selectSource(cp, context, deps, history) { request: { kind: "source", target: { sourceSha: sha }, endpoint: null, phase: "bisect" } }; } +async function currentInput(now, deps) { + const context = { ...deps.context, now, environmentFingerprint: fingerprint(deps.environments) }; + validateManifest(context.manifest); + requireValue(["readSource", "authenticateRecord", "readTrustedCheckpoints"].every(k => typeof deps.approval?.[k] === "function"), + "pinned approval assets and authenticated intake readers required"); + const snapshot = await (deps.intake?.readIssueSnapshot ?? intake.readIssueSnapshot)({ + ...deps.approval, transport: deps.intakeTransport ?? deps.transport, repository: context.repository, + issueNumber: context.issueNumber, reviewedManifest: context.manifest, now, + }); + if (snapshot.status !== "complete") return report(snapshot.status === "ineligible" ? "stopped" : "blocked", snapshot.reason); + const current = snapshot.snapshot; + requireValue(current.investigations?.status === "complete" && current.input.reproduction && + equal(current.input.reproduction.manifest, context.manifest) && + current.discussionFingerprint === context.manifest.discussionFingerprint, "incomplete or stale approved input"); + context.issue = current.input.discussion.issue; + context.issueFingerprint = current.discussionFingerprint; + requireValue(isEligibleIssue(context.issue), "issue is no longer eligible"); + const active = current.investigations.active.filter(a => !(a.basis === "trusted-checkpoint-active-run" && + a.commentId === context.commentId && a.owner.runId === context.owner.runId && a.owner.attempt === context.owner.attempt)); + return active.length ? report("blocked", "active linked investigation", { active }) : { context }; +} + +async function latestStable(context, deps, signal) { + const resolved = await (deps.toolchains?.resolveLatestStableSdk ?? toolchains.resolveLatestStableSdk)( + { policy: context.policy, platform: deps.platform, signal }, + { clock: () => new Date(context.now), ...deps.toolchainDependencies }); + requireValue(resolved.status === "resolved", `latest stable resolution blocked: ${resolved.error?.message}`); + requireValue(resolved.toolchain.sdkVersion === context.stableSdk.version && + resolved.toolchain.metadata.some(m => m.url === context.stableSdk.releaseIndexUrl && m.sha256 === context.stableSdk.releaseIndexDigest), + "latest stable identity changed; authenticated endpoint context must be refreshed"); + return resolved.toolchain; +} + async function advanceInvestigation({ now }, deps) { let history, cp, context; const signal = AbortSignal.any([AbortSignal.timeout(300000), ...(deps.signal ? [deps.signal] : [])]); try { - context = { ...deps.context, now, environmentFingerprint: fingerprint(deps.environments) }; - validateManifest(context.manifest); - requireValue(["readSource", "authenticateRecord", "readTrustedCheckpoints"].every(k => typeof deps.approval?.[k] === "function"), - "pinned approval assets and authenticated intake readers required"); - const snapshot = await (deps.intake?.readIssueSnapshot ?? intake.readIssueSnapshot)({ - ...deps.approval, transport: deps.intakeTransport ?? deps.transport, repository: context.repository, - issueNumber: context.issueNumber, reviewedManifest: context.manifest, now, - }); - if (snapshot.status !== "complete") return report(snapshot.status === "ineligible" ? "stopped" : "blocked", snapshot.reason); - const current = snapshot.snapshot; - requireValue(current.investigations?.status === "complete" && current.input.reproduction && - equal(current.input.reproduction.manifest, context.manifest) && - current.discussionFingerprint === context.manifest.discussionFingerprint, "incomplete or stale approved input"); - context.issue = current.input.discussion.issue; - context.issueFingerprint = current.discussionFingerprint; - requireValue(isEligibleIssue(context.issue), "issue is no longer eligible"); - const active = current.investigations.active.filter(a => !(a.basis === "trusted-checkpoint-active-run" && - a.commentId === context.commentId && a.owner.runId === context.owner.runId && a.owner.attempt === context.owner.attempt)); - if (active.length) return report("blocked", "active linked investigation", { active }); + const input = await currentInput(now, deps); + if (!input.context) return input; + context = input.context; let loaded = await loadCheckpoint(deps.transport, context); cp = loaded.checkpoint; if (context.pinnedMainSha !== cp.state.pinnedMainSha) { @@ -209,12 +226,7 @@ async function advanceInvestigation({ now }, deps) { } validateState(cp.state); requireValue(cp.state.epoch === fingerprintEpoch(context.manifest, cp.state.budget, context.pinnedMainSha), "stale epoch"); - const resolved = await (deps.toolchains?.resolveLatestStableSdk ?? toolchains.resolveLatestStableSdk)( - { policy: context.policy, platform: deps.platform, signal }, { clock: () => new Date(now), ...deps.toolchainDependencies }); - requireValue(resolved.status === "resolved", `latest stable resolution blocked: ${resolved.error?.message}`); - requireValue(resolved.toolchain.sdkVersion === context.stableSdk.version && - resolved.toolchain.metadata.some(m => m.url === context.stableSdk.releaseIndexUrl && m.sha256 === context.stableSdk.releaseIndexDigest), - "latest stable identity changed; authenticated endpoint context must be refreshed"); + const resolved = await latestStable(context, deps, signal); if (cp.pending) { const owner = cp.pending.owner; const run = await deps.transport.getRun(owner.runId, owner.attempt); @@ -263,7 +275,7 @@ async function advanceInvestigation({ now }, deps) { return { status: "work", reservation, binding: { issue: context.issue, issueFingerprint: context.issueFingerprint, pinnedMainSha: context.pinnedMainSha, environmentFingerprint: context.environmentFingerprint, - }, resolved: resolved.toolchain, commands: history?.commands ?? [] }; + }, resolved, commands: history?.commands ?? [] }; } catch (error) { return report("blocked", error.message, { ...(cp ? { budget: cp.state.budget.consumed } : {}), commands: history?.commands ?? [] }); } finally { if (history) await history.dispose(); } @@ -271,22 +283,44 @@ async function advanceInvestigation({ now }, deps) { async function executeInvestigationWork(work, deps) { requireValue(work.status === "work" && work.reservation.execute, "work has no new execution permission"); - const context = { ...deps.context, now: (deps.clock ?? (() => new Date()))().toISOString(), - issue: work.binding.issue, issueFingerprint: work.binding.issueFingerprint, - pinnedMainSha: work.binding.pinnedMainSha, environmentFingerprint: fingerprint(deps.environments) }; - const loaded = await loadCheckpoint(deps.transport, context); + const clock = deps.clock ?? (() => new Date()); + const input = await currentInput(clock().toISOString(), deps); + requireValue(input.context, input.reason); + const context = input.context; + let loaded = await loadCheckpoint(deps.transport, context); + if (context.pinnedMainSha !== loaded.checkpoint.state.pinnedMainSha) { + context.pinnedMainSha = loaded.checkpoint.state.pinnedMainSha; + loaded = await loadCheckpoint(deps.transport, context); + } const r = loaded.checkpoint.pending; requireValue(loaded.executionAllowed && r?.id === work.reservation.id && r.generation === work.reservation.generation && equal(r.request, work.reservation.request), "work reservation no longer current"); + requireValue(equal(work.binding, { issue: context.issue, issueFingerprint: context.issueFingerprint, + pinnedMainSha: context.pinnedMainSha, environmentFingerprint: context.environmentFingerprint }), "stale work binding"); requireValue(typeof deps.collectObservation === "function", "trusted observation collector required"); requireValue(r.request.kind !== "source" || deps.worker.reproductionToolchain, "frozen reproduction toolchain required separately from revision build SDK"); - const started = await beginWork(deps.transport, context, { generation: r.generation, attemptId: r.id }); + const remaining = Date.parse(r.deadline) - clock().getTime(); + requireValue(remaining > 0, "reservation deadline exhausted"); + const signal = AbortSignal.any([AbortSignal.timeout(remaining), ...(deps.signal ? [deps.signal] : [])]); + const checkDeadline = () => { + signal.throwIfAborted(); + requireValue(clock().getTime() < Date.parse(r.deadline), "reservation deadline exhausted"); + }; + checkDeadline(); + const resolved = await latestStable({ ...context, now: clock().toISOString() }, deps, signal); + if (r.request.kind === "source") { + const history = await historyWorkspace(deps.history, deps.execute ?? execute, signal); + try { requireValue(history.main === context.pinnedMainSha, "current main changed; source work must be reselected"); } + finally { await history.dispose(); } + } + checkDeadline(); + const started = await beginWork(deps.transport, { ...context, now: clock().toISOString() }, + { generation: r.generation, attemptId: r.id }); if (!started.execute) return report("blocked", "work execution already claimed"); const manifest = loaded.checkpoint.state.manifest; - const signal = AbortSignal.any([AbortSignal.timeout(Math.max(1, Date.parse(r.deadline) - Date.parse(context.now))), - ...(deps.signal ? [deps.signal] : [])]); + checkDeadline(); let result; if (r.request.kind === "source") { result = await (deps.testSourceRevision ?? testSourceRevision)({ @@ -299,8 +333,9 @@ async function executeInvestigationWork(work, deps) { }, deps.workspaceDependencies); } else { const acquired = await (deps.acquireToolchain ?? toolchains.acquireToolchain)({ - policy: context.policy, toolchain: work.resolved, storageDirectory: deps.worker.storageDirectory, signal, + policy: context.policy, toolchain: resolved, storageDirectory: deps.worker.storageDirectory, signal, }, deps.toolchainDependencies); + checkDeadline(); if (acquired.status !== "acquired") result = acquired; else { const verified = (deps.verifyToolchainIdentity ?? toolchains.verifyToolchainIdentity)({ @@ -309,14 +344,17 @@ async function executeInvestigationWork(work, deps) { requireValue(verified.status === "verified", "stable acquisition identity rejected"); requireValue(acquired.identity.archive.sha256 === context.stableSdk.archives.find(a => a.os === deps.platform.os && a.architecture === deps.platform.architecture)?.sha256, "stable archive identity changed"); + checkDeadline(); const product = await deps.inventoryReleased(acquired.identity, r.id); + checkDeadline(); result = await (deps.runReproduction ?? runReproduction)({ ...deps.worker, manifest, product, toolchain: acquired.identity, }, { execute: (exe, args, settings) => (deps.workerExecute ?? execute)(exe, args, { ...settings, signal }) }); } } - signal.throwIfAborted(); + checkDeadline(); const collected = await deps.collectObservation({ result, reservation: r, manifest, context }); + checkDeadline(); validateObservation(collected.observation, manifest); requireValue(equal(collected.observation.outcome, result.outcome), "collector changed worker outcome"); if (result.outcome.outcome !== "untestable") { @@ -327,7 +365,7 @@ async function executeInvestigationWork(work, deps) { collected.observation.cells.every(c => equal(c.attempts, reproduction.cells.find(r => r.id === c.id)?.attempts)), "collector omitted original repeated assertions"); } - return commitObservation(deps.transport, { ...context, now: (deps.clock ?? (() => new Date()))().toISOString() }, { + return commitObservation(deps.transport, { ...context, now: clock().toISOString() }, { generation: started.generation, attemptId: r.id, observation: collected.observation, evidence: collected.evidence, }); } diff --git a/eng/regression-bisect/engine.test.cjs b/eng/regression-bisect/engine.test.cjs index 488c4dc5453..885bd86a47e 100644 --- a/eng/regression-bisect/engine.test.cjs +++ b/eng/regression-bisect/engine.test.cjs @@ -43,8 +43,11 @@ async function fixture(t, outcomes = ["product-failure", "product-failure", "pas h.context.environmentFingerprint = model.fingerprintInput(environments); await h.start(); const trace = [], calls = []; - t.after(() => t.diagnostic(JSON.stringify({ bisectCount: calls.length, - calls: calls.map(args => ({ args: [...args], exitCode: args.exitCode })) }))); + t.after(() => { + t.diagnostic(JSON.stringify({ bisectCount: calls.length, + calls: calls.map(args => ({ args: [...args], exitCode: args.exitCode })) })); + assert.ok(calls.every(c => c.endpointsSaved), "bisect preceded durable endpoints"); + }); const snapshot = { status: "complete", snapshot: { discussionFingerprint: h.context.issueFingerprint, input: { discussion: { issue: h.context.issue }, reproduction: { manifest: h.context.manifest } }, @@ -65,14 +68,16 @@ async function fixture(t, outcomes = ["product-failure", "product-failure", "pas execute: async (exe, args, settings) => { let call; if (args.includes("bisect")) { - const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; - assert.ok(Object.values(cp.state.endpointEvidence).every(Boolean), "bisect preceded durable endpoints"); call = args.slice(args.indexOf("bisect")); calls.push(call); } const result = spawnSync(exe, args, { ...settings, encoding: "utf8", timeout: settings.timeoutMs, maxBuffer: settings.maxBytes }); - if (result.error) throw result.error; if (call) call.exitCode = result.status; + if (result.error) throw result.error; + if (call) { + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + call.endpointsSaved = Object.values(cp.state.endpointEvidence).every(Boolean); + } return { status: "completed", complete: !result.signal, exitCode: result.status, stdout: result.stdout, stderr: result.stderr }; }, }; @@ -583,6 +588,18 @@ test("default intake freezes complete corrected discussion and immutable bytes b assert.equal(work.reservation.request.phase, "stable"); assert.equal(reads, 1); assert.equal(h.calls.length, 0); + const worker = async () => assert.fail("changed approved bytes or discussion must not execute"); + const dependencies = { ...h.deps, clock: () => new Date(h.context.now), worker: {}, + acquireToolchain: worker, collectObservation: worker }; + h.deps.approval.readSource = async () => { reads++; return Buffer.from("changed immutable source"); }; + await assert.rejects(executeInvestigationWork(work, dependencies), /hash mismatch/); + assert.equal(reads, 2); + comment.body = "later correction"; + await assert.rejects(executeInvestigationWork(work, dependencies), /discussion.*stale/); + assert.equal(reads, 2, "stale discussion must be rejected before reading reproduction assets"); + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + assert.equal(cp.state.budget.consumed.reservedMinutes, 110); + assert.equal(cp.pending.executionGeneration, undefined); }); test("worker seam preserves original FSI/project inputs and charges before invoking existing runners", async t => { @@ -698,3 +715,84 @@ test("worker-result integrity mismatches cannot commit collector-shaped success" }); } }); + +test("reserved workers re-read current input and endpoint identities before execution", async t => { + const variants = { + closed: h => { h.snapshot.snapshot.input.discussion.issue = { ...h.context.issue, state: "closed" }; }, + label: h => { h.snapshot.snapshot.input.discussion.issue = { ...h.context.issue, labels: ["regression"] }; }, + discussion: h => { h.snapshot.snapshot.discussionFingerprint = f.digest("new correction"); }, + incomplete: h => { h.snapshot.status = "blocked"; h.snapshot.reason = "incomplete current discussion"; }, + active: h => { h.snapshot.snapshot.investigations.active.push({ basis: "timeline-cross-reference" }); }, + approval: h => { h.context.authorization.approvalRevision = f.sha("changed approval"); }, + reproduction: h => { h.context.manifest.assertions.pass.stdout = "changed"; f.seal(h.context.manifest); }, + environment: h => { h.deps.environments["linux-x64"].imageDigest = f.digest("new runner"); }, + stable: h => { h.deps.toolchains.resolveLatestStableSdk = async () => ({ + status: "resolved", toolchain: { sdkVersion: "10.0.999", metadata: [] }, + }); }, + main: h => { h.git("commit", "--quiet", "--allow-empty", "-m", "new main"); }, + }; + for (const kind of ["released", "source"]) + for (const [name, mutate] of Object.entries(variants)) { + if (kind === "released" && name === "main") continue; + await t.test(`${kind}/${name}`, async t => { + const h = await fixture(t); + if (kind === "source") await h.finish(await h.advance()); + const work = await h.advance(); + const before = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + mutate(h); + let executions = 0; + const worker = async () => { executions++; throw new Error("stale worker started"); }; + await assert.rejects(executeInvestigationWork(work, { + ...h.deps, clock: () => new Date(h.context.now), worker: { reproductionToolchain: {} }, + acquireToolchain: worker, testSourceRevision: worker, collectObservation: worker, + }), /eligible|stale|current|incomplete|active|approval|identity|epoch/); + assert.equal(executions, 0); + assert.equal(h.calls.length, 0); + const after = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + assert.deepEqual(after.observations, before.observations); + assert.deepEqual(after.state.budget.consumed, before.state.budget.consumed); + assert.equal(after.pending.executionGeneration, undefined); + }); + } +}); + +test("elapsed reservation deadlines prevent later worker stages without refunding claims", async t => { + for (const stage of ["before", "claim", "acquisition", "inventory"]) + for (const kind of ["released", "source"]) { + if (kind === "source" && ["acquisition", "inventory"].includes(stage)) continue; + await t.test(`${kind}/${stage}`, async t => { + const h = await fixture(t); + if (kind === "source") await h.finish(await h.advance()); + const work = await h.advance(); + let now = h.context.now; + const expire = () => { now = work.reservation.deadline; }; + if (stage === "before") expire(); + if (stage === "claim") { + const update = h.transport.updateComment; + h.transport.updateComment = async (...args) => { const value = await update(...args); expire(); return value; }; + } + const calls = []; + const before = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + await assert.rejects(executeInvestigationWork(work, { + ...h.deps, clock: () => new Date(now), worker: { reproductionToolchain: {} }, + acquireToolchain: async () => { + calls.push("acquisition"); + if (stage === "acquisition") expire(); + return { status: "acquired", identity: { requested: {}, + archive: { sha256: h.context.stableSdk.archives[0].sha256 } } }; + }, + verifyToolchainIdentity: () => ({ status: "verified" }), + inventoryReleased: async () => { calls.push("inventory"); if (stage === "inventory") expire(); return {}; }, + runReproduction: async () => { calls.push("reproduction"); throw new Error("expired reproduction started"); }, + testSourceRevision: async () => { calls.push("source"); throw new Error("expired source started"); }, + collectObservation: async () => assert.fail("expired work cannot collect"), + }), /deadline|timeout/i); + assert.deepEqual(calls, stage === "acquisition" ? ["acquisition"] : + stage === "inventory" ? ["acquisition", "inventory"] : []); + const after = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + assert.deepEqual(after.state.budget.consumed, before.state.budget.consumed); + assert.deepEqual(after.observations, before.observations); + assert.equal(after.pending.id, work.reservation.id); + }); + } +}); diff --git a/eng/regression-bisect/fixtures/reproduction.cjs b/eng/regression-bisect/fixtures/reproduction.cjs index e1683370919..bf7757eae45 100644 --- a/eng/regression-bisect/fixtures/reproduction.cjs +++ b/eng/regression-bisect/fixtures/reproduction.cjs @@ -75,4 +75,52 @@ async function fixture(t, { kind = "source", executable = false, tfms = ["net10. return { root, file, product, toolchain, manifest: m, sourceDirectory: path.join(root, "source") }; } -module.exports = { fixture }; +async function projectBuild({ product, toolchain, cells, settings, defines = [], nullable }) { + const write = async (location, contents) => { + await fs.mkdir(path.dirname(location), { recursive: true }); + await fs.writeFile(location, contents); + return { path: location, sha256: hash(contents) }; + }; + const roles = Object.fromEntries(product.files.map(p => [p.role, p])); + const packageDirectory = path.join(settings.env.NUGET_PACKAGES, "fsharp.core", product.corePackageVersion); + const core = await write(path.join(packageDirectory, "lib", "netstandard2.0", "FSharp.Core.dll"), await fs.readFile(roles.core.path)); + await write(path.join(packageDirectory, `fsharp.core.${product.corePackageVersion}.nupkg`), await fs.readFile(roles.corePackage.path)); + const sdkImports = []; + for (const name of ["Sdk.props", "Sdk.targets"]) + sdkImports.push((await write(path.join(toolchain.sdkBasePath, "Sdks", "Microsoft.NET.Sdk", "Sdk", name), name)).path); + const rows = []; + for (const [index, cell] of cells.entries()) { + const row = { + project: path.join(settings.cwd, cell.project), targetFramework: cell.tfm, configuration: cell.configuration, + expectedLanguageVersion: cell.languageVersion, expectedOptimize: cell.configuration === "Release", + expectedDefines: defines, expectedNullable: nullable, projectReferences: cell.projectReferences ?? [], + assetsFile: path.join(settings.cwd, ".runner", `${index}.assets.json`), + targetPath: path.join(settings.cwd, "bin", cell.tfm, `${path.parse(cell.project).name}.dll`), + sdkVersion: toolchain.sdkVersion, hostPath: toolchain.hostPath, complete: true, + imports: [...["props", "targets", "sdkTargets", "overrides"].map(r => roles[r].path), ...sdkImports], + fscTasks: [{ compilerPath: roles.compiler.path, taskAssembly: roles.tasks.path, targetsPath: roles.targets.path, + hostPath: toolchain.hostPath, references: [core.path], languageVersion: cell.languageVersion, + optimize: cell.configuration === "Release", defines, nullable, + commandLine: `"${toolchain.hostPath}" "${roles.compiler.path}" --langversion:${cell.languageVersion}` }], + }; + const identity = `FSharp.Core/${product.corePackageVersion}`; + await write(row.assetsFile, JSON.stringify({ + targets: { [cell.tfm]: { [identity]: { + compile: { "lib/netstandard2.0/FSharp.Core.dll": {} }, runtime: { "lib/netstandard2.0/FSharp.Core.dll": {} }, + } } }, + libraries: { [identity]: { type: "package", path: `fsharp.core/${product.corePackageVersion}` } }, + packageFolders: { [settings.env.NUGET_PACKAGES]: {} }, + })); + await write(row.targetPath, "synthetic output"); + const runtime = toolchain.runtimes.find(r => r.name === product.runtime.name && r.version === product.runtime.version); + await write(path.join(settings.cwd, ".runner", `compiler-core-${index}.jsonl`), JSON.stringify({ + ...roles.core, path: path.join(path.dirname(roles.compiler.path), "FSharp.Core.dll"), + assemblyName: "FSharp.Core", complete: true, entryAssembly: roles.compiler.path, + runtimeVersion: runtime.version, runtimePath: path.join(runtime.path, runtime.version), architecture: toolchain.architecture, + })); + rows.push(row); + } + return { core, structured: { cells: rows, diagnostics: [] } }; +} + +module.exports = { fixture, projectBuild }; diff --git a/eng/regression-bisect/repro.cjs b/eng/regression-bisect/repro.cjs index 8dcbf14d797..3a51cb5b644 100644 --- a/eng/regression-bisect/repro.cjs +++ b/eng/regression-bisect/repro.cjs @@ -361,8 +361,8 @@ async function runReproduction(options, dependencies = {}) { if (installation.product !== "sdk") await verifyRetainedArchive(installation.archive); await checkedFile({ path: toolchain.hostPath, sha256: toolchain.hostSha256 }, toolchain.root); requireValue(["source", "released"].includes(options.product.kind) && options.product.buildInstanceId, "unidentified product build"); - if (options.product.kind === "source") requireValue(/^[0-9a-f]{40}$/.test(options.product.sourceSha) && - (!toolchain.requested.sourceSha || options.product.sourceSha === toolchain.requested.sourceSha), "source build revision mismatch"); + if (options.product.kind === "source") requireValue(/^[0-9a-f]{40}$/.test(options.product.sourceSha), + "source build revision missing"); const platform = toolchain.requested.platform; requireValue(manifest.matrix.every(c => c.architecture === toolchain.architecture && c.os === platform.os), "matrix requires a different worker platform"); diff --git a/eng/regression-bisect/repro.test.cjs b/eng/regression-bisect/repro.test.cjs index 84c91e8f9be..8b4408de1fb 100644 --- a/eng/regression-bisect/repro.test.cjs +++ b/eng/regression-bisect/repro.test.cjs @@ -2,7 +2,7 @@ const test = require("node:test"); const assert = require("node:assert/strict"); const fs = require("node:fs/promises"); const path = require("node:path"); -const { fixture } = require("./fixtures/reproduction.cjs"); +const { fixture, projectBuild } = require("./fixtures/reproduction.cjs"); const { seal, archiveReceipt, policy } = require("./fixtures/contracts.cjs"); const { verifyToolchainIdentity } = require("./toolchains.cjs"); const { selectFsiCommand, verifyProjectIdentities, evaluateAssertion, runReproduction } = require("./repro.cjs"); @@ -264,6 +264,25 @@ async function runnerFixture(t, options = {}) { return f; } +test("source-derived sample SDK provenance does not replace product, host, runtime or Core checks", async t => { + for (const [name, mutate] of Object.entries({ + compatible: () => {}, + product: f => { f.product.sourceSha = "HEAD"; }, + host: f => { f.toolchain.hostSha256 = hash("different host"); }, + runtime: f => { f.toolchain.runtimes[0].sha256 = hash("different runtime"); }, + core: f => { f.loadedCore = { sha256: hash("different Core") }; }, + archive: f => fs.writeFile(f.toolchain.archive.retained.path, "different archive"), + })) await t.test(name, async t => { + const f = await runnerFixture(t); + Object.assign(f.toolchain.requested, { kind: "source", sourceSha: "b".repeat(40) }); + await mutate(f); + const result = await runReproduction(f, { execute: f.execute }); + assert.equal(result.outcome.outcome, name === "compatible" ? "pass" : "untestable", result.error); + if (name === "compatible") assert.equal(result.cells[0].attempts.length, 2); + else assert.equal(result.outcome.reason, "wrong-identity"); + }); +}); + async function projectRunnerFixture(t, { declaredTfms = [], libraryTfm = "net10.0", mutate, buildExitCode = 0, undeclared = false, nullable, defines = nullable ? ["NULLABLE"] : [] } = {}) { const f = await runnerFixture(t, { project: true }); @@ -297,47 +316,14 @@ async function projectRunnerFixture(t, { declaredTfms = [], libraryTfm = "net10. args.includes(`-p:TargetFramework=${c.tfm}`)); assert.ok(cell, "build must select a declared root cell"); const write = (location, contents) => f.file(path.relative(f.root, location), contents); - const packageDirectory = path.join(settings.env.NUGET_PACKAGES, "fsharp.core", f.product.corePackageVersion); - const core = await write(path.join(packageDirectory, "lib", "netstandard2.0", "FSharp.Core.dll"), - await fs.readFile(roles.core.path)); - await write(path.join(packageDirectory, "fsharp.core.10.0.100.nupkg"), await fs.readFile(roles.corePackage.path)); - f.inspectFiles = [{ ...roles.core, ...core }]; - f.loadedCore = core; const builds = [cell]; if (cell.project === app.project) builds.push(f.manifest.matrix.find(c => c.project === "Library.fsproj" && c.tfm === libraryTfm) ?? { ...cell, project: "Library.fsproj", tfm: libraryTfm }); - const rows = []; - for (const [index, current] of builds.entries()) { - const row = structuredClone(f.evidence[0]); - Object.assign(row, { - project: path.join(settings.cwd, current.project), targetFramework: current.tfm, - configuration: current.configuration, expectedLanguageVersion: current.languageVersion, - expectedOptimize: current.configuration === "Release", expectedDefines: defines, expectedNullable: nullable, - projectReferences: current.project === app.project ? ["Library.fsproj"] : [], - assetsFile: path.join(settings.cwd, ".runner", `${index}.assets.json`), - targetPath: path.join(settings.cwd, "bin", current.tfm, `${path.parse(current.project).name}.dll`), - fscTasks: row.fsc.map(task => ({ - ...task, references: [core.path], languageVersion: current.languageVersion, defines, nullable, - })), - }); - delete row.fsc; - delete row.core; - const identity = `FSharp.Core/${f.product.corePackageVersion}`; - await write(row.assetsFile, JSON.stringify({ - targets: { [current.tfm]: { [identity]: { - compile: { "lib/netstandard2.0/FSharp.Core.dll": {} }, runtime: { "lib/netstandard2.0/FSharp.Core.dll": {} }, - } } }, - libraries: { [identity]: { type: "package", path: `fsharp.core/${f.product.corePackageVersion}` } }, - packageFolders: { [settings.env.NUGET_PACKAGES]: {} }, - })); - await write(row.targetPath, "synthetic output"); - await write(path.join(settings.cwd, ".runner", `compiler-core-${index}.jsonl`), JSON.stringify({ - ...roles.core, assemblyName: "FSharp.Core", complete: true, entryAssembly: roles.compiler.path, - runtimeVersion: "10.0.0", runtimePath: path.join(f.toolchain.runtimes[0].path, "10.0.0"), architecture: "x64", - })); - rows.push(row); - } - structured = { cells: rows, diagnostics: [] }; + const result = await projectBuild({ product: f.product, toolchain: f.toolchain, settings, defines, nullable, + cells: builds.map(c => ({ ...c, projectReferences: c.project === app.project ? ["Library.fsproj"] : [] })) }); + f.inspectFiles = [{ ...roles.core, ...result.core }]; + f.loadedCore = result.core; + structured = result.structured; if (mutate) await mutate(structured, f); await write(args.find(arg => arg.startsWith("-bl:")).slice(4), "synthetic binlog"); return { ...success, exitCode: buildExitCode }; diff --git a/eng/regression-bisect/workspace.test.cjs b/eng/regression-bisect/workspace.test.cjs index 229406a17a2..7054da0d619 100644 --- a/eng/regression-bisect/workspace.test.cjs +++ b/eng/regression-bisect/workspace.test.cjs @@ -2,7 +2,7 @@ const test = require("node:test"); const assert = require("node:assert/strict"); const fs = require("node:fs/promises"); const path = require("node:path"); -const { fixture } = require("./fixtures/reproduction.cjs"); +const { fixture, projectBuild } = require("./fixtures/reproduction.cjs"); const { seal, archiveReceipt } = require("./fixtures/contracts.cjs"); const { createHash } = require("node:crypto"); const { fingerprintInput } = require("./model.cjs"); @@ -130,8 +130,24 @@ async function controlled(t, os = "windows") { { path: "lib/netstandard2.0/FSharp.Core.dll", sha256: hash(`core-${builds.at(-1).sha}`) }, ], ...f.packageIdentity } : {}) }); } + if (result.some(r => path.basename(r.path) === "fsc.dll")) { + const compiler = result.find(r => path.basename(r.path) === "fsc.dll"); + const root = path.dirname(compiler.path); + f.builtProduct = { + runtime: { name: "Microsoft.NETCore.App", version: `${activeSdk.split(".")[0]}.0.0` }, + corePackageVersion: "1.0.0", files: [ + ...result.map(r => ({ ...r, role: r.path === compiler.path ? "compiler" : + r.path.endsWith(".nupkg") ? "corePackage" : r.assemblyName === "FSharp.Core" ? "core" : + r.assemblyName === "FSharp.Build" ? "tasks" : "fsi" })), + ...Object.entries({ props: "Microsoft.FSharp.NetSdk.props", targets: "Microsoft.FSharp.Targets", + sdkTargets: "Microsoft.FSharp.NetSdk.targets", overrides: "Microsoft.FSharp.Overrides.NetSdk.targets" }) + .map(([role, name]) => ({ role, path: path.join(root, name) })), + ], + }; + } return ok(JSON.stringify(result)); } + if (args.includes("binlog")) return ok(JSON.stringify(f.projectEvidence.structured)); if (args[0] === "msbuild") return ok(JSON.stringify({ Properties: await properties(settings.cwd, args[1]) })); if (args[0] === "build" && path.basename(args[1]) === "Evidence.csproj") { f.helperArgs = args; @@ -148,6 +164,23 @@ async function controlled(t, os = "windows") { f.packs = (f.packs ?? 0) + 1; return ok(""); } + if (args[0] === "build" && args.includes("--no-incremental")) { + assert.equal(exe, f.toolchain.hostPath); + f.projectEvidence = await projectBuild({ product: f.builtProduct, toolchain: f.toolchain, + cells: f.manifest.matrix, settings }); + await fs.writeFile(args.find(a => a.startsWith("-bl:")).slice(4), "fixture binlog"); + return ok(""); + } + if (args[0] === "exec" && args.at(-1).endsWith("App.dll")) { + f.reproductions = (f.reproductions ?? 0) + 1; + const core = f.projectEvidence.core, runtime = args[2]; + await fs.writeFile(settings.env.REPRO_CORE_EVIDENCE, JSON.stringify({ + ...core, assemblyName: "FSharp.Core", assemblyVersion: "1.0.0.0", complete: true, + architecture: "x64", runtimeVersion: runtime, + runtimePath: path.join(f.toolchain.root, "shared", "Microsoft.NETCore.App", runtime), + })); + return { ...ok(""), ...f.manifest.assertions.pass }; + } if (args.includes("--exec")) { f.reproductions = (f.reproductions ?? 0) + 1; const fsi = args[3]; @@ -259,6 +292,51 @@ test("reserved source identity and a separate sample toolchain never retarget hi assert.match(duplicate.error.message, /EEXIST/); }); +test("compatible historical projects preserve the frozen sample runtime and independent SDK provenance", async t => { + for (const acquisition of ["released", "other-source"]) await t.test(acquisition, async t => { + const f = await controlled(t); + const project = await f.file("source/App.fsproj", ""); + f.manifest.kind = "project"; + f.manifest.sources = [{ path: "App.fsproj", sha256: project.sha256 }]; + Object.assign(f.manifest.matrix[0], { project: "App.fsproj", command: ["run-project", "App.fsproj"] }); + seal(f.manifest); + const runtime = await f.file("dotnet/shared/Microsoft.NETCore.App/9.0.0/System.Private.CoreLib.dll", "historical runtime"); + const receipt = archiveReceipt({ + name: "runtime.zip", url: "https://builds.dotnet.microsoft.com/dotnet/Runtime/9.0.0/dotnet-runtime-9.0.0-win-x64.zip", + rid: "win-x64", architecture: "x64", provenance: { kind: "publisher-checksum" }, + }, (await f.file("runtime.zip", "runtime archive")).path, "runtime archive"); + receipt.sha512 = receipt.actualSha512; + f.toolchain.installations.push({ product: "dotnet", version: "9.0.0", archive: receipt }); + f.toolchain.requested.runtimeRequirements.push({ name: "Microsoft.NETCore.App", version: "9.0.0", + role: "sample", archiveUrl: receipt.url }); + f.toolchain.runtimes.push({ name: "Microsoft.NETCore.App", version: "9.0.0", + path: path.dirname(path.dirname(runtime.path)), sha256: runtime.sha256 }); + if (acquisition === "other-source") Object.assign(f.toolchain.requested, { kind: "source", sourceSha: f.shas[0] }); + const frozen = fingerprintInput(f.toolchain); + const result = await testSourceRevision({ + ...f.options, reproductionToolchain: f.toolchain, + sampleRuntimes: { "net10.0": { name: "Microsoft.NETCore.App", version: "10.0.0" } }, + request: { ...f.options.request, sourceSha: f.shas[1], manifestDigest: fingerprintInput(f.manifest), + approvalDigest: f.manifest.approvedInputDigest }, + }, f.deps); + assert.equal(result.outcome.outcome, "pass", JSON.stringify(result)); + assert.equal(result.actualSourceSha, f.shas[1]); + assert.equal(result.toolchain.sdkVersion, "9.0.100"); + assert.equal(result.product.targetFramework, "net9.0"); + assert.equal(result.globalJson.before, result.globalJson.after); + assert.equal(result.reproduction.cells[0].attempts.length, 2); + for (const record of result.reproduction.cells[0].evidence) { + assert.equal(record.projects[0].sdkVersion, "10.0.100"); + assert.equal(record.projects[0].targetFramework, "net10.0"); + assert.equal(record.compilerRuntimeSelection.runtime.version, "9.0.0"); + assert.equal(record.compilerCoreLoads[0].runtimeVersion, "9.0.0"); + assert.equal(record.loadedCore.runtimeVersion, "10.0.0"); + assert.equal(record.loadedCore.sha256, record.compilerCoreLoads[0].sha256); + } + assert.equal(fingerprintInput(f.toolchain), frozen); + }); +}); + test("large product inventories hash every dependency without unbounded inspection arguments", async t => { const f = await controlled(t); f.afterBuild = async source => { From b105d2a688592cb024b0b0211eb8becd4eece156 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sun, 20 Sep 2026 04:46:23 +0200 Subject: [PATCH 29/99] Confirm regression source boundaries and guard evidence-based reports Require fresh charged candidate and parent runs, retain history and PR uncertainty, and prepare authenticated issue-specific publication intents without publishing results. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/regression-bisect/README.md | 216 ++++++++- eng/regression-bisect/attribution.cjs | 261 +++++++++++ eng/regression-bisect/attribution.test.cjs | 331 ++++++++++++++ eng/regression-bisect/engine.cjs | 5 +- eng/regression-bisect/model.cjs | 124 ++++- eng/regression-bisect/report.cjs | 327 +++++++++++++ eng/regression-bisect/report.test.cjs | 505 +++++++++++++++++++++ eng/regression-bisect/state.cjs | 74 ++- 8 files changed, 1832 insertions(+), 11 deletions(-) create mode 100644 eng/regression-bisect/attribution.cjs create mode 100644 eng/regression-bisect/attribution.test.cjs create mode 100644 eng/regression-bisect/report.cjs create mode 100644 eng/regression-bisect/report.test.cjs diff --git a/eng/regression-bisect/README.md b/eng/regression-bisect/README.md index c2c11e4ecd5..242938c1eef 100644 --- a/eng/regression-bisect/README.md +++ b/eng/regression-bisect/README.md @@ -4,7 +4,8 @@ This directory defines version 1 of the deterministic input, evidence, state, and admission contracts, complete issue intake, authenticated issue-comment checkpoints, and isolated exact toolchain acquisition. The endpoint-first engine composes those contracts with the existing reproduction and clean source-build runners. -It does not activate a workflow or attribute a culprit. +Fresh confirmation and deterministic publication guards consume the engine's +intent; none of these modules activates a workflow or publishes an issue report. GitHub access is injected by the trusted controller; tests use synthetic records and an in-memory transport, never live issue mutations. Records under `fixtures` are not reproduction approvals. @@ -362,6 +363,219 @@ pre-admission bisect reset. Repository files were not mutated for those probes. This validates the original product runners, not a real regression's admitted interval, an activated workflow, or fresh culprit attribution. +## Fresh boundary confirmation and association + +`attribution.cjs` exports `confirmBoundary({ intent, now }, dependencies)` and +`resolveCommitAssociations({ repository, sourceSha, pinnedMainSha }, dependencies)`. +All dependencies are trusted controller services, not worker-authored JSON. +The controller must hold the existing whole-issue lock throughout confirmation. + +Confirmation uses the engine's dependencies and its returned `confirmation` +intent, or a `blocked` intent retaining an interval for explicit parent/history +analysis. Generation, phase, interval and authenticated candidate failure must +match the current checkpoint. Endpoints are read from authenticated observations, +and admission is rechecked against actual Git ancestry. Caller-supplied parent +lists are checked against commit objects, never taken as ancestry evidence. +The optional `associationTransport` is the read-only GitHub `request` transport. +Without it, association evidence is `null` (not looked up), not a claim that no +PR exists. + +The engine exports its existing `currentInput`, `historyWorkspace`, +`authenticated` and `capacity` helpers for this composition. There is no second +build or assertion evaluator. Each candidate and relevant parent is separately +reserved, claimed and run through `executeInvestigationWork`, hence +`workspace.testSourceRevision` and `repro.runReproduction`. Whole-parent-set +capacity is checked before the first run. Every run consumes the existing +allowance; rejection, cancellation and changed inputs never refund it. +New build IDs must differ from every search observation and one another. +`validateObservation` receives the new reservation's `finalConfirmation`. +The reviewed producer's raw bytes are reread through `approval.readSource` +using the manifest source with its SHA-256. An absent producer must use the +runner's existing `no-producer`, `0.0.0`, empty-byte digest identity. + +Results are `{ status: "confirmed" | "unconfirmed" | "blocked", reason, +confirmation?, generation?, budget? }`. A successful or completed uncertain +analysis is saved through `state.recordConfirmation`; a rejected worker still +owns its charged pending reservation and cannot be silently retried. +`state.confirmation` is optional for older checkpoints, but strict when present: +it binds epoch, manifest digest, starting generation, candidate, candidate/search +observation digests, all parents with comparability/outcome references, remaining +interval, history, optional associations, and `latentRootCause: "unproven"`. +Authenticated checkpoint loading checks each fresh observation's charged +confirmation reservation and execution generation. + +Classifications deliberately differ: + +| Classification | Required evidence | +| --- | --- | +| `verified-introducing-source-boundary` | Independent deterministic candidate failure and sole comparable parent pass, unchanged sample SDK/runtime/environment and approved assertions. This is not a semantic root-cause claim. | +| `verified-triggering-change` | All comparable merge parents pass and integration fails, or source/build-input changes are coupled. The commit is a tested boundary; latent responsibility remains unknown. | +| `unconfirmed-candidate` | Root/missing/outside-lineage parent, insufficient allowance, skip/flake, identity/setup failure, changed result, ambiguous interval, or contradictory history. Remaining interval, parents, observations, skips and spent allowance remain visible. | + +History inspection is bounded to 128 evidence commits and 4,096 interval SHAs. +It retains actual parents, message SHA-256, changed paths, full revert/cherry-pick +references and a normalized patch-similarity SHA-1 (`patchId`, **not** a Git object +identity or proof of provenance). Referenced objects are fetched only from the +trusted local mirror. Reverts/reintroductions, cherry-picks/backports, repeated +patches and observed failure-then-descendant-pass evidence prevent unique +attribution. Missing related objects remain explicit uncertainty. Toolchain +changes retain each observation's requested/resolved build SDK and runtime; +global/build-input changes conservatively couple source and toolchain blame. + +Association resolution uses an isolated Git history and GET requests to +`commits/{commit_sha}/pulls`, `pulls/{pull_number}` and +`pulls/{pull_number}/commits`. It verifies exact repository/URL, merged status, +full merge/base/head SHAs, head/commit membership and tested-main ancestry. +Pagination is bounded to ten 100-entry pages and 100 associations; truncation, +rate limiting, duplicate or inconsistent responses return `incomplete`. +Unmerged/closed-only PRs are not introducing associations. A lone verified +main-lineage association is `unique`; multiple or servicing/off-lineage +associations remain `ambiguous`. `none` does not invalidate a verified source +boundary. Search results, titles and patch similarity never authorize a PR link. + +`model.validateConfirmation`, `validateConfirmationEvidence(checkpoint)` and +`validateAssociations` enforce these additional evidence shapes. They validate +structure and consistency, not the authenticity of arbitrary supplied data. + +## Deterministic reports and publication guards + +`report.cjs` exports these controller APIs: + +| Export | Contract | +| --- | --- | +| `validateReport(report)` | Strict bounded evidence schema; rejects arbitrary Markdown, extra fields, stale fingerprints, unsupported destinations and unsafe evidence references. | +| `renderReport(report)` | Deterministic escaped Markdown from validated evidence. Does not execute instructions, fetch URLs or publish. | +| `preparePublication({ now, generation }, dependencies)` | Under the issue lock, rechecks fresh approved inputs, complete discussion, eligibility, active investigations, current heads, authenticated checkpoint and result digest. Returns `ready`, `pending`, `published`, `blocked` or `requeue`. | +| `completePublication({ now, generation, digest, commentId }, dependencies)` | Repeats freshness checks and authenticates the actual existing result before recording publication completion. Does not create or edit the result comment. | + +Dependencies are `{ context, transport, withLock(action), +readCurrentContext({ now, repository, issueNumber }) }`. `context` is the full +existing state authentication context; it is not agent-supplied authorization. +The mandatory fresh trusted reader returns `manifest`, `authorization`, `policy`, +`policyRevision`, `workflowRevision`, `pinnedMainSha` (fresh main HEAD), +`stableSdk`, `environmentFingerprint`, and `readSource({ path, sha256 })`. +It must read approved assets and policy/workflow/main heads from trusted current +GitHub data, not echo a cached dispatch. The selected issue is fixed by the +authenticated context and manifest. There is no arbitrary destination parameter. + +Preparation uses the real `intake.readIssueSnapshot` and +`findActiveInvestigations`, including complete comment/timeline pagination and +raw approved-source checks. Only authenticated checkpoint/result records are +excluded from the discussion fingerprint. Marker copies, altered signed bodies, +different IDs, writers, apps or runs remain ordinary changed input. The current +checkpoint generation and publication digest are checked under the same lock. +Closed, unlabelled or too-old issues, corrections, approval/policy/workflow +changes, moving main, missing required evidence, competing investigations and +stale generations produce no publishable body. Spent allowance is preserved. +Only terminal `complete` or `blocked` checkpoints are reportable; an old +confirmation cannot be published while fresh confirmation is underway. + +The report schema is: + +```js +{ + schemaVersion: 1, + kind: "regression-investigation", + state, // validated state; publication digest/commentId normalized to null + observations: [{ + digest, data, // authenticated observation, including exact repeated assertions + evidence: [{ url, sha256, required, available }] + }], + deadline, inputDigest, reproductionDigest, policyDigest, checkedHead +} +``` + +This binds the issue/epoch/result digest without including mutable publication +bookkeeping. Rendering includes approval/input identities, checked versus tested +main, frozen source/producer/package hashes, approved operation descriptors and +matrix settings, expected assertions, exact observations and build/artifact +identities, outcomes/repetitions, evidence references, interval/skips/parents, +verified or ambiguous PR associations, causal uncertainty, allowance use and +resume state. The operation descriptors are the manifest's approved commands, +expanded by the existing runner, not reporter-authored shell strings. +Long display data is bounded and escaped; full evidence remains in authenticated +observation references and the signed structured payload. + +Observation artifacts may additionally retain `assemblyVersion`. Released +cells may retain `releasedToolchain: { sdk, compilerRuntime, archive }` using the +existing exact product-identity and archive-receipt schemas. The released receipt +must match the selected SDK hash, platform and official publisher checksum; +the source-only measured-preview exception cannot authorize a released archive. +Reports render both requested/resolved source toolchains and actual released +receipts. Measured official-HTTPS hashes are explicitly **measured**, never +publisher-provided checksums. Missing historical details remain visibly absent. +A verified boundary/trigger report cannot be published without the selected +stable endpoint's archive/runtime receipt and bundled compiler/Core assembly +versions; legacy incomplete evidence produces a blocked intent, not success. + +`state.recordPublication(transport, context, { generation, status, digest, +commentId })` updates the existing authenticated pending/published/blocked fields. +Preparation persists a deterministic pending digest before returning `ready`. +Repeated dispatch returns `pending`, not renewed write authorization. Complete +authenticated discussion scanning recovers a matching result after a successful +write whose response was lost. It never trusts a copied marker or caller Boolean. +`state.authenticateDecodedRecord(transport, context, comment)` returns the +authenticated `{ kind, payload, commentId, owner }` or `null`. + +The later trusted controller must re-run preparation as close as possible to +the write, hold the whole-issue lock, and publish only the fixed signed payload: + +```js +{ + kind: "result", + status: "Regression investigation result", + payload: { schemaVersion: 1, digest, report } +} +``` + +Use `state.serializeWorkflowRecord` with the **actual final comment ID** in its +authentication context. The returned display `body` alone is not an +authenticated result and must not be published as one. For a lost response, +recover a matching authenticated result or keep the pending intent blocked; +do not blindly create another comment. These modules never perform the actual +result write. + +Evidence transport is restricted to exact repository-bound GitHub artifact URLs: +`https://api.github.com/repos/dotnet/fsharp/actions/artifacts//zip` or +`https://github.com/dotnet/fsharp/actions/runs//artifacts/`. +Encoded segments, traversal, credentials, query strings, other origins and local +attachment paths are rejected **before** fetching. The injected +`readEvidence(url, { redirect: "error", maxBytes: 4194304 })` must enforce those +limits without following redirects. Missing required artifacts block; optional +expired artifacts are disclosed. Untrusted log text is display data, never a +new URL to fetch. + +### Confirmation/report validation + +The targeted attribution/report suite passes 110 tests; the complete scoped +suite passes 1,154. All changed scripts pass `node --check` and whitespace +validation. Real Git fixtures cover parent sets, merges, actual reverted and +cherry-picked patches, toolchain coupling and bounded association lookups. +Product execution is injected in those tests; their SHA/build-ID/call-count +checks are not a claim of a real regression's introducing commit. + +Fresh original-runner acceptance is retained in +`artifacts\regression-confirmation-trial`. The source worker clean-built +`5a8b9adb034f420a5c2c1f577670c5eeeb454b7e` in independent build +`14b25691-b71f-4712-8801-76088a20ed91`, with unchanged `global.json`, +approved measured archive provenance and two passing FSI assertions. +`fsharp-workspace-smoke-jNrofZ\result.json` has SHA-256 +`9bb0f436d8df53ab5e12e41bba26b115d032692aea27568ca6ea6a432d502b2b`. +Official metadata and retained released SDK `10.0.401` identities were rechecked. +The scoped evidence-helper Release build/test commands exited zero (the helper +contains no managed test cases). + +The original reproduction smoke then matched all 43 expectations: 23 passes, +six intended product failures and 14 identity rejections, using the released +SDK and fresh source products. Its complete evidence is +`Q:\rb912-confirmation-20260920\smoke-results.json`, SHA-256 +`1e3dec3266eb0e7a3bda888f9724d6ce02008823f664740ddcb33d7cc02d6499`. +Initial acceptance setup attempts rejected an incorrect startup-hook path and +inherited repository build imports; those failures remain in the local trial +evidence. The successful reproduction scratch is outside repository imports. +These are safe existing product fixtures, not issue authorization, live +publication, or verified attribution of an actual regression. + ## Exact toolchain acquisition `toolchains.cjs` exports four operations. All options are trusted-controller diff --git a/eng/regression-bisect/attribution.cjs b/eng/regression-bisect/attribution.cjs new file mode 100644 index 00000000000..9f177c9fc4d --- /dev/null +++ b/eng/regression-bisect/attribution.cjs @@ -0,0 +1,261 @@ +const { createHash } = require("node:crypto"); +const { + fingerprintInput: fingerprint, validateObservation, validateConfirmation, validateAssociations, admitBisection, +} = require("./model.cjs"); +const engine = require("./engine.cjs"); +const state = require("./state.cjs"); +const { execute } = require("./repro-process.cjs"); +const { requireValue } = require("./toolchain-io.cjs"); + +const equal = (a, b) => fingerprint(a) === fingerprint(b); +const sha = value => typeof value === "string" && /^[a-f0-9]{40}$/.test(value); +const hash = bytes => createHash("sha256").update(bytes).digest("hex"); +const boundedReason = error => String(error.message ?? error).slice(0, 1024); + +async function commitEvidence(history, sourceSha) { + const parents = (await history.git(["rev-list", "--parents", "-n", "1", sourceSha])).split(" ").slice(1); + const message = await history.git(["show", "-s", "--format=%B", sourceSha]); + const changedPaths = (await history.git(["diff-tree", "--root", "-m", "--no-commit-id", "--name-only", + "-r", "-z", sourceSha])).split("\0").filter(Boolean); + // This is only a patch-similarity fingerprint, never proof of provenance. + const patch = parents.length <= 1 ? await history.git(["show", "--format=", "--no-ext-diff", + "--no-textconv", "--no-renames", "--root", sourceSha]) : null; + const patchId = patch === null || patch === "" ? null : createHash("sha1").update(patch + .replace(/^index .*\n|^@@.*\n/gm, "").replace(/\s/g, "")).digest("hex"); + const references = regex => [...message.matchAll(regex)].map(m => m[1]); + return { + sha: sourceSha, parents, messageDigest: hash(message), changedPaths: [...new Set(changedPaths)].sort(), patchId, + reverts: references(/\bThis reverts commit ([a-f0-9]{40})\b/g), + cherryPicks: references(/\bcherry picked from commit ([a-f0-9]{40})\b/g), + suspiciousMessage: /\b(?:revert(?:ed|ing)?|reintroduc\w*|cherry[ -]pick\w*|backport\w*)\b/i.test(message), + }; +} + +async function resolveCommitAssociations({ repository, sourceSha, pinnedMainSha }, deps) { + requireValue(repository === "dotnet/fsharp" && sha(sourceSha) && sha(pinnedMainSha), "invalid association target"); + const result = { schemaVersion: 1, repository, sourceSha, status: "incomplete", reason: "association lookup incomplete", pulls: [] }; + let history; + try { + history = await engine.historyWorkspace(deps.history, deps.execute ?? execute, deps.signal); + requireValue(history.main === pinnedMainSha && await history.ancestor(sourceSha, pinnedMainSha), + "association source is not on tested main"); + const request = async (route, params) => { + const r = await deps.transport.request(`GET /repos/{owner}/{repo}/${route}`, { owner: "dotnet", repo: "fsharp", ...params }); + requireValue(r && (!r.status || r.status === 200) && r.data !== undefined && + Buffer.byteLength(JSON.stringify(r.data)) <= 4 * 1024 * 1024, "incomplete association API response"); + return r; + }; + const pages = async (route, params) => { + const rows = []; + for (let page = 1; page <= 10; page++) { + const r = await request(route, { ...params, per_page: 100, page }); + requireValue(Array.isArray(r.data) && r.data.length <= 100, "invalid association page"); + rows.push(...r.data); + requireValue(rows.length <= 1000, "association response bound exceeded"); + const next = r.headers?.link === undefined ? r.data.length === 100 : /;\s*rel="?next"?/.test(r.headers.link); + if (!next) return rows; + } + throw new Error("association pagination incomplete"); + }; + const rows = await pages("commits/{commit_sha}/pulls", { commit_sha: sourceSha }); + requireValue(rows.length <= 100 && new Set(rows.map(p => p.number)).size === rows.length, + "duplicate or oversized association set"); + for (const row of rows.sort((a, b) => a.number - b.number)) { + requireValue(Number.isSafeInteger(row.number) && row.number > 0, "invalid associated PR number"); + const { data: p } = await request("pulls/{pull_number}", { pull_number: row.number }); + requireValue(p.number === row.number && p.html_url === `https://github.com/${repository}/pull/${p.number}` && + p.base?.repo?.full_name === repository && ["open", "closed"].includes(p.state) && + typeof p.merged === "boolean", "foreign or incomplete PR association"); + if (!p.merged) { + requireValue(p.merged_at === null, "inconsistent merge status"); + continue; + } + requireValue(p.state === "closed" && Number.isFinite(Date.parse(p.merged_at)) && + sha(p.merge_commit_sha) && sha(p.base.sha) && sha(p.head?.sha) && + typeof p.base.ref === "string" && p.base.ref.length <= 256, "incomplete merged PR details"); + const commits = await pages("pulls/{pull_number}/commits", { pull_number: p.number }); + requireValue(commits.every(c => sha(c.sha)) && commits.some(c => c.sha === p.head.sha) && + typeof p.head.repo?.full_name === "string", "invalid PR head/commit evidence"); + const relation = p.merge_commit_sha === sourceSha ? "merge-commit" : "contained-commit"; + requireValue(relation === "merge-commit" || commits.some(c => c.sha === sourceSha), "PR has no exact source commit"); + const onLineage = await history.ancestor(p.merge_commit_sha, pinnedMainSha) && + await history.ancestor(sourceSha, p.merge_commit_sha) && + p.base.sha !== sourceSha && await history.ancestor(p.base.sha, sourceSha); + result.pulls.push({ number: p.number, url: p.html_url, mergeSha: p.merge_commit_sha, + baseSha: p.base.sha, headSha: p.head.sha, baseRef: p.base.ref, relation, onLineage }); + } + result.status = result.pulls.length === 0 ? "none" : + result.pulls.length === 1 && result.pulls[0].onLineage && result.pulls[0].baseRef === "main" ? "unique" : "ambiguous"; + result.reason = { + none: "No verified merged PR association; a source boundary does not require a PR.", + unique: "Exact commit association and Git ancestry establish one merged PR on tested main.", + ambiguous: "Multiple associations or servicing/off-lineage evidence; no unique introducing PR.", + }[result.status]; + } catch (error) { + result.status = "incomplete"; + result.reason = boundedReason(error); + } finally { if (history) await history.dispose(); } + validateAssociations(result); + return result; +} + +async function confirmBoundary({ intent, now }, deps) { + let cp, context, history, confirmation; + const clock = deps.clock ?? (() => new Date()); + const signal = AbortSignal.any([AbortSignal.timeout(300000), ...(deps.signal ? [deps.signal] : [])]); + try { + const input = await engine.currentInput(now, deps); + requireValue(input.context, input.reason); + context = input.context; + const loaded = await state.loadCheckpoint(deps.transport, context); + cp = loaded.checkpoint; + requireValue(loaded.executionAllowed && cp.pending === null, loaded.blockers.join("; ") || "pending worker"); + requireValue(intent?.requiresFreshConfirmation === true && ["confirmation", "blocked"].includes(intent.status) && + intent.generation === cp.generation && cp.state.phase === intent.status && + intent.interval && cp.state.interval && + equal(cp.state.interval, (({ remaining, ...i }) => i)(intent.interval)), "stale or unauthenticated confirmation intent"); + const candidate = intent.candidate ?? cp.state.interval.badSha; + requireValue(candidate === cp.state.interval.badSha, "candidate does not match admitted interval"); + confirmation = { + schemaVersion: 1, epoch: cp.state.epoch, manifestDigest: cp.state.approvalFingerprint, + startGeneration: cp.generation, candidate, classification: "unconfirmed-candidate", reason: "confirmation not completed", + interval: { goodSha: cp.state.interval.goodSha, badSha: candidate, remaining: [...intent.interval.remaining] }, + parents: [], candidateObservationDigest: null, searchObservationDigest: null, + history: { commits: [], monotonicity: "not-contradicted", toolchainCoupled: false }, + associations: null, latentRootCause: "unproven", + }; + history = await engine.historyWorkspace(deps.history, deps.execute ?? execute, signal); + requireValue(history.main === cp.state.pinnedMainSha, "pinned main changed"); + const candidateInfo = await commitEvidence(history, candidate); + confirmation.parents = candidateInfo.parents.map(sourceSha => ({ sourceSha, comparable: false, observationDigest: null })); + requireValue(intent.parents === undefined || equal(intent.parents, candidateInfo.parents), "claimed parents differ from Git objects"); + requireValue(candidateInfo.parents.length > 0, "root commit has no parent boundary"); + const remaining = (await history.git(["rev-list", "--topo-order", "--max-count=4097", candidate, + `^${cp.state.interval.goodSha}`])).split("\n").filter(Boolean); + confirmation.interval.remaining = remaining; + requireValue(remaining.length <= 4096 && equal(remaining, intent.interval.remaining), "candidate interval differs from Git ancestry"); + for (const parent of confirmation.parents) parent.comparable = + await history.ancestor(cp.state.interval.goodSha, parent.sourceSha) && await history.ancestor(parent.sourceSha, history.main); + requireValue(confirmation.parents.every(p => p.comparable), "parent outside verified comparable lineage"); + const observations = Object.entries(cp.observations).filter(([, r]) => r.valid).map(([id]) => + [id, engine.authenticated(cp, id, deps.environments)]); + const source = (target, outcome) => observations.find(([, o]) => o.kind === "source" && + o.target.sourceSha === target && o.outcome.outcome === outcome); + const search = source(candidate, "product-failure"); + requireValue(search, "candidate lacks authenticated product failure"); + confirmation.searchObservationDigest = search[0]; + const good = source(cp.state.interval.goodSha, "pass"); + requireValue(good, "interval lacks authenticated passing source"); + const admission = admitBisection({ + manifest: cp.state.manifest, stable: engine.authenticated(cp, cp.state.endpointEvidence.stable, deps.environments), + main: engine.authenticated(cp, cp.state.endpointEvidence.main, deps.environments), good: good[1], + ancestry: { goodSha: good[1].target.sourceSha, badSha: history.main, pinnedMainSha: history.main, + isAncestor: await history.ancestor(good[1].target.sourceSha, history.main) }, + budget: { ...cp.state.budget, nextWorkItemId: "none" }, + }); + requireValue(admission.decision === "admit", admission.reason); + const shas = [...new Set([...remaining, ...candidateInfo.parents])]; + requireValue(shas.length <= 128, "history evidence bound exceeded; parent uncertainty"); + for (const revision of shas) { + const { suspiciousMessage, ...evidence } = revision === candidate ? candidateInfo : await commitEvidence(history, revision); + confirmation.history.commits.push(evidence); + if (suspiciousMessage || evidence.reverts.length || evidence.cherryPicks.length) + confirmation.history.monotonicity = "uncertain"; + for (const referenced of [...evidence.reverts, ...evidence.cherryPicks]) { + if (shas.includes(referenced)) continue; + requireValue(shas.length < 128, "related patch history bound exceeded"); + await history.git(["fetch", "--quiet", "--no-tags", "--no-recurse-submodules", + deps.history.mirrorDirectory, referenced]); + shas.push(referenced); + } + } + const patches = confirmation.history.commits.map(c => c.patchId).filter(Boolean); + if (new Set(patches).size !== patches.length) confirmation.history.monotonicity = "uncertain"; + for (const [, bad] of observations.filter(([, o]) => o.kind === "source" && o.outcome.outcome === "product-failure")) + for (const [, pass] of observations.filter(([, o]) => o.kind === "source" && o.outcome.outcome === "pass")) + if (await history.ancestor(bad.target.sourceSha, pass.target.sourceSha)) + confirmation.history.monotonicity = "contradicted"; + requireValue(confirmation.history.monotonicity === "not-contradicted", + "revert, reintroduction, cherry-pick or nonmonotonic evidence prevents unique attribution"); + requireValue(remaining.length === 1 || candidateInfo.parents.length > 1, + "unconfirmed interval contains other possible source commits"); + const producer = cp.state.manifest.producer; + if (producer.name !== "no-producer") { + const source = cp.state.manifest.sources.find(s => s.sha256 === producer.sha256); + requireValue(source, "frozen producer must be a reviewed source asset"); + const bytes = await deps.approval.readSource(structuredClone(source)); + requireValue(bytes instanceof Uint8Array && hash(bytes) === producer.sha256, "frozen producer hash mismatch"); + } else requireValue(producer.version === "0.0.0" && producer.sha256 === hash(""), "invalid absent producer identity"); + const targets = [candidate, ...candidateInfo.parents]; + requireValue(engine.capacity(cp, targets.length, now, targets), "budget cannot afford independent candidate and all parents"); + const initialBuilds = new Set(observations.map(([, o]) => o.buildInstanceId)); + for (const sourceSha of targets) { + const current = await engine.currentInput(clock().toISOString(), deps); + requireValue(current.context, current.reason); + context = current.context; + const fresh = await state.loadCheckpoint(deps.transport, context); + requireValue(fresh.executionAllowed && fresh.checkpoint.generation === cp.generation && + fresh.checkpoint.state.epoch === confirmation.epoch, "confirmation checkpoint changed"); + const reservation = await state.reserveWork(deps.transport, context, { + generation: cp.generation, attemptId: fingerprint({ epoch: confirmation.epoch, + startGeneration: confirmation.startGeneration, sourceSha, owner: context.owner }).slice(0, 32), + request: { kind: "source", target: { sourceSha }, endpoint: null, phase: "confirmation" }, + }); + cp = await engine.executeInvestigationWork({ status: "work", reservation, binding: { + issue: context.issue, issueFingerprint: context.issueFingerprint, + pinnedMainSha: context.pinnedMainSha, environmentFingerprint: context.environmentFingerprint, + } }, deps); + requireValue(cp.observations, cp.reason ?? "confirmation worker did not commit"); + const record = cp.records.find(r => r.id === reservation.id); + const o = engine.authenticated(cp, record.observationDigest, deps.environments); + validateObservation(o, cp.state.manifest, { finalConfirmation: { runId: reservation.owner.runId, buildInstanceId: reservation.id } }); + requireValue(!initialBuilds.has(o.buildInstanceId) && o.target.sourceSha === sourceSha, "reused build or wrong source"); + initialBuilds.add(o.buildInstanceId); + requireValue(o.cells.every(c => c.outcome.outcome === "untestable" || c.buildToolchain) && + o.cells.every(c => { + const previous = search[1].cells.find(p => p.id === c.id); + return equal(c.environment, previous.environment) && equal(c.sdk, previous.sdk) && equal(c.runtime, previous.runtime); + }), "incomparable sample environment or missing revision toolchain"); + if (sourceSha === candidate) confirmation.candidateObservationDigest = record.observationDigest; + else confirmation.parents.find(p => p.sourceSha === sourceSha).observationDigest = record.observationDigest; + } + const candidateObservation = cp.observations[confirmation.candidateObservationDigest].data; + requireValue(candidateObservation.outcome.outcome === "product-failure", "candidate no longer deterministically fails"); + requireValue(confirmation.parents.every(p => cp.observations[p.observationDigest].data.outcome.outcome === "pass"), + "parent failure, skip or flake leaves the boundary unconfirmed"); + confirmation.history.toolchainCoupled = candidateInfo.changedPaths.some(p => + /(?:^|\/)(?:global\.json|NuGet\.config|build\.(?:cmd|sh|ps1))$|\.(?:runtimeconfig\.json|props|targets|[fc]sproj|proj)$|^eng\//i.test(p)) || + confirmation.parents.some(p => cp.observations[p.observationDigest].data.cells.some(c => { + const current = candidateObservation.cells.find(n => n.id === c.id).buildToolchain; + return !equal(current.sdk, c.buildToolchain.sdk) || !equal(current.compilerRuntime, c.buildToolchain.compilerRuntime); + })); + confirmation.classification = candidateInfo.parents.length > 1 || confirmation.history.toolchainCoupled ? + "verified-triggering-change" : "verified-introducing-source-boundary"; + confirmation.reason = candidateInfo.parents.length > 1 ? "Fresh merge failure and all comparable parents pass: integration trigger only." : + confirmation.history.toolchainCoupled ? "Fresh source boundary; source and build toolchain responsibility remains coupled/unknown." : + "Fresh independent candidate failure and parent pass establish the introducing source boundary, not a semantic root cause."; + if (deps.associationTransport) { + confirmation.associations = await resolveCommitAssociations({ repository: context.repository, + sourceSha: candidate, pinnedMainSha: context.pinnedMainSha }, + { transport: deps.associationTransport, history: deps.history, execute: deps.execute, signal: deps.signal }); + } + validateConfirmation(confirmation); + cp = await state.recordConfirmation(deps.transport, context, { generation: cp.generation, confirmation }); + return { status: "confirmed", reason: confirmation.reason, confirmation, generation: cp.generation, budget: cp.state.budget.consumed }; + } catch (error) { + const reason = boundedReason(error); + if (confirmation) { + confirmation.classification = "unconfirmed-candidate"; + confirmation.reason = reason; + validateConfirmation(confirmation); + const loaded = await state.loadCheckpoint(deps.transport, context); + cp = loaded.checkpoint; + if (loaded.executionAllowed && cp.pending === null && cp.state.epoch === confirmation.epoch) + cp = await state.recordConfirmation(deps.transport, context, { generation: cp.generation, confirmation }); + } + return { status: confirmation ? "unconfirmed" : "blocked", reason, + ...(confirmation ? { confirmation } : {}), ...(cp ? { generation: cp.generation, budget: cp.state.budget.consumed } : {}) }; + } finally { if (history) await history.dispose(); } +} + +module.exports = { confirmBoundary, resolveCommitAssociations }; diff --git a/eng/regression-bisect/attribution.test.cjs b/eng/regression-bisect/attribution.test.cjs new file mode 100644 index 00000000000..8db6a20ff67 --- /dev/null +++ b/eng/regression-bisect/attribution.test.cjs @@ -0,0 +1,331 @@ +const { test } = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs/promises"); +const path = require("node:path"); +const os = require("node:os"); +const { execFileSync, spawnSync } = require("node:child_process"); +const f = require("./fixtures/contracts.cjs"); +const { harness } = require("./fixtures/checkpoint.cjs"); +const model = require("./model.cjs"); +const state = require("./state.cjs"); +const { advanceInvestigation } = require("./engine.cjs"); +const { confirmBoundary, resolveCommitAssociations } = require("./attribution.cjs"); + +const gitExecutable = execFileSync(process.platform === "win32" ? "where.exe" : "which", + [process.platform === "win32" ? "git.exe" : "git"], { encoding: "utf8" }).trim().split(/\r?\n/)[0]; + +async function fixture(t, kind = "linear") { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "attribution-test-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const mirror = path.join(root, "mirror"), scratch = path.join(root, "scratch"); + await fs.mkdir(mirror); await fs.mkdir(scratch); + await fs.writeFile(path.join(root, "config"), ""); + const git = (...args) => execFileSync(gitExecutable, + ["-c", "core.hooksPath=", "-c", "commit.gpgSign=false", ...args], { + cwd: mirror, encoding: "utf8", env: { ...process.env, GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: path.join(root, "config"), GIT_AUTHOR_NAME: "Fixture", + GIT_AUTHOR_EMAIL: "fixture@example.invalid", GIT_COMMITTER_NAME: "Fixture", + GIT_COMMITTER_EMAIL: "fixture@example.invalid" }, + }).trim(); + git("init", "--quiet", "--initial-branch=main"); + async function commit(name, file = "source.fs", content = name) { + await fs.writeFile(path.join(mirror, file), content); + git("add", "--", file); git("commit", "--quiet", "-m", name); + return git("rev-parse", "HEAD"); + } + const good = await commit("base"); + let related; + if (kind === "merge" || kind === "foreign-parent") { + git("checkout", "--quiet", "-b", "side"); + related = await commit("side", "side.fs"); + git("checkout", "--quiet", "main"); + await commit("main-side", "main.fs"); + git("merge", "--quiet", "--no-ff", "side", "-m", "merge integration"); + } else if (kind === "backport") { + git("checkout", "--quiet", "-b", "servicing"); + related = await commit("upstream change"); + git("checkout", "--quiet", "main"); + await commit("main-side", "main.fs"); + git("cherry-pick", "--quiet", "-x", related); + } else if (kind === "revert") { + related = await commit("change"); + git("revert", "--no-edit", related); + git("revert", "--no-edit", git("rev-parse", "HEAD")); + } else if (kind !== "root") { + await commit("regression", kind === "toolchain" ? "global.json" : "source.fs"); + } + const candidate = git("rev-parse", "HEAD"); + const parents = git("rev-list", "--parents", "-n", "1", candidate).split(" ").slice(1); + const h = harness(); + h.context.pinnedMainSha = candidate; + h.context.manifest.searchHints.sourceShas = [good]; + h.context.manifest.sources.push({ path: "fixtures/producer.dll", sha256: f.digest("producer") }); + f.seal(h.context.manifest); + Object.assign(h.context.authorization, { manifestDigest: model.fingerprintInput(h.context.manifest), + inputDigest: h.context.manifest.approvedInputDigest }); + const environments = Object.fromEntries(h.context.manifest.matrix.map(c => [c.id, + { os: c.os, architecture: c.architecture, imageDigest: f.digest("runner image") }])); + h.context.environmentFingerprint = model.fingerprintInput(environments); + await h.start(); + const read = async () => (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + function observation(sourceSha, id, outcome = "product-failure", kind = "source") { + const o = f.observation(kind, outcome, h.context.manifest, sourceSha); + o.buildInstanceId = id; + for (const c of o.cells) { + for (const a of Object.values(c.artifacts)) a.buildInstanceId = id; + if (kind === "source") c.buildToolchain = f.buildToolchain(sourceSha, c.environment.os); + } + return o; + } + for (const [endpoint, outcome, sha] of [["stable", "product-failure", null], + ["main", "product-failure", candidate], ["good", "pass", good]]) { + const kind = endpoint === "stable" ? "released" : "source"; + const r = await state.reserveWork(h.transport, h.context, { generation: (await read()).generation, + attemptId: `search-${endpoint}`, request: { kind, + target: kind === "source" ? { sourceSha: sha } : { sdk: h.context.stableSdk }, + endpoint, phase: endpoint === "good" ? "good-search" : endpoint } }); + await state.commitObservation(h.transport, h.context, { generation: r.generation, + attemptId: r.id, observation: observation(sha, r.id, outcome, kind), evidence: [] }); + } + const calls = []; + const deps = { + transport: h.transport, context: h.context, environments, + execute: async (exe, args, options) => { + const r = spawnSync(exe, args, { ...options, encoding: "utf8", timeout: options.timeoutMs, maxBuffer: options.maxBytes }); + if (r.error) throw r.error; + return { status: "completed", complete: !r.signal, exitCode: r.status, stdout: r.stdout, stderr: r.stderr }; + }, + history: { gitExecutable, mirrorDirectory: mirror, scratchDirectory: scratch, mainRef: "refs/heads/main" }, + intake: { readIssueSnapshot: async () => ({ status: "complete", snapshot: { + discussionFingerprint: h.context.issueFingerprint, + input: { discussion: { issue: h.context.issue }, reproduction: { manifest: h.context.manifest } }, + investigations: { status: "complete", active: [] }, + } }) }, + approval: { readSource: async source => Buffer.from(source.path.endsWith(".dll") ? "producer" : 'printfn "ok"'), + authenticateRecord: async () => false, readTrustedCheckpoints: async () => [] }, + toolchains: { resolveLatestStableSdk: async () => ({ status: "resolved", toolchain: { + sdkVersion: h.context.stableSdk.version, + metadata: [{ url: h.context.stableSdk.releaseIndexUrl, sha256: h.context.stableSdk.releaseIndexDigest }], + } }) }, + platform: { os: "linux", architecture: "x64", rid: "linux-x64" }, + clock: () => new Date(h.context.now), worker: { reproductionToolchain: { frozen: true } }, + testSourceRevision: async options => { + calls.push(options); + const cp = await read(); + assert.equal(cp.pending.id, options.buildInstanceId); + assert.ok(cp.pending.executionGeneration); + const o = observation(options.request.sourceSha, options.buildInstanceId, + options.request.sourceSha === candidate ? "product-failure" : "pass"); + return { status: "tested", actualSourceSha: options.request.sourceSha, buildInstanceId: options.buildInstanceId, + outcome: o.outcome, reproduction: { cells: o.cells }, observation: o }; + }, + collectObservation: async ({ result }) => ({ observation: result.observation, evidence: [] }), + }; + let intent = await advanceInvestigation({ now: h.context.now }, deps); + if (kind === "root") { + const cp = await state.recordSearchProgress(h.transport, h.context, { generation: (await read()).generation, + phase: "blocked", interval: { goodSha: good, badSha: candidate, gitLog: [] } }); + intent = { status: "blocked", requiresFreshConfirmation: true, generation: cp.generation, + interval: { ...cp.state.interval, remaining: [candidate] } }; + } + return Object.assign(h, { deps, intent, calls, candidate, good, parents, related, read, git, observation, + confirm: () => confirmBoundary({ intent, now: h.context.now }, deps) }); +} + +test("fresh independent candidate and parent, not search evidence, prove a linear boundary", async t => { + const h = await fixture(t); + assert.equal(h.intent.status, "confirmation"); + const before = await h.read(); + const result = await h.confirm(); + assert.equal(result.status, "confirmed", JSON.stringify(result)); + assert.equal(result.confirmation.classification, "verified-introducing-source-boundary"); + assert.deepEqual(h.calls.map(c => c.request.sourceSha), [h.candidate, h.good]); + assert.equal(new Set(h.calls.map(c => c.buildInstanceId)).size, 2); + assert.ok(h.calls.every(c => c.request.phase === "confirmation" && + model.fingerprintInput(c.manifest) === model.fingerprintInput(h.context.manifest))); + const cp = await h.read(); + assert.deepEqual(cp.state.confirmation, result.confirmation); + assert.equal(cp.state.budget.consumed.workerUnits, before.state.budget.consumed.workerUnits + 2); + for (const d of [result.confirmation.candidateObservationDigest, ...result.confirmation.parents.map(p => p.observationDigest)]) { + assert.ok(!before.observations[d]); + const o = cp.observations[d].data; + assert.equal(o.cached, false); + assert.ok(o.cells.every(c => c.attempts.length === 2 && c.buildToolchain.sourceSha === o.target.sourceSha)); + } + assert.equal(result.confirmation.latentRootCause, "unproven"); + assert.deepEqual(await fs.readdir(h.deps.history.scratchDirectory), []); + for (const mutate of [ + c => { c.markdown = "arbitrary claim"; }, + c => { c.latentRootCause = h.candidate; }, + c => { c.candidateObservationDigest = c.searchObservationDigest; }, + c => { c.parents = []; }, + c => { c.epoch = f.digest("old epoch"); }, + c => { c.startGeneration = cp.generation; }, + ]) { + const forged = structuredClone(cp); + mutate(forged.state.confirmation); + assert.throws(() => model.validateConfirmationEvidence(forged)); + } +}); + +test("fresh failures and identity contradictions never yield unique blame", async t => { + const variants = ["candidate-pass", "parent-fail", "flake", "skip", "cached", "reused-build", + "wrong-sha", "changed-matrix", "changed-producer", "changed-environment", "changed-sdk", + "missing-toolchain", "original-attempts"]; + for (const variant of variants) await t.test(variant, async t => { + const h = await fixture(t); + const run = h.deps.testSourceRevision; + h.deps.testSourceRevision = async options => { + const result = await run(options), o = result.observation; + if (variant === "candidate-pass" || variant === "parent-fail" || variant === "flake" || variant === "skip") { + const attempts = variant === "flake" ? [f.attempt("pass"), f.attempt("product-failure")] : + [f.attempt(variant === "candidate-pass" ? "pass" : variant === "skip" ? "untestable" : "product-failure"), + f.attempt(variant === "candidate-pass" ? "pass" : variant === "skip" ? "untestable" : "product-failure")]; + for (const c of o.cells) { c.attempts = attempts; c.outcome = model.classifyAttempts(attempts, h.context.manifest); } + result.outcome = o.outcome = o.cells[0].outcome; + } + if (variant === "cached") o.cached = true; + if (variant === "reused-build") result.buildInstanceId = o.buildInstanceId = "search-main"; + if (variant === "wrong-sha") result.actualSourceSha = o.target.sourceSha = h.good; + if (variant === "changed-matrix") o.matrixFingerprint = f.digest("other matrix"); + if (variant === "changed-producer") o.cells[0].producer.sha256 = f.digest("other bytes"); + if (variant === "changed-environment") o.cells[0].environment.imageDigest = f.digest("other image"); + if (variant === "changed-sdk") o.cells[0].sdk.sha256 = f.digest("other SDK"); + if (variant === "missing-toolchain") delete o.cells[0].buildToolchain; + if (variant === "original-attempts") result.reproduction = { cells: [{ id: o.cells[0].id, attempts: [f.attempt("pass")] }] }; + return result; + }; + const result = await h.confirm(); + assert.notEqual(result.status, "confirmed", JSON.stringify(result)); + assert.notEqual(result.confirmation?.classification, "verified-introducing-source-boundary"); + assert.ok((await h.read()).state.budget.consumed.workerUnits > 3); + }); +}); + +test("untrusted intent, generation, assets and insufficient allowance cannot execute", async t => { + for (const variant of ["generation", "candidate", "parents", "producer", "budget", "deadline", "spent", "approval"]) + await t.test(variant, async t => { + const h = await fixture(t); + if (variant === "generation") h.intent.generation--; + if (variant === "candidate") h.intent.candidate = h.good; + if (variant === "parents") h.intent.parents = []; + if (variant === "producer") h.deps.approval.readSource = async () => Buffer.from("different producer"); + if (variant === "approval") h.context.authorization.inputDigest = f.digest("forged"); + if (variant === "spent") { + for (const sourceSha of [h.candidate, h.good]) { + const r = await state.reserveWork(h.transport, h.context, { generation: (await h.read()).generation, + attemptId: `spent-${sourceSha}`, request: { kind: "source", target: { sourceSha }, + endpoint: null, phase: "confirmation" } }); + await state.commitObservation(h.transport, h.context, { generation: r.generation, attemptId: r.id, + observation: h.observation(sourceSha, r.id, sourceSha === h.good ? "pass" : "product-failure"), evidence: [] }); + } + h.intent.generation = (await h.read()).generation; + } + if (variant === "budget" || variant === "deadline") { + const cp = await h.read(); + cp.deadline = new Date(Date.parse(h.context.now) + (variant === "budget" ? 110 : 220) * 60000 - 1).toISOString(); + h.checkpoint(cp); + h.intent.generation = cp.generation; + } + const result = await h.confirm(); + assert.notEqual(result.status, "confirmed", JSON.stringify(result)); + assert.equal(h.calls.length, 0); + }); +}); + +test("real merge, root, backport, revert and toolchain boundaries retain uncertainty", async t => { + for (const kind of ["merge", "root", "backport", "revert", "toolchain"]) + await t.test(kind, async t => { + const h = await fixture(t, kind); + const result = await h.confirm(); + assert.notEqual(result.confirmation?.classification, "verified-introducing-source-boundary", JSON.stringify(result)); + if (kind === "merge" || kind === "toolchain") { + assert.equal(result.status, "confirmed", JSON.stringify(result)); + assert.equal(result.confirmation.classification, "verified-triggering-change"); + assert.deepEqual(h.calls.map(c => c.request.sourceSha), [h.candidate, ...h.parents]); + if (kind === "toolchain") assert.equal(result.confirmation.history.toolchainCoupled, true); + } else { + assert.notEqual(result.status, "confirmed"); + assert.ok(result.reason); + if (kind !== "root") { + const commits = result.confirmation.history.commits; + assert.ok(commits.some(c => c.sha === h.related)); + assert.equal(commits.find(c => c.sha === h.related).patchId, + commits.find(c => c.sha === h.candidate).patchId); + } + } + }); +}); + +test("merge parent failure and parent confirmation budget preserve the entire parent set", async t => { + for (const variant of ["failure", "budget", "outside"]) await t.test(variant, async t => { + const h = await fixture(t, "merge"); + if (variant === "failure") { + const run = h.deps.testSourceRevision; + h.deps.testSourceRevision = async options => { + const r = await run(options); + for (const c of r.observation.cells) { + c.attempts = [f.attempt("product-failure"), f.attempt("product-failure")]; + c.outcome = model.classifyAttempts(c.attempts, h.context.manifest); + } + r.outcome = r.observation.outcome = r.observation.cells[0].outcome; + return r; + }; + } else { + const cp = await h.read(); + if (variant === "budget") cp.deadline = new Date(Date.parse(h.context.now) + 220 * 60000).toISOString(); + else cp.state.interval.goodSha = h.parents[0]; + h.checkpoint(cp); + h.intent.generation = cp.generation; + h.intent.interval.goodSha = cp.state.interval.goodSha; + } + const r = await h.confirm(); + assert.notEqual(r.status, "confirmed", JSON.stringify(r)); + assert.equal(r.confirmation.parents.length, 2); + if (variant !== "failure") assert.equal(h.calls.length, 0); + }); +}); + +function associationTransport(h, variant) { + const pull = n => ({ number: n, html_url: `https://github.com/dotnet/fsharp/pull/${n}`, + state: "closed", merged: true, merged_at: "2026-09-01T00:00:00Z", merge_commit_sha: h.candidate, + base: { sha: h.good, ref: "main", repo: { full_name: "dotnet/fsharp" } }, + head: { sha: h.candidate, repo: { full_name: "dotnet/fsharp" } } }); + const requests = []; + return { requests, async request(route, args) { + requests.push([route, args]); + assert.equal(args.owner, "dotnet"); assert.equal(args.repo, "fsharp"); + if (variant === "rate-limited") return { status: 403, data: {} }; + if (route.endsWith("/commits/{commit_sha}/pulls")) { + assert.equal(args.commit_sha, h.candidate); + return { data: variant === "none" ? [] : variant === "several" ? [pull(1), pull(2)] : [pull(1)], + headers: { link: variant === "incomplete" ? '; rel="next"' : "" } }; + } + if (route.endsWith("/commits")) return { data: [{ sha: h.candidate }], headers: { link: "" } }; + const p = pull(args.pull_number); + if (variant === "closed") { p.merged = false; p.merged_at = null; } + if (variant === "foreign") p.base.repo.full_name = "other/repo"; + if (variant === "mismatch") p.merge_commit_sha = f.sha("missing"); + if (variant === "servicing") p.base.ref = "release/dev17.0"; + if (variant === "unsafe-url") p.html_url = "https://evil.invalid/pull/1"; + return { data: p }; + } }; +} + +test("commit association mapping never guesses a PR from incomplete or foreign evidence", async t => { + for (const [variant, expected] of [["none", "none"], ["one", "unique"], ["several", "ambiguous"], + ["closed", "none"], ["rate-limited", "incomplete"], ["incomplete", "incomplete"], + ["foreign", "incomplete"], ["mismatch", "incomplete"], ["servicing", "ambiguous"], ["unsafe-url", "incomplete"]]) + await t.test(variant, async t => { + const h = await fixture(t); + const transport = associationTransport(h, variant); + const r = await resolveCommitAssociations({ repository: "dotnet/fsharp", sourceSha: h.candidate, + pinnedMainSha: h.candidate }, { transport, history: h.deps.history, execute: h.deps.execute }); + assert.equal(r.status, expected, JSON.stringify(r)); + assert.ok(r.reason); + assert.equal(r.sourceSha, h.candidate); + model.validateAssociations(r); + if (variant === "several") assert.equal(r.pulls.length, 2); + assert.ok(transport.requests.every(([r]) => r.startsWith("GET "))); + }); +}); diff --git a/eng/regression-bisect/engine.cjs b/eng/regression-bisect/engine.cjs index 9d3227729be..6cf8b84044c 100644 --- a/eng/regression-bisect/engine.cjs +++ b/eng/regression-bisect/engine.cjs @@ -370,4 +370,7 @@ async function executeInvestigationWork(work, deps) { }); } -module.exports = { advanceInvestigation, executeInvestigationWork }; +module.exports = { + advanceInvestigation, executeInvestigationWork, + historyWorkspace, currentInput, authenticated, capacity, +}; diff --git a/eng/regression-bisect/model.cjs b/eng/regression-bisect/model.cjs index aa6e890e26a..446b310da9d 100644 --- a/eng/regression-bisect/model.cjs +++ b/eng/regression-bisect/model.cjs @@ -228,10 +228,11 @@ function stableSdk(value, path) { } const productIdentity = shape({ version: pinnedVersion, sha256: digest, verified: literal(true) }); -const artifact = shape({ +const artifact = (v, p) => object(v, { sha256: digest, origin: oneOf(["local-build", "released-sdk"]), sourceSha: nullable(sourceSha), buildInstanceId: text, -}); + ...(v && Object.hasOwn(v, "assemblyVersion") ? { assemblyVersion: pinnedVersion } : {}), +}, p); const archiveReceipt = (v, p) => object(v, { name: text, url: httpsUrl, rid: text, architecture, provenance: (v, p) => object(v, v?.kind === "publisher-checksum" @@ -256,6 +257,9 @@ const observationCell = (v, p) => object(v, { artifacts: shape({ compiler: nullable(artifact), tasks: nullable(artifact), targets: nullable(artifact), core: nullable(artifact) }), producer: identity, attempts: list(attemptSchema, 1, trustedPolicy.limits.assertionRepetitions), outcome: validateOutcome, ...(v && Object.hasOwn(v, "buildToolchain") ? { buildToolchain } : {}), + ...(v && Object.hasOwn(v, "releasedToolchain") ? { + releasedToolchain: shape({ sdk: productIdentity, compilerRuntime: productIdentity, archive: archiveReceipt }), + } : {}), }, p); function validateObservation(observation, manifest, options = { finalConfirmation: null }) { @@ -323,6 +327,24 @@ function validateObservation(observation, manifest, options = { finalConfirmatio check(cell.sdk.version === observation.target.sdk.version && cell.sdk.sha256 === archive?.sha256, "observation.cells.sdk", "released SDK identity mismatch"); } + if (cell.releasedToolchain) { + const released = cell.releasedToolchain; + check(observation.kind === "released" && equal(released.sdk, cell.sdk) && + released.archive.sha256 === cell.sdk.sha256 && + released.archive.architecture === cell.environment.architecture, + "observation.cells.releasedToolchain", "released archive identity mismatch"); + const rid = `${{ windows: "win", linux: "linux", macos: "osx" }[cell.environment.os]}-${cell.environment.architecture}`; + check(released.archive.rid === rid || cell.environment.os === "linux" && released.archive.rid === `linux-musl-${cell.environment.architecture}`, + "observation.cells.releasedToolchain", "released archive platform mismatch"); + const { validateArchiveReceipt } = require("./archive-provenance.cjs"); + try { + validateArchiveReceipt({ + policy: trustedPolicy, kind: "released", sourceSha: null, product: "sdk", + version: cell.sdk.version, platform: { ...cell.environment, rid: released.archive.rid }, + archive: released.archive, + }, released.archive, cell.environment.os === "windows" ? filePath.win32 : filePath.posix); + } catch (error) { throw new ContractError(`observation.cells.releasedToolchain.archive: ${error.message}`); } + } check(equal(cell.outcome, classifyAttempts(cell.attempts, manifest)), "observation.cells.outcome", "does not match repeated evidence"); } const results = observation.cells.map(c => c.outcome); @@ -444,6 +466,98 @@ function admitBisection(input) { } } +function validateAssociations(value) { + object(value, { + schemaVersion: literal(1), repository: literal(trustedPolicy.repository), sourceSha, + status: oneOf(["unique", "none", "ambiguous", "incomplete"]), reason: text, + pulls: list(shape({ + number: positive, url: httpsUrl, mergeSha: sourceSha, baseSha: sourceSha, headSha: sourceSha, + baseRef: text, relation: oneOf(["merge-commit", "contained-commit"]), onLineage: bool, + }), 0, 100), + }, "associations"); + unique(value.pulls.map(p => p.number), "associations.pulls"); + for (const p of value.pulls) check(p.url === `https://github.com/${value.repository}/pull/${p.number}`, + "associations.url", "unverified repository URL"); + check(value.status !== "none" || value.pulls.length === 0, "associations", "none cannot contain associations"); + check(value.status !== "unique" || value.pulls.length === 1 && value.pulls[0].onLineage && + value.pulls[0].baseRef === "main", "associations", "unique mapping requires tested main lineage"); + return true; +} + +function validateConfirmation(value) { + object(value, { + schemaVersion: literal(1), epoch: digest, manifestDigest: digest, startGeneration: integer, + candidate: nullable(sourceSha), + classification: oneOf(["verified-introducing-source-boundary", "verified-triggering-change", "unconfirmed-candidate"]), + reason: text, + interval: nullable(shape({ goodSha: sourceSha, badSha: sourceSha, remaining: list(sourceSha, 0, 4096) })), + parents: list(shape({ sourceSha, comparable: bool, observationDigest: nullable(digest) }), 0, 16), + candidateObservationDigest: nullable(digest), searchObservationDigest: nullable(digest), + history: shape({ + commits: list(shape({ sha: sourceSha, parents: list(sourceSha, 0, 16), messageDigest: digest, + changedPaths: list(repositoryPath), patchId: nullable(sourceSha), reverts: list(sourceSha), + cherryPicks: list(sourceSha) }), 0, 128), + monotonicity: oneOf(["not-contradicted", "uncertain", "contradicted"]), toolchainCoupled: bool, + }), + associations: nullable(validateAssociations), latentRootCause: literal("unproven"), + }, "confirmation"); + unique(value.parents.map(p => p.sourceSha), "confirmation.parents"); + unique(value.history.commits.map(c => c.sha), "confirmation.history"); + check(value.associations === null || value.associations.sourceSha === value.candidate, + "confirmation.associations", "wrong candidate"); + if (value.classification !== "unconfirmed-candidate") { + check(value.candidate !== null && value.candidateObservationDigest !== null && value.searchObservationDigest !== null && + value.parents.length > 0 && value.parents.every(p => p.comparable && p.observationDigest !== null) && + value.history.monotonicity === "not-contradicted", "confirmation", "missing independent boundary evidence"); + unique([value.candidateObservationDigest, ...value.parents.map(p => p.observationDigest)], "confirmation.observations"); + const candidate = value.history.commits.find(c => c.sha === value.candidate); + check(candidate && equal(candidate.parents, value.parents.map(p => p.sourceSha)), + "confirmation.parents", "parent set does not match Git evidence"); + check(value.classification !== "verified-introducing-source-boundary" || + value.parents.length === 1 && !value.history.toolchainCoupled, + "confirmation", "merge/toolchain evidence only establishes a trigger"); + } + return true; +} + +function validateConfirmationEvidence(cp) { + const c = cp.state.confirmation; + if (!c) return true; + validateConfirmation(c); + check(c.epoch === cp.state.epoch && c.manifestDigest === cp.state.approvalFingerprint && + c.startGeneration <= cp.generation, "confirmation", "stale confirmation"); + if (c.searchObservationDigest !== null) { + const search = cp.observations[c.searchObservationDigest]; + const reservation = cp.records.find(r => r.id === search?.attemptId); + check(search?.valid && search.data.kind === "source" && search.data.target.sourceSha === c.candidate && + search.data.outcome.outcome === "product-failure" && reservation?.generation <= c.startGeneration, + "confirmation", "unauthenticated search candidate"); + } + const seen = new Set(); + for (const [sha, id, expected] of [[c.candidate, c.candidateObservationDigest, "product-failure"], + ...c.parents.map(p => [p.sourceSha, p.observationDigest, "pass"])]) { + if (id === null) continue; + const record = cp.observations[id]; + const reservation = cp.records.find(r => r.id === record?.attemptId); + check(record?.valid && reservation?.status === "committed" && reservation.request.phase === "confirmation" && + reservation.generation > c.startGeneration && reservation.executionGeneration !== undefined && + record.data.kind === "source" && record.data.target.sourceSha === sha && + !seen.has(record.data.buildInstanceId), "confirmation", "not an independent charged confirmation"); + validateObservation(record.data, cp.state.manifest, + { finalConfirmation: { runId: reservation.owner.runId, buildInstanceId: reservation.id } }); + seen.add(record.data.buildInstanceId); + if (c.classification !== "unconfirmed-candidate") { + const search = cp.observations[c.searchObservationDigest].data; + check(record.data.outcome.outcome === expected && record.data.cells.every(cell => { + const old = search.cells.find(s => s.id === cell.id); + return cell.buildToolchain && equal(cell.environment, old.environment) && + equal(cell.sdk, old.sdk) && equal(cell.runtime, old.runtime); + }), "confirmation", "boundary outcome/toolchain mismatch"); + } + } + return true; +} + function validateState(state) { try { object(state, { @@ -462,6 +576,7 @@ function validateState(state) { } : {}), }, p)), publication: shape({ digest: nullable(digest), commentId: nullable(positive) }), + ...(state && Object.hasOwn(state, "confirmation") ? { confirmation: validateConfirmation } : {}), }, "state"); validateBudget(state.budget, state.manifest); check(state.repository === state.manifest.repository && state.issueNumber === state.manifest.issueNumber && @@ -472,7 +587,9 @@ function validateState(state) { const budget = state.budget; const evidence = [...Object.values(state.endpointEvidence).filter(x => x !== null), ...state.testedRevisions.map(r => r.observationDigest), ...state.skips.map(r => r.observationDigest), - ...(state.interval?.decisions ?? []).map(r => r.observationDigest)]; + ...(state.interval?.decisions ?? []).map(r => r.observationDigest), + ...(state.confirmation ? [state.confirmation.candidateObservationDigest, state.confirmation.searchObservationDigest, + ...state.confirmation.parents.map(p => p.observationDigest)].filter(Boolean) : [])]; check(evidence.every(d => budget.authenticatedObservationDigests.includes(d)), "state", "unauthenticated evidence"); if (state.interval?.decisions) { unique(state.interval.decisions.map(d => d.sha), "state.interval.decisions"); @@ -487,4 +604,5 @@ function validateState(state) { module.exports = { isEligibleIssue, validatePolicy, validateManifest, fingerprintInput, classifyAttempts, validateObservation, admitBisection, validateOutcome, validateState, fingerprintEpoch, + validateAssociations, validateConfirmation, validateConfirmationEvidence, }; diff --git a/eng/regression-bisect/report.cjs b/eng/regression-bisect/report.cjs new file mode 100644 index 00000000000..e44636ae533 --- /dev/null +++ b/eng/regression-bisect/report.cjs @@ -0,0 +1,327 @@ +const model = require("./model.cjs"); +const state = require("./state.cjs"); +const { readIssueSnapshot } = require("./intake.cjs"); +const { createHash } = require("node:crypto"); + +const fingerprint = model.fingerprintInput; +const equal = (a, b) => fingerprint(a) === fingerprint(b); +const requireValue = (value, message) => { if (!value) throw new Error(message); }; +const digest = value => typeof value === "string" && /^[a-f0-9]{64}$/.test(value); +const sha = value => typeof value === "string" && /^[a-f0-9]{40}$/.test(value); +const positive = value => Number.isSafeInteger(value) && value > 0; +const hash = bytes => createHash("sha256").update(bytes).digest("hex"); +function fields(value, names) { + requireValue(value && Object.getPrototypeOf(value) === Object.prototype && + equal(Object.keys(value).sort(), names.split(" ").sort()), "invalid report schema fields"); +} +function reproductionDigest(manifest) { + const { discussionFingerprint, approvedInputDigest, searchHints, ...reproduction } = manifest; + return fingerprint(reproduction); +} +function safeEvidenceUrl(url) { + requireValue(typeof url === "string" && ( + /^https:\/\/api\.github\.com\/repos\/dotnet\/fsharp\/actions\/artifacts\/[1-9]\d*\/zip$/.test(url) || + /^https:\/\/github\.com\/dotnet\/fsharp\/actions\/runs\/[1-9]\d*\/artifacts\/[1-9]\d*$/.test(url)), + "unsafe evidence origin or path"); +} +function validateReport(report) { + fields(report, "schemaVersion kind state observations deadline inputDigest reproductionDigest policyDigest checkedHead"); + requireValue(report.schemaVersion === 1 && report.kind === "regression-investigation", "invalid report kind"); + model.validateState(report.state); + const s = report.state; + requireValue(s.publication.digest === null && s.publication.commentId === null && + report.inputDigest === s.manifest.approvedInputDigest && + report.reproductionDigest === reproductionDigest(s.manifest) && + report.policyDigest === fingerprint(s.budget.policy) && sha(report.checkedHead) && report.checkedHead === s.pinnedMainSha && + typeof report.deadline === "string" && Number.isFinite(Date.parse(report.deadline)), "report identity mismatch"); + requireValue(Array.isArray(report.observations) && report.observations.length <= s.budget.policy.limits.workerUnits, + "invalid report observations"); + const identities = []; + for (const record of report.observations) { + fields(record, "digest data evidence"); + model.validateObservation(record.data, s.manifest); + requireValue(record.digest === fingerprint(record.data) && + record.data.provenance.workflowRevision === s.budget.workflowRevision && + record.data.provenance.policyRevision === s.budget.policyRevision, "report observation identity mismatch"); + requireValue(fingerprint(Object.fromEntries(record.data.cells.map(c => [c.id, c.environment]))) === + s.budget.environmentFingerprint, "report observation environment mismatch"); + requireValue(Array.isArray(record.evidence) && record.evidence.length <= 16, "invalid report evidence"); + for (const item of record.evidence) { + fields(item, "required sha256 url available"); + requireValue(digest(item.sha256) && typeof item.required === "boolean" && typeof item.available === "boolean" && + (!item.required || item.available), "invalid or unavailable required report evidence"); + safeEvidenceUrl(item.url); + } + identities.push(record.digest); + } + requireValue(equal(identities, s.budget.authenticatedObservationDigests), "report evidence authentication mismatch"); + if (s.confirmation) { + model.validateConfirmation(s.confirmation); + requireValue(s.confirmation.epoch === s.epoch && s.confirmation.manifestDigest === s.approvalFingerprint, + "stale report confirmation"); + for (const id of [s.confirmation.candidateObservationDigest, s.confirmation.searchObservationDigest, + ...s.confirmation.parents.map(p => p.observationDigest)].filter(Boolean)) { + requireValue(identities.includes(id), "unauthenticated report confirmation"); + } + if (s.confirmation.classification !== "unconfirmed-candidate") { + const released = report.observations.find(o => o.digest === s.endpointEvidence.stable)?.data; + requireValue(released?.kind === "released" && equal(released.target.sdk, s.budget.stableSdk) && + released.outcome.outcome === "product-failure" && released.cells.every(cell => + cell.releasedToolchain && cell.runtime && cell.artifacts.compiler?.assemblyVersion && + cell.artifacts.core?.assemblyVersion), + "verified report requires released endpoint archive, compiler runtime and compiler/Core assembly versions"); + } + } + requireValue(Buffer.byteLength(JSON.stringify(report)) <= 2 * 1024 * 1024, "report evidence too large"); + return true; +} + +// Markdown is generated only from typed evidence, never supplied by the issue or a worker. +function display(value, limit = 320) { + const text = typeof value === "string" ? value : JSON.stringify(value); + return text.slice(0, limit).replace(/[\u0000-\u001f\u007f-\u009f\u2028-\u202e\u2066-\u2069]/g, c => + c === "\n" ? "\\n" : c === "\r" ? "\\r" : c === "\t" ? "\\t" : "\uFFFD") + .replace(/[\\`*_{}\[\]()#+.!|]/g, "\\$&") + .replace(/&/g, "&").replace(//g, ">").replace(/@/g, "@") + + (text.length > limit ? " … [truncated; see authenticated evidence]" : ""); +} +function renderReport(report) { + validateReport(report); + const s = report.state, m = s.manifest, b = s.budget, c = s.confirmation; + const lines = [ + "## Regression investigation", + `Repository/issue: ${s.repository}#${s.issueNumber}`, + `Classification: ${c ? c.classification.replaceAll("-", " ") : "unconfirmed candidate"}.`, + `Stop reason: ${display(c?.reason ?? `checkpoint phase ${s.phase}; no verified source confirmation recorded`)}.`, + "The latent root cause is unproven. Bisection does not establish semantic causation.", + `Input: ${report.inputDigest}; discussion: ${s.issueFingerprint}; epoch: ${s.epoch}.`, + `Approval manifest: ${s.approvalFingerprint}; approval revision: ${b.authorization.approvalRevision}; path: ${display(b.authorization.approvalPath)}.`, + `Policy: ${report.policyDigest} at ${b.policyRevision}; workflow: ${b.workflowRevision}; reproduction: ${report.reproductionDigest}.`, + `Tested pinned main: ${s.pinnedMainSha}; current-head check: ${report.checkedHead} (matched at preparation; recheck before writing).`, + `Sample environment fingerprint: ${b.environmentFingerprint ?? "not recorded"}.`, + "", + "### Approved commands and frozen reproduction", + `Reproduction kind: ${m.kind}; timeout: ${m.timeoutSeconds}s. Operations below are runner commands, not arbitrary shell input.`, + ...m.sources.map(source => `- Source ${display(source.path)}: SHA-256 ${source.sha256}.`), + `Retained producer: ${display(m.producer.name)} ${m.producer.version}; SHA-256 ${m.producer.sha256}.`, + ...m.packages.map(p => `- Package ${display(p.id)} ${p.version}: SHA-256 ${p.sha256}.`), + ...m.matrix.map(cell => `- ${display(cell.id)}: ${cell.os}/${cell.architecture}; TFM ${cell.tfm}; ${cell.configuration}; language ${cell.languageVersion}; approved operation ${display(cell.command.join(" "))}.`), + "", + "### Expected assertions", + ...Object.entries(m.assertions).map(([name, assertion]) => + `- ${name}: exit ${assertion.exitCode}; stdout ${display(JSON.stringify(assertion.stdout), 1024)}; stderr ${display(JSON.stringify(assertion.stderr), 1024)}; assertion digest ${fingerprint(assertion)}.`), + "", + "### Released SDK selection", + `Latest stable selected: ${b.stableSdk.version}; official release index ${display(b.stableSdk.releaseIndexUrl)}; metadata SHA-256 ${b.stableSdk.releaseIndexDigest}.`, + ...b.stableSdk.archives.map(a => `- ${a.os}/${a.architecture}: archive SHA-256 ${a.sha256}.`), + ...(s.endpointEvidence.stable ? [] : ["Released endpoint archive/runtime/compiler/Core evidence: not recorded."]), + "", + "### Endpoint and candidate evidence", + ...Object.entries(s.endpointEvidence).map(([name, id]) => `- Endpoint ${name}: ${id ?? "not recorded"}.`), + `Candidate: ${c?.candidate ?? "not confirmed"}.`, + `Remaining interval: ${display(c?.interval ?? s.interval ?? "not recorded", 1600)}.`, + `Parent evidence: ${c?.parents.length ? "" : "not recorded"}`, + ...(c?.parents ?? []).map(p => `- ${p.sourceSha}: comparable ${p.comparable}; observation ${p.observationDigest ?? "not recorded"}.`), + ...s.skips.map(skip => `- Skipped ${skip.sourceSha}: ${skip.reason}; observation ${skip.observationDigest}.`), + `Monotonicity: ${c?.history.monotonicity ?? "uncertain"}; source/toolchain coupled: ${c ? c.history.toolchainCoupled : "not established"}.`, + ...(c?.history.commits ?? []).map(commit => + `- History ${commit.sha}: parents ${commit.parents.join(", ") || "root"}; patch ${commit.patchId ?? "not recorded"}; reverts ${commit.reverts.join(", ") || "none recorded"}; cherry-picks ${commit.cherryPicks.join(", ") || "none recorded"}; changed paths ${display(commit.changedPaths, 512)}; message SHA-256 ${commit.messageDigest}.`), + `PR association: ${c?.associations?.status ?? "not established"}; ${display(c?.associations?.reason ?? "no verified association evidence")}.`, + ...(c?.associations?.pulls ?? []).map(p => + `- ${p.url}: ${p.relation}; merge ${p.mergeSha}; base ${p.baseSha}; head ${p.headSha}; base ref ${display(p.baseRef)}; tested lineage ${p.onLineage}.`), + "", + "### Authenticated observations and logs", + ]; + const toolchains = new Set(), samples = new Set(), attempts = new Set(); + for (const record of report.observations) { + const o = record.data; + lines.push(`- Observation ${record.digest}: ${o.kind} ${o.kind === "source" ? o.target.sourceSha : o.target.sdk.version}; ${o.outcome.outcome}/${o.outcome.reason}; build ID ${display(o.buildInstanceId)}; cached ${o.cached}; run ${display(o.provenance.runId)}; worker ${display(o.provenance.workerId)}.`); + for (const cell of o.cells) { + const sample = fingerprint({ environment: cell.environment, sdk: cell.sdk, runtime: cell.runtime }); + lines.push(samples.has(sample) + ? ` - Matrix ${display(cell.id)}: same sample SDK/runtime/environment ${sample}.` + : ` - Matrix ${display(cell.id)}: sample ${sample}; ${cell.environment.os}/${cell.environment.architecture}; image ${cell.environment.imageDigest}; SDK ${cell.sdk?.version ?? "not recorded"} (${cell.sdk?.sha256 ?? "unavailable"}); runtime ${cell.runtime?.version ?? "not recorded"} (${cell.runtime?.sha256 ?? "unavailable"}).`); + samples.add(sample); + lines.push(" - Artifacts below belong to this observation's source/build ID:"); + for (const [name, artifact] of Object.entries(cell.artifacts)) { + lines.push(` - ${name}: ${artifact ? `${artifact.origin}; SHA-256 ${artifact.sha256}` + + (name === "targets" ? "" : `; assembly version ${artifact.assemblyVersion ?? "not recorded"}`) : "identity unavailable"}.`); + } + const released = o.kind === "released"; + const t = released ? cell.releasedToolchain : cell.buildToolchain; + const toolchainId = t && fingerprint({ ...t, sourceSha: null }); + if (t && toolchains.has(toolchainId)) { + lines.push(` - ${released ? "Released" : "Revision"} toolchain: same identity and archive receipt ${toolchainId}, for this observation.`); + } else if (t) { + toolchains.add(toolchainId); + lines.push(released + ? ` - Released toolchain ${toolchainId}: selected SDK ${t.sdk.version}; SHA-256 ${t.sdk.sha256}; Released compiler runtime ${t.compilerRuntime.version}; SHA-256 ${t.compilerRuntime.sha256}.` + : ` - Revision toolchain ${toolchainId}: global.json SHA-256 ${t.globalJsonDigest}; evidence ${t.evidenceDigest}; requested SDK/tools/runtime ${t.requestedSdk}/${t.requestedTools}/${t.requestedCompilerRuntime}; resolved SDK/runtime ${t.sdk.version}/${t.compilerRuntime.version}.`); + const a = t.archive; + lines.push(` - ${released ? "Released" : "Source"} archive receipt: ${display(a.url)}; ${a.rid}; SHA-256 ${a.sha256}; actual SHA-512 ${a.actualSha512}; ${a.bytesReceived} bytes; received ${a.receivedAt}; retained SHA-256 ${a.retained.sha256}.`); + lines.push(a.provenance.kind === "publisher-checksum" + ? ` - Publisher-provided SHA-512 ${a.sha512}.` + : ` - Approved official-HTTPS measured archive hash, NOT a publisher-provided checksum; approval ${a.provenance.approvalRevision}/${a.provenance.approvalDigest}.`); + lines.push(` - Archive final origin ${display(a.finalOrigin)}; final URL ${display(a.redirectUrl)}; redirects ${display(a.redirects)}.`); + } else lines.push(released ? " - Released SDK archive receipt and compiler runtime: not recorded." : + " - Revision-specific requested/resolved build toolchain and archive receipt: not recorded."); + cell.attempts.forEach((attempt, i) => { + const id = fingerprint(attempt); + lines.push(attempts.has(id) + ? ` - Assertion repetition ${i + 1}: identical checks and streams to ${id}; ${attempt.execution.status}; exit ${attempt.execution.exitCode}.` + : ` - Assertion repetition ${i + 1}: ${attempt.execution.status}; exit ${attempt.execution.exitCode}; checks ${display(attempt.checks)}; stdout ${display(JSON.stringify(attempt.execution.stdout), 256)}; stderr ${display(JSON.stringify(attempt.execution.stderr), 256)}; full attempt digest ${id}.`); + attempts.add(id); + }); + } + lines.push(...record.evidence.map(e => ` - ${e.required ? "Required" : "Optional"} log/evidence reference: ${display(e.url)}; SHA-256 ${e.sha256}; ${e.available ? "hash verified" : "unavailable or expired (including hash mismatch)"}.`)); + if (!record.evidence.length) lines.push(" - External log references not recorded; assertion streams are retained in the authenticated result envelope."); + } + lines.push("", "### Budget and resume", + `Allowance ${display(b.allowanceId)}; deadline ${report.deadline}; ${b.consumed.sourceBuilds}/${b.policy.limits.sourceBuilds} source builds; ${b.consumed.workerUnits}/${b.policy.limits.workerUnits} worker units; ${b.consumed.reservedMinutes}/${b.policy.limits.reservedMinutes} reserved minutes.`, + `Charged attempts: ${display(b.attempts, 1600)}. Cancellations and stale inputs do not refund allowance.`, + `Resume: ${s.phase === "complete" ? "confirmation complete; publication only" : "trusted controller may resume from the authenticated checkpoint after refreshing approval and inputs"}; phase ${s.phase}; no automatic reset of spent budget.`); + const body = lines.join("\n"); + requireValue(Buffer.byteLength(body) < 32 * 1024, "rendered report exceeds bounded display size"); + return body; +} + +function reportFromCheckpoint(cp, checkedHead, evidenceHashes) { + const snapshot = structuredClone(cp.state); + snapshot.publication = { digest: null, commentId: null }; + return { + schemaVersion: 1, kind: "regression-investigation", state: snapshot, + observations: Object.entries(cp.observations).filter(([, o]) => o.valid) + .map(([digest, o]) => ({ digest, data: structuredClone(o.data), + evidence: o.evidence.map(e => ({ ...e, available: evidenceHashes.get(e.url) === e.sha256 })) })), + deadline: cp.deadline, inputDigest: cp.inputDigest, reproductionDigest: cp.reproductionDigest, + policyDigest: cp.policyDigest, checkedHead, + }; +} + +/** + * deps.context is the full existing state context, including trusted record + * identity, keys, owner, signing workflow revision and checkpoint anchor. + * withLock(action) holds the per-issue controller lock across every read/write. + * readCurrentContext({now,repository,issueNumber}) must freshly read trusted + * manifest, authorization, policy, policyRevision, workflowRevision, pinnedMainSha + * (current main HEAD), stableSdk, environmentFingerprint, and readSource({path,sha256}). + * readSource enforces approved pinned tracked-blob/symlink policy. No caller + * snapshot, report body, destination, intake replacement or authentication callback + * is accepted. transport.request is real paginated intake's Octokit-shaped seam. + * readEvidence must honor redirect:"error" and maxBytes; redirects are never + * followed by this wrapper. Prepare updates only checkpoint/receipt, not a result. + */ +async function preparePublication(options, deps) { + try { + fields(options, "now generation"); + requireValue(typeof options.now === "string" && Number.isFinite(Date.parse(options.now)) && + Number.isSafeInteger(options.generation) && options.generation >= 0, "invalid publication request"); + requireValue(typeof deps.withLock === "function" && typeof deps.readCurrentContext === "function", + "trusted current context reader and controller lock required"); + return await deps.withLock(async () => { + const context = { ...deps.context, now: options.now }; + requireValue(context.repository === "dotnet/fsharp" && positive(context.issueNumber), "invalid selected issue"); + const evidenceHashes = new Map(); + const transport = { ...deps.transport, readEvidence: async url => { + safeEvidenceUrl(url); + const bytes = await deps.transport.readEvidence(url, { redirect: "error", maxBytes: 4 * 1024 * 1024 }); + requireValue(bytes === null || Buffer.isBuffer(bytes) && bytes.length <= 4 * 1024 * 1024, + "oversized or invalid evidence response"); + evidenceHashes.set(url, bytes === null ? null : hash(bytes)); + return bytes; + } }; + const loaded = await state.loadCheckpoint(transport, context); + const cp = loaded.checkpoint; + const originalGeneration = cp.generation - (cp.publicationStatus === "pending" ? 1 : cp.publicationStatus === "published" ? 2 : 0); + requireValue(options.generation === cp.generation || options.generation === originalGeneration || + cp.publicationStatus === "published" && options.generation === cp.generation - 1, + "stale checkpoint generation"); + const stop = async (reason, status = "blocked") => { + await state.recordPublication(transport, context, { generation: cp.generation, + status: "blocked", digest: cp.state.publication.digest, commentId: cp.state.publication.commentId }); + return { status, reason, budget: structuredClone(cp.state.budget.consumed) }; + }; + let fresh; + try { + fresh = await deps.readCurrentContext({ now: options.now, repository: context.repository, issueNumber: context.issueNumber }); + requireValue(fresh && typeof fresh.readSource === "function" && digest(fresh.environmentFingerprint), + "fresh approval sources and environment fingerprint required"); + model.validateManifest(fresh.manifest); + model.validatePolicy(fresh.policy); + } catch (error) { return stop(error.message); } + const records = new Map(), checkpoints = new Map([[context.commentId, loaded]]); + const authenticateRecord = async comment => { + const decoded = await state.authenticateDecodedRecord(transport, { ...context, commentId: comment.id }, comment); + if (!decoded) return false; + records.set(comment.id, decoded); + if (decoded.kind === "checkpoint" && !checkpoints.has(comment.id)) { + checkpoints.set(comment.id, await state.loadCheckpoint(transport, { ...context, commentId: comment.id })); + } + return true; + }; + const snapshot = await readIssueSnapshot({ + repository: context.repository, issueNumber: context.issueNumber, transport, now: options.now, + reviewedManifest: fresh.manifest, readSource: fresh.readSource, authenticateRecord, + readTrustedCheckpoints: async () => [...checkpoints.values()].map(v => ({ + status: "loaded", state: v.checkpoint.state, commentId: v.commentId, owner: v.owner, pending: v.checkpoint.pending, + })), + }); + if (snapshot.status !== "complete") return stop(snapshot.reason, + snapshot.status === "ineligible" ? "blocked" : /stale|mismatch/.test(snapshot.reason) ? "requeue" : "blocked"); + const current = snapshot.snapshot; + const mismatches = [ + ["discussion", current.discussionFingerprint, cp.state.issueFingerprint], + ["manifest", fresh.manifest, cp.state.manifest], ["approval", fresh.authorization, cp.state.budget.authorization], + ["policy", fresh.policy, cp.state.budget.policy], ["policy revision", fresh.policyRevision, cp.state.budget.policyRevision], + ["workflow revision", fresh.workflowRevision, cp.state.budget.workflowRevision], + ["main head", fresh.pinnedMainSha, cp.state.pinnedMainSha], ["stable SDK", fresh.stableSdk, cp.state.budget.stableSdk], + ["environment", fresh.environmentFingerprint, cp.state.budget.environmentFingerprint], + ].filter(([, actual, expected]) => actual === undefined || expected === undefined || !equal(actual, expected)).map(([name]) => name); + if (mismatches.length) return stop(`stale ${mismatches.join(", ")}; requeue with spent allowance preserved`, "requeue"); + if (!loaded.publicationAllowed) return stop(loaded.blockers.join("; ") || "publication is stopped"); + if (current.investigations.active.some(active => active.commentId !== context.commentId || + active.owner?.runId !== context.owner.runId || active.owner?.attempt !== context.owner.attempt)) { + return stop("another active investigation owns this issue"); + } + if (cp.pending || !["complete", "blocked"].includes(cp.state.phase)) { + return stop("work is pending or search has not reached a reportable state"); + } + const report = reportFromCheckpoint(cp, fresh.pinnedMainSha, evidenceHashes); + let body; + try { body = renderReport(report); } catch (error) { return stop(error.message); } + const resultDigest = fingerprint(report); + // Enforce the combined human report + signed envelope limit before issuing an intent. + state.serializeWorkflowRecord({ kind: "result", status: "Regression investigation result", + payload: { schemaVersion: 1, digest: resultDigest, report } }, context); + const matches = [...records.values()].filter(r => r.kind === "result" && + r.payload.digest === resultDigest && equal(r.payload.report, report)); + if (matches.length > 1) return stop("conflicting duplicate authenticated results"); + if (cp.state.publication.digest && cp.state.publication.digest !== resultDigest) return stop("pending publication digest changed", "requeue"); + if (matches.length) { + requireValue(cp.state.publication.digest === resultDigest, "result has no matching publication intent"); + const saved = await state.recordPublication(transport, context, { generation: cp.generation, + status: "published", digest: resultDigest, commentId: matches[0].commentId }); + return { status: "published", digest: resultDigest, commentId: matches[0].commentId, generation: saved.generation }; + } + if (cp.publicationStatus === "published") return stop("authenticated published result is missing"); + const duplicate = cp.publicationStatus === "pending"; + const saved = await state.recordPublication(transport, context, { generation: cp.generation, + status: "pending", digest: resultDigest, commentId: null }); + return { status: duplicate ? "pending" : "ready", repository: cp.state.repository, issueNumber: cp.state.issueNumber, + digest: resultDigest, generation: saved.generation, report, body }; + }); + } catch (error) { return { status: "blocked", reason: error.message }; } +} + +async function completePublication(options, deps) { + try { + fields(options, "now generation digest commentId"); + requireValue(digest(options.digest) && positive(options.commentId), "invalid publication completion"); + const result = await preparePublication({ now: options.now, generation: options.generation }, deps); + if (result.status === "published") requireValue(result.digest === options.digest && + result.commentId === options.commentId, "publication completion identity mismatch"); + return result; + } catch (error) { return { status: "blocked", reason: error.message }; } +} + +module.exports = { renderReport, validateReport, preparePublication, completePublication }; diff --git a/eng/regression-bisect/report.test.cjs b/eng/regression-bisect/report.test.cjs new file mode 100644 index 00000000000..2fcc3a1d594 --- /dev/null +++ b/eng/regression-bisect/report.test.cjs @@ -0,0 +1,505 @@ +const { test } = require("node:test"); +const assert = require("node:assert/strict"); +const f = require("./fixtures/contracts.cjs"); +const { harness } = require("./fixtures/checkpoint.cjs"); +const model = require("./model.cjs"); +const state = require("./state.cjs"); +const intake = require("./intake.cjs"); +const { preparePublication, renderReport, completePublication } = require("./report.cjs"); + +async function fixture({ outcome = "product-failure", evidence = [], producerName = "repro", matrixCount = 1, + archiveKind = "publisher-checksum" } = {}) { + const h = harness(), calls = [], comments = []; + const issue = { ...h.context.issue, id: 900, title: "Regression", body: "Reviewed reproduction", + user: { id: 20 }, updated_at: h.context.now }; + const statusRequest = h.transport.request; + h.transport.request = async (route, args) => { + calls.push({ route, args }); + if (route.includes("/statuses")) return statusRequest(route, args); + if (route.endsWith("/issues/{issue_number}")) return { data: structuredClone(issue) }; + if (route.endsWith("/comments")) return { + data: structuredClone(args.page === 1 ? comments : h.comment.body.startsWith("unused") ? [] : [h.comment]), + headers: { link: args.page === 1 ? '; rel="next"' : "" }, + }; + if (route.endsWith("/timeline")) return { data: [], headers: { link: "" } }; + if (route.includes("/actions/runs/")) return { data: await h.transport.getRun(args.run_id, args.attempt_number) }; + throw new Error(`unexpected route ${route}`); + }; + const initial = await intake.readIssueSnapshot({ repository: h.context.repository, issueNumber: 123, transport: h.transport }); + assert.equal(initial.status, "complete", initial.reason); + h.context.issueFingerprint = initial.snapshot.discussionFingerprint; + h.context.manifest.discussionFingerprint = h.context.issueFingerprint; + h.context.manifest.producer.name = producerName; + if (archiveKind === "approved-official-https-measured") { + h.context.manifest.matrix[0].id = "windows-x64"; + h.context.manifest.matrix[0].os = "windows"; + h.context.pinnedMainSha = f.policy().toolchains.measuredArchives[0].sourceSha; + } + if (matrixCount === 2) h.context.manifest.matrix.push({ + ...h.context.manifest.matrix[0], id: "windows-x64", os: "windows", + }); + f.seal(h.context.manifest); + h.context.stableSdk = f.observation("released", "product-failure", h.context.manifest).target.sdk; + h.context.authorization.manifestDigest = model.fingerprintInput(h.context.manifest); + h.context.authorization.inputDigest = h.context.manifest.approvedInputDigest; + const environments = Object.fromEntries(h.context.manifest.matrix.map(c => + [c.id, { os: c.os, architecture: c.architecture, imageDigest: f.digest("runner image") }])); + h.context.environmentFingerprint = model.fingerprintInput(environments); + await h.start(); + const pending = await state.reserveWork(h.transport, h.context, { generation: 0, attemptId: "report-main", + request: { kind: "source", target: { sourceSha: h.context.pinnedMainSha }, endpoint: "main", phase: "main" } }); + const observation = f.observation("source", outcome, h.context.manifest, h.context.pinnedMainSha); + observation.buildInstanceId = pending.id; + for (const cell of observation.cells) { + for (const artifact of Object.values(cell.artifacts)) artifact.buildInstanceId = pending.id; + cell.buildToolchain = f.buildToolchain(observation.target.sourceSha, cell.environment.os, archiveKind); + } + await state.commitObservation(h.transport, h.context, { + generation: pending.generation, attemptId: pending.id, observation, evidence, + }); + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + cp.state.phase = "blocked"; + h.checkpoint(cp); + await state.loadCheckpoint(h.transport, h.context); + const fresh = { manifest: structuredClone(h.context.manifest), authorization: structuredClone(h.context.authorization), + policy: f.policy(), policyRevision: h.context.policyRevision, workflowRevision: h.context.workflowRevision, + pinnedMainSha: h.context.pinnedMainSha, stableSdk: structuredClone(h.context.stableSdk), + environmentFingerprint: h.context.environmentFingerprint, readSource: async () => Buffer.from('printfn "ok"') }; + let reads = 0, lockDepth = 0; + const deps = { context: h.context, transport: h.transport, + withLock: fn => h.lock(async () => { lockDepth++; try { return await fn(); } finally { lockDepth--; } }), + readCurrentContext: async () => { assert.equal(lockDepth, 1); reads++; return fresh; } }; + const generation = (await state.loadCheckpoint(h.transport, h.context)).checkpoint.generation; + return Object.assign(h, { calls, comments, issue, fresh, deps, generation, getReads: () => reads, + prepare: options => preparePublication({ now: h.context.now, generation, ...options }, deps) }); +} + +test("fresh complete intake produces a bounded deterministic pending intent, never a report write", async () => { + const h = await fixture(); + const result = await h.prepare(); + assert.equal(result.status, "ready", JSON.stringify(result)); + assert.equal(h.getReads(), 1); + assert.ok(h.calls.some(c => c.route.endsWith("/comments") && c.args.page === 2)); + assert.ok(h.calls.some(c => c.route.endsWith("/timeline"))); + assert.ok(h.calls.some(c => c.route.includes("/actions/runs/"))); + assert.equal(result.repository, "dotnet/fsharp"); + assert.equal(result.issueNumber, 123); + assert.equal(result.body, renderReport(result.report)); + assert.equal(result.digest, model.fingerprintInput(result.report)); + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + assert.equal(cp.publicationStatus, "pending"); + assert.deepEqual(cp.state.publication, { digest: result.digest, commentId: null }); + assert.equal(cp.state.budget.consumed.workerUnits, 1); + assert.ok(h.calls.every(c => !c.route.startsWith("POST ") || c.route.includes("/statuses/"))); + const duplicate = await h.prepare(); + assert.equal(duplicate.status, "pending"); + assert.equal(duplicate.digest, result.digest); + assert.equal(duplicate.generation, result.generation); +}); + +test("reports expose assertions, matrix, identities, provenance, uncertainty and charged resume allowance", async () => { + const h = await fixture(); + const result = await h.prepare(); + assert.equal(result.status, "ready", result.reason); + for (const text of ["Expected assertions", "Approved commands", "net10", "Release", "language", + "producer", "SDK", "runtime", "compiler", "tasks", "targets", "core", "global", "requested", + "resolved", "publisher", "report-main", "Endpoint", "Parent", "interval", "unconfirmed", + "latent root cause", "PR", "Budget", "110", "Resume", "archive receipt", "not recorded"]) { + assert.ok(result.body.toLowerCase().includes(text.toLowerCase()), text); + } + assert.ok(result.body.includes(h.context.pinnedMainSha)); + assert.ok(result.body.includes(h.context.manifest.approvedInputDigest)); + assert.ok(result.body.includes(f.digest("producer"))); +}); + +test("measured HTTPS archives never become claimed publisher checksums", async () => { + const h = await fixture({ archiveKind: "approved-official-https-measured" }); + const result = await h.prepare(); + assert.equal(result.status, "ready", result.reason); + assert.match(result.body, /official-HTTPS measured archive hash, NOT a publisher-provided checksum/); + assert.ok(!result.body.includes("- Publisher-provided SHA-512")); +}); + +test("ineligible or stale fresh context blocks publication without refund", async t => { + const changes = { + closed: h => { h.issue.state = "closed"; }, + unlabelled: h => { h.issue.labels = []; }, + old: h => { h.issue.created_at = "2026-08-01T00:00:00Z"; }, + discussion: h => { h.comments.push({ id: 800, body: "correction", user: { id: 21 } }); }, + title: h => { h.issue.title = "corrected"; }, + reproduction: h => { h.fresh.readSource = async () => Buffer.from("changed"); }, + manifest: h => { h.fresh.manifest.timeoutSeconds--; f.seal(h.fresh.manifest); }, + approval: h => { h.fresh.authorization.approvalRevision = f.sha("new-approval"); }, + policy: h => { h.fresh.policyRevision = f.sha("new-policy"); }, + workflow: h => { h.fresh.workflowRevision = f.sha("new-workflow"); }, + main: h => { h.fresh.pinnedMainSha = f.sha("new-main"); }, + environment: h => { h.fresh.environmentFingerprint = f.digest("changed image"); }, + missingEnvironment: h => { delete h.fresh.environmentFingerprint; }, + }; + for (const [name, change] of Object.entries(changes)) await t.test(name, async () => { + const h = await fixture(); change(h); + const result = await h.prepare(); + assert.ok(["blocked", "requeue"].includes(result.status), JSON.stringify(result)); + assert.equal(result.body, undefined); + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + assert.equal(cp.publicationStatus, "blocked"); + assert.equal(cp.state.budget.consumed.workerUnits, 1); + assert.equal(cp.state.budget.consumed.reservedMinutes, 110); + }); +}); + +test("fresh reader, lock, bounded intent, and exact generation are mandatory", async t => { + for (const change of [ + h => { delete h.deps.readCurrentContext; }, + h => { delete h.deps.withLock; }, + h => { h.deps.readCurrentContext = async () => { throw new Error("approval unavailable"); }; }, + ]) await t.test("trusted seam missing", async () => { + const h = await fixture(); change(h); + assert.equal((await h.prepare()).status, "blocked"); + }); + const h = await fixture(); + for (const args of [{ generation: h.generation - 1 }, { markdown: "# forged" }, + { issueNumber: 912 }, { repository: "attacker/repo" }]) { + assert.equal((await h.prepare(args)).status, "blocked"); + } +}); + +test("incomplete intake blocks rather than publishing a partial discussion", async () => { + const h = await fixture(), request = h.transport.request; + h.transport.request = async (route, args) => route.endsWith("/timeline") + ? { data: [], headers: { link: '; rel="next"' } } : request(route, args); + const result = await h.prepare(); + assert.equal(result.status, "blocked"); + assert.match(result.reason, /incomplete|page limit/); +}); + +test("signed lost-response results recover only at exact writer/app/comment/run and body", async t => { + for (const mutation of ["none", "id", "writer", "app", "run", "markdown", "digest"]) await t.test(mutation, async () => { + const h = await fixture(); + const ready = await h.prepare(); + assert.equal(ready.status, "ready", ready.reason); + const context = { ...h.context, commentId: 789 }; + const resultPayload = { schemaVersion: 1, digest: ready.digest, report: ready.report }; + if (mutation === "digest") resultPayload.digest = f.digest("another"); + let body; + try { body = state.serializeWorkflowRecord({ kind: "result", status: "Regression investigation result", payload: resultPayload }, context); } + catch (error) { + assert.equal(mutation, "digest"); + assert.match(error.message, /digest/); + return; + } + const comment = { ...structuredClone(h.comment), id: 789, body }; + if (mutation === "id") comment.id++; + if (mutation === "writer") comment.user.id++; + if (mutation === "app") comment.performed_via_github_app.id++; + if (mutation === "run") h.runs.get("12345/1").head_sha = f.sha("forged"); + if (mutation === "markdown") comment.body = "# invented blame\n\n" + comment.body; + h.comments.push(comment); + const recovered = await h.prepare(); + if (mutation === "none") { + assert.equal(recovered.status, "published", JSON.stringify(recovered)); + assert.equal(recovered.commentId, 789); + const cp = (await state.loadCheckpoint(h.transport, h.context)).checkpoint; + assert.equal(cp.publicationStatus, "published"); + const retry = await completePublication({ now: h.context.now, generation: ready.generation, + digest: ready.digest, commentId: 789 }, h.deps); + assert.equal(retry.status, "published", JSON.stringify(retry)); + } else { + assert.ok(["blocked", "requeue"].includes(recovered.status), JSON.stringify(recovered)); + assert.notEqual(recovered.status, "published"); + } + }); +}); + +test("completion refreshes inputs again immediately before acknowledging a result", async () => { + const h = await fixture(), ready = await h.prepare(); + const body = state.serializeWorkflowRecord({ kind: "result", status: "Regression investigation result", + payload: { schemaVersion: 1, digest: ready.digest, report: ready.report } }, { ...h.context, commentId: 789 }); + h.comments.push({ ...structuredClone(h.comment), id: 789, body }); + h.issue.body += " corrected since preparation"; + const result = await completePublication({ now: h.context.now, generation: ready.generation, + digest: ready.digest, commentId: 789 }, h.deps); + assert.ok(["blocked", "requeue"].includes(result.status)); + assert.equal(h.getReads(), 2); +}); + +test("unsafe evidence origins and paths are rejected before any evidence read", async t => { + for (const url of ["https://attacker.invalid/log", "https://api.github.com/repos/other/repo/actions/artifacts/1/zip", + "https://api.github.com/repos/dotnet/fsharp/actions/artifacts/../1/zip", + "https://api.github.com/repos/dotnet/fsharp/actions/artifacts/%31/zip", + "https://api.github.com/repos/dotnet/fsharp/actions/artifacts/1/zip?token=secret"]) { + await t.test(url, async () => { + const h = await fixture({ evidence: [{ url, sha256: f.digest("durable evidence"), required: true }] }); + let reads = 0; + h.transport.readEvidence = async () => { reads++; return Buffer.from("durable evidence"); }; + assert.equal((await h.prepare()).status, "blocked"); + assert.equal(reads, 0); + }); + } + const h = await fixture({ evidence: [{ url: "https://api.github.com/repos/dotnet/fsharp/actions/artifacts/12/zip", + sha256: f.digest("durable evidence"), required: true }] }); + assert.equal((await h.prepare()).status, "ready"); +}); + +test("renderer rejects arbitrary fields and escapes all untrusted display text", async () => { + const h = await fixture({ producerName: "