diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 new file mode 100644 index 000000000000..2cb079c514fa --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 @@ -0,0 +1,2269 @@ +#!/usr/bin/env pwsh +<# +.SYNOPSIS + Deterministic, read-only collector for exactly one open dotnet/aspnetcore test-quarantine issue. + +.DESCRIPTION + Gathers public evidence for a single quarantine issue -- the issue body itself, Azure DevOps + build metadata, authoritative VSTMR test-result detail (errorMessage/stackTrace), and GitHub + "Build Insights" check-run snapshots (corroborating only, never authoritative) -- then emits + either: + * a "candidate" object that independently satisfies test-quarantine-kbe-shadow-candidate.schema.json + and is ready for Evaluate-TestQuarantineKbeCandidate.ps1, or + * a structured "incomplete" outcome explaining exactly which evidence could not be + established, without inferring a pass, a recurrence, a signature, a platform/configuration, + or a validated duplicate from anything missing, ambiguous, or unverifiable. + + This script makes no repository-state mutations. It only reads public GitHub/Azure DevOps + endpoints (or, in fixture mode, a local fixture file) and writes local files: the dossier, an + optional candidate, and capped/redacted evidence text files under -EvidenceRoot. + +.PARAMETER IssueNumber + The dotnet/aspnetcore issue number to evaluate. Must be the canonical, currently open, + automation-generated test-quarantine issue for the test(s) it names. + +.PARAMETER Signature + Optional manual failure-signature override. Supply this when the issue body does not contain + a deterministically extractable "## Error Message" (or equivalent) code block. When omitted and + extraction is ambiguous, the collector fails closed with reason code + 'signature-extraction-ambiguous' rather than guessing. + +.PARAMETER OutputFile + Path to write the dossier JSON (always written, whether the outcome is 'candidate' or + 'incomplete'). + +.PARAMETER CandidateFile + Path to write the candidate JSON. Only written when the dossier outcome is 'candidate'. + +.PARAMETER EvidenceRoot + Directory to materialize capped/redacted raw evidence text files into. Created if missing. + +.PARAMETER FixtureRoot + Optional path to a directory containing a single consolidated 'fixture.json' file that stands + in for every live network call (GitHub issue/check-runs/search/commits and Azure DevOps + build/VSTMR data). Used by the test suite and by the shadow workflow's self-test mode so this + script's deterministic logic can be exercised with zero network access. See README.md for the + fixture.json shape. + +.PARAMETER GitHubToken + GitHub token for the GitHub REST calls (issue, check-runs, search, commits). Falls back to the + GITHUB_TOKEN environment variable. Authenticated calls are strongly preferred: the search API's + unauthenticated rate limit (10 requests/minute) is exhausted by a single run's four duplicate + searches plus one retry. + +.PARAMETER EventRef + Trusted workflow event ref. Live callers must pass github.ref explicitly. Fixture mode defaults + to refs/heads/main only when this parameter is omitted. + +.PARAMETER EventSha + Trusted workflow event SHA. Live callers must pass github.sha explicitly. Fixture mode defaults + to the checked-out repository SHA only when this parameter is omitted. +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [int]$IssueNumber, + + [string]$Signature, + + [Parameter(Mandatory = $true)] + [string]$OutputFile, + + [Parameter(Mandatory = $true)] + [string]$CandidateFile, + + [Parameter(Mandatory = $true)] + [string]$EvidenceRoot, + + [string]$FixtureRoot, + + [string]$GitHubToken = $env:GITHUB_TOKEN, + + [string]$Repository = "dotnet/aspnetcore", + + [string]$RepositoryRoot = "$PSScriptRoot/../../../..", + + [string]$EventRef, + + [string]$EventSha, + + [string]$DossierSchemaFile = "$PSScriptRoot/test-quarantine-kbe-shadow-dossier.schema.json", + + [string]$CandidateSchemaFile = "$PSScriptRoot/test-quarantine-kbe-shadow-candidate.schema.json", + + [int]$RecurrenceScanBuildCap = 20, + + [int]$DuplicateSearchWindowDays = 90, + + [int]$DuplicateSearchPageSize = 100, + + [int]$DuplicateSearchMaxPages = 3 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$ado = "https://dev.azure.com/dnceng-public/public/_apis" +$vstmr = "https://vstmr.dev.azure.com/dnceng-public/public/_apis" +$pipelineDefinitionIds = @(83, 87) +$canonicalQuarantineLabel = "test-failure" +$workflowIdMarker = "" +$workflowCallMarker = "" +$trustedIssueActors = @("app/github-actions", "github-actions[bot]") +$minimumFailureBuilds = 2 +$minimumNegativeLogs = 1 +$excerptCap = 2000 +$rawLogCap = 12000 + +# Same secret-shaped patterns the production test-quarantine.md deterministic collector scrubs +# before surfacing captured CI text. Helix work-item upload steps can log a live Azure DevOps +# bearer JWT on failure; the raw evidence gathered here is untrusted input and must never leak +# a live credential into an uploaded artifact. +$secretPatterns = @( + [regex]'eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}' + [regex]'eyJ[A-Za-z0-9_-]{20,}' + [regex]'\bgh[pousr]_[A-Za-z0-9]{20,}\b' + [regex]'\bgithub_pat_[A-Za-z0-9_]{20,}\b' + [regex]'(?i)\bbearer\s+[A-Za-z0-9._~+/=-]{20,}' + [regex]'(?i)\bhttps?://[^/\s:@"]+:[^@\s/"]{6,}@' + [regex]'(?i)[?&]sig=[A-Za-z0-9%/+_=-]{20,}' + [regex]'(?i)\b(?:AccountKey|SharedAccessKey|AccessKey|Password|Pwd)=[^;\s"'']{12,}' + [regex]'[A-Za-z0-9][A-Za-z0-9+/=_-]{51,}' +) + +function ConvertTo-Redacted +{ + param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Value) + + $result = $Value + foreach ($pattern in $secretPatterns) + { + $result = $pattern.Replace($result, "[REDACTED]") + } + + return $result +} + +function Get-Sha256String +{ + param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Value) + + $bytes = [System.Text.Encoding]::UTF8.GetBytes($Value) + $hash = [System.Security.Cryptography.SHA256]::HashData($bytes) + + return [System.Convert]::ToHexString($hash).ToLowerInvariant() +} + +function Get-CappedExcerpt +{ + param( + [Parameter(Mandatory = $true)][AllowEmptyString()][string]$Value, + [int]$Cap = $excerptCap, + # Phrases (e.g. the quarantined test's fully qualified name) that must survive + # redaction verbatim. The "long high-entropy run" secret pattern intentionally + # has no notion of word structure -- a descriptive xUnit test name easily exceeds + # its 52-character threshold and would otherwise be redacted to "[REDACTED]", + # silently destroying the exact marker Evaluate-TestQuarantineKbeCandidate.ps1 + # needs to associate a failure line with this test. Each phrase is swapped for a + # private-use-area sentinel before redaction runs and restored immediately after. + [string[]]$ProtectedPhrases = @() + ) + + $working = $Value + $placeholders = [ordered]@{} + $tokenIndex = 0 + foreach ($phrase in $ProtectedPhrases) + { + if ([string]::IsNullOrEmpty($phrase) -or -not $working.Contains($phrase)) + { + continue + } + $token = "`u{E000}PROTECTED-$tokenIndex`u{E000}" + $placeholders[$token] = $phrase + $working = $working.Replace($phrase, $token) + $tokenIndex++ + } + + $redacted = ConvertTo-Redacted -Value $working + foreach ($token in $placeholders.Keys) + { + $redacted = $redacted.Replace($token, [string]$placeholders[$token]) + } + + # The evidence text this function builds always places the failed/passed-test marker line + # and the (already-extracted) signature first, so capping to the first $Cap characters here + # can only ever truncate the tail of a long stack trace -- never the marker or signature the + # evaluator's association window needs. See Assemble-EvidenceText below. + $normalized = [System.Text.RegularExpressions.Regex]::Replace($redacted, "[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "?") + if ($normalized.Length -gt $Cap) + { + $normalized = $normalized.Substring(0, $Cap) + } + + return $normalized +} + +function ConvertTo-Iso8601String +{ + # ConvertFrom-Json (and Invoke-RestMethod, which uses it internally) auto-converts + # ISO-8601-shaped JSON string values into [datetime] instances. A bare [string] cast + # on one of those then renders using the current culture (e.g. "08/22/2026 03:28:01"), + # silently dropping the 'Z' suffix and sub-second precision and producing a value that + # is no longer schema-valid 'date-time' text. Route every timestamp read from parsed + # JSON through this so the emitted dossier/candidate always carries a real ISO-8601 + # UTC string regardless of whether the runtime parsed it into a DateTime or left it + # as a string. + param([AllowNull()]$Value) + + if ($null -eq $Value) + { + return $null + } + if ($Value -is [datetime]) + { + return $Value.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ") + } + return [string]$Value +} + +function Test-HasProperty +{ + # `$obj.PSObject.Properties.Name -contains $name` throws "The property 'Name' cannot be + # found on this object" under Set-StrictMode -Version Latest whenever the object has zero + # properties (an empty JSON object `{}}`, which every fixture.json category not exercised by + # a given test/fixture legitimately is). Iterating .PSObject.Properties directly is safe for + # both the empty and non-empty case. + param([AllowNull()]$Object, [Parameter(Mandatory = $true)][string]$Name) + + if ($null -eq $Object) + { + return $false + } + foreach ($property in $Object.PSObject.Properties) + { + if ($property.Name -eq $Name) + { + return $true + } + } + return $false +} + +function Add-MissingEvidence +{ + param( + [Parameter(Mandatory = $true)] + [AllowEmptyCollection()] + [System.Collections.Generic.List[object]]$List, + [Parameter(Mandatory = $true)][string]$Kind, + [Parameter(Mandatory = $true)][string]$Detail + ) + + $null = $List.Add([ordered]@{ kind = $Kind; detail = $Detail }) +} + +# --------------------------------------------------------------------------- +# Fixture / live network abstraction. Fixture mode reads one consolidated JSON +# document; live mode calls the public GitHub/Azure DevOps REST APIs. +# --------------------------------------------------------------------------- + +$isFixtureMode = -not [string]::IsNullOrEmpty($FixtureRoot) +$fixture = $null +if ($isFixtureMode) +{ + $fixturePath = Join-Path $FixtureRoot "fixture.json" + if (-not (Test-Path -LiteralPath $fixturePath)) + { + throw "Fixture mode requested but '$fixturePath' does not exist." + } + $fixture = Get-Content -LiteralPath $fixturePath -Raw | ConvertFrom-Json -Depth 32 +} + +function Get-GitHubHeaders +{ + # A real bearer token, never a literal placeholder. Authenticated GitHub REST/Search calls + # get a materially higher rate limit (up to 30 search requests/minute vs. 10 unauthenticated) + # and are required in practice: a single run's four duplicate-search categories alone can + # exhaust the unauthenticated search quota. + $headers = @{ Accept = "application/vnd.github+json"; "User-Agent" = "aspnetcore-test-quarantine-kbe-shadow" } + if (-not [string]::IsNullOrEmpty($GitHubToken)) + { + $headers["Authorization"] = "Bearer $GitHubToken" + } + return $headers +} + +function Write-RateLimitDiagnostic +{ + # Non-blocking, informational only: surfaces the authenticated GitHub rate-limit headroom in + # the workflow log so a maintainer can see at a glance whether the token is actually being + # used and how much quota remains. Never fails the run if the header is absent. + param($ResponseHeaders) + + if ($null -eq $ResponseHeaders) + { + return + } + $remaining = $ResponseHeaders["X-RateLimit-Remaining"] + $limit = $ResponseHeaders["X-RateLimit-Limit"] + if ($remaining -and $limit) + { + Write-Host "GitHub API rate limit: $remaining/$limit remaining." + } +} + +function Get-GitHubIssue +{ + param([Parameter(Mandatory = $true)][int]$Number) + + if ($isFixtureMode) + { + if ([int]$fixture.issue.number -ne $Number) + { + throw "Fixture issue number $($fixture.issue.number) does not match requested issue $Number." + } + return $fixture.issue + } + + $headers = Get-GitHubHeaders + $responseHeaders = $null + $result = Invoke-RestMethod -Uri "https://api.github.com/repos/$Repository/issues/$Number" -Headers $headers -Method Get -TimeoutSec 30 -ResponseHeadersVariable responseHeaders + Write-RateLimitDiagnostic -ResponseHeaders $responseHeaders + return $result +} + +function Test-DispatchShaOnMainComparison +{ + param( + [Parameter(Mandatory = $true)]$Comparison, + [Parameter(Mandatory = $true)][string]$DispatchSha + ) + + return [string]$Comparison.merge_base_commit.sha -eq $DispatchSha -and + [string]$Comparison.status -in @("ahead", "identical") +} + +function Get-TrustedMainRefResult +{ + param([Parameter(Mandatory = $true)][string]$DispatchSha) + + if ($isFixtureMode) + { + if (Test-HasProperty -Object $fixture -Name "main_branch") + { + $currentMainSha = [string]$fixture.main_branch.sha + $isMember = if (Test-HasProperty -Object $fixture.main_branch -Name "contains_event_sha") + { + $fixtureComparison = if ([bool]$fixture.main_branch.contains_event_sha) + { + [ordered]@{ + status = "ahead" + merge_base_commit = [ordered]@{ sha = $DispatchSha } + } + } + else + { + [ordered]@{ + status = "diverged" + merge_base_commit = [ordered]@{ sha = $currentMainSha } + } + } + Test-DispatchShaOnMainComparison -Comparison $fixtureComparison -DispatchSha $DispatchSha + } + else + { + $DispatchSha.Equals($currentMainSha, [System.StringComparison]::OrdinalIgnoreCase) + } + return [ordered]@{ Checked = $true; CurrentMainSha = $currentMainSha; DispatchShaOnMain = $isMember } + } + return [ordered]@{ Checked = $false; CurrentMainSha = $null; DispatchShaOnMain = $true } + } + + try + { + $headers = Get-GitHubHeaders + $result = Invoke-RestMethod -Uri "https://api.github.com/repos/$Repository/commits/main" -Headers $headers -Method Get -TimeoutSec 30 + $currentMainSha = [string]$result.sha + if ($DispatchSha.Equals($currentMainSha, [System.StringComparison]::OrdinalIgnoreCase)) + { + return [ordered]@{ Checked = $true; CurrentMainSha = $currentMainSha; DispatchShaOnMain = $true } + } + + $comparison = Invoke-RestMethod -Uri "https://api.github.com/repos/$Repository/compare/$DispatchSha...$currentMainSha" -Headers $headers -Method Get -TimeoutSec 30 + $isMember = Test-DispatchShaOnMainComparison -Comparison $comparison -DispatchSha $DispatchSha + return [ordered]@{ Checked = $true; CurrentMainSha = $currentMainSha; DispatchShaOnMain = $isMember } + } + catch + { + return [ordered]@{ Checked = $true; CurrentMainSha = $null; DispatchShaOnMain = $false } + } +} + +function Get-BuildProperty +{ + param( + [Parameter(Mandatory = $true)]$Build, + [Parameter(Mandatory = $true)][string]$Name + ) + + if (Test-HasProperty -Object $Build -Name $Name) + { + return $Build.$Name + } + return $null +} + +function Get-AzdoBuildValidation +{ + param( + [Parameter(Mandatory = $true)]$Build, + [Parameter(Mandatory = $true)][int]$DefinitionId, + [Parameter(Mandatory = $true)][ValidateSet("failure", "negative")][string]$Role + ) + + $sourceBranch = [string](Get-BuildProperty -Build $Build -Name "sourceBranch") + $status = [string](Get-BuildProperty -Build $Build -Name "status") + $result = [string](Get-BuildProperty -Build $Build -Name "result") + $allowedResults = if ($Role -eq "failure") { @("failed", "partiallySucceeded") } else { @("succeeded") } + $reasons = [System.Collections.Generic.List[string]]::new() + + if ($DefinitionId -notin $pipelineDefinitionIds) + { + $reasons.Add("definition") + } + if ($sourceBranch -ne "refs/heads/main") + { + $reasons.Add("source-branch") + } + if ($status -ne "completed") + { + $reasons.Add("status") + } + if ($result -notin $allowedResults) + { + $reasons.Add("result") + } + + return [ordered]@{ + Valid = $reasons.Count -eq 0 + SourceBranch = $sourceBranch + Status = $status + Result = $result + Reasons = @($reasons) + } +} + +function Get-AzdoBuild +{ + param([Parameter(Mandatory = $true)][int]$BuildId) + + if ($isFixtureMode) + { + $key = [string]$BuildId + if (Test-HasProperty -Object $fixture.azdo_builds -Name $key) + { + return $fixture.azdo_builds.$key + } + return $null + } + + try + { + return Invoke-RestMethod -Uri "$ado/build/builds/${BuildId}?api-version=7.1" -Method Get -TimeoutSec 30 + } + catch + { + return $null + } +} + +function Merge-AzdoBuildLists +{ + # Pure, side-effect-free merge/dedupe used by the live branch of + # Get-AzdoRecurrenceCandidateBuilds below. Extracted so the merge/dedupe/cap semantics can be + # unit-tested directly with synthetic input, without any network access, independent of + # whichever live resultFilter values happen to be queried. + param( + [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$Lists, + [Parameter(Mandatory = $true)][int]$Cap + ) + + $merged = [ordered]@{} + foreach ($list in $Lists) + { + foreach ($build in @($list)) + { + $merged[[string]$build.id] = $build + } + } + + return @($merged.Values | Sort-Object -Property startTime -Descending | Select-Object -First $Cap) +} + +function Get-AzdoRecurrenceCandidateBuilds +{ + # Azure DevOps' `resultFilter` does not support a comma-separated multi-value combination + # (verified live: "resultFilter=failed,partiallySucceeded" silently behaves like a single, + # different filter, not their union). ASP.NET Core test failures routinely land in a + # `partiallySucceeded` build (confirmed for aspnetcore#68947's own cited build 1551326), so a + # `failed`-only query misses real recurrence evidence. Issue one request per result value and + # merge, deduping by build id, via Merge-AzdoBuildLists. + param([Parameter(Mandatory = $true)][int]$DefinitionId) + + if ($isFixtureMode) + { + $key = [string]$DefinitionId + if (Test-HasProperty -Object $fixture.recurrence_scan -Name $key) + { + return @($fixture.recurrence_scan.$key) + } + return @() + } + + $resultLists = [System.Collections.Generic.List[object]]::new() + foreach ($resultFilter in @("failed", "partiallySucceeded")) + { + try + { + $result = Invoke-RestMethod -Uri "$ado/build/builds?definitions=$DefinitionId&branchName=refs/heads/main&resultFilter=$resultFilter&`$top=$RecurrenceScanBuildCap&api-version=7.1" -Method Get -TimeoutSec 30 + $null = $resultLists.Add(@($result.value)) + } + catch + { + continue + } + } + + return Merge-AzdoBuildLists -Lists @($resultLists) -Cap $RecurrenceScanBuildCap +} + +function Get-AzdoNegativeBuildQueryUri +{ + param( + [Parameter(Mandatory = $true)][int]$DefinitionId, + [Parameter(Mandatory = $true)][System.DateTimeOffset]$MinimumStartTime, + [Parameter(Mandatory = $true)][System.DateTimeOffset]$MaximumStartTime, + [Parameter(Mandatory = $true)][int]$Cap + ) + + $minimumTime = [System.Uri]::EscapeDataString($MinimumStartTime.ToUniversalTime().ToString("O")) + $maximumTime = [System.Uri]::EscapeDataString($MaximumStartTime.ToUniversalTime().ToString("O")) + return "$ado/build/builds?definitions=$DefinitionId&branchName=refs/heads/main&resultFilter=succeeded&queryOrder=startTimeDescending&minTime=$minimumTime&maxTime=$maximumTime&`$top=$Cap&api-version=7.1" +} + +function Get-AzdoNegativeCandidateBuilds +{ + param( + [Parameter(Mandatory = $true)][int]$DefinitionId, + [Parameter(Mandatory = $true)][System.DateTimeOffset]$MinimumStartTime, + [Parameter(Mandatory = $true)][System.DateTimeOffset]$MaximumStartTime + ) + + if ($isFixtureMode) + { + $key = [string]$DefinitionId + if (Test-HasProperty -Object $fixture.negative_scan -Name $key) + { + return @($fixture.negative_scan.$key) + } + return @() + } + + try + { + $uri = Get-AzdoNegativeBuildQueryUri ` + -DefinitionId $DefinitionId ` + -MinimumStartTime $MinimumStartTime ` + -MaximumStartTime $MaximumStartTime ` + -Cap $RecurrenceScanBuildCap + $result = Invoke-RestMethod -Uri $uri -Method Get -TimeoutSec 30 + return @($result.value) + } + catch + { + return @() + } +} + +function Get-VstmrSummaryRows +{ + # The `resultsbyBuild` summary rows carry only identity + outcome (id, runId, + # automatedTestName, outcome) for ordinary xUnit tests -- verified live against + # aspnetcore#68947's own cited build: no comment/errorMessage/stackTrace field is present. + # Use this only to locate the (runId, resultId) pair; fetch Get-VstmrDetail for the actual + # error text. + param( + [Parameter(Mandatory = $true)][int]$BuildId, + [Parameter(Mandatory = $true)][string]$TestName + ) + + if ($isFixtureMode) + { + $key = "$BuildId" + if (Test-HasProperty -Object $fixture.vstmr_summary -Name $key) + { + return @($fixture.vstmr_summary.$key | Where-Object { [string]$_.automatedTestName -eq $TestName }) + } + return @() + } + + try + { + $result = Invoke-RestMethod -Uri "$vstmr/testresults/resultsbyBuild?buildId=$BuildId&api-version=7.1-preview.1" -Method Get -TimeoutSec 60 + return @($result.value | Where-Object { [string]$_.automatedTestName -eq $TestName }) + } + catch + { + return @() + } +} + +$vstmrDetailCache = @{} + +function Get-VstmrDetail +{ + # The authoritative source for a specific result's errorMessage/stackTrace (and, when + # present, Helix job/work-item coordinates via a `comment` field -- only ever populated on a + # Helix work item's own crash/'.WorkItemExecution' pseudo-test row, not on ordinary xUnit + # test rows). This is the "detailed VSTMR result" endpoint, distinct from resultsbyBuild. + param( + [Parameter(Mandatory = $true)][int]$RunId, + [Parameter(Mandatory = $true)][int]$ResultId + ) + + $cacheKey = "${RunId}:${ResultId}" + if ($vstmrDetailCache.ContainsKey($cacheKey)) + { + return $vstmrDetailCache[$cacheKey] + } + + $detail = if ($isFixtureMode) + { + if (Test-HasProperty -Object $fixture.vstmr_detail -Name $cacheKey) + { + $fixture.vstmr_detail.$cacheKey + } + else + { + $null + } + } + else + { + try + { + Invoke-RestMethod -Uri "$ado/test/Runs/$RunId/results/${ResultId}?api-version=7.1" -Method Get -TimeoutSec 30 + } + catch + { + $null + } + } + + $vstmrDetailCache[$cacheKey] = $detail + return $detail +} + +$vstmrRunCache = @{} + +function Get-VstmrRunName +{ + # The TestRun's `name` (e.g. "Quarantine-Mono-Linux-Release-xunit") is the only authoritative, + # cheaply-available signal for which platform/configuration leg a result ran on -- + # `buildConfiguration.platform`/`.flavor` are empty strings on every run observed live. + param([Parameter(Mandatory = $true)][int]$RunId) + + if ($vstmrRunCache.ContainsKey($RunId)) + { + return $vstmrRunCache[$RunId] + } + + $name = if ($isFixtureMode) + { + $key = "$RunId" + if (Test-HasProperty -Object $fixture.vstmr_runs -Name $key) + { + [string]$fixture.vstmr_runs.$key.name + } + else + { + $null + } + } + else + { + try + { + [string](Invoke-RestMethod -Uri "$ado/test/runs/${RunId}?api-version=7.1" -Method Get -TimeoutSec 30).name + } + catch + { + $null + } + } + + $vstmrRunCache[$RunId] = $name + return $name +} + +function Get-TestRunEnvironmentFromName +{ + param([AllowNull()][string]$RunName) + + $testRunIdentity = "unknown" + $platform = "unknown" + $configuration = "unknown" + if ([string]::IsNullOrEmpty($RunName)) + { + return [ordered]@{ TestRunIdentity = $testRunIdentity; Platform = $platform; Configuration = $configuration } + } + + $normalizedRunName = $RunName.Trim() + $normalizedRunName = [regex]::Replace($normalizedRunName, '(?i)((?:xunit|js|open))_[1-9][0-9]*$', '$1') + $hasKnownRunFamily = + $normalizedRunName -match '(?i)(?:^|[-.])(?:xunit|js|open)(?:$|[-.])' + if ($hasKnownRunFamily) + { + $testRunIdentity = $normalizedRunName.ToLowerInvariant() + } + + if ($normalizedRunName -match "(?i)\bwindows\b") { $platform = "Windows" } + elseif ($normalizedRunName -match "(?i)\b(?:linux|ubuntu)\b") { $platform = "Linux" } + elseif ($normalizedRunName -match "(?i)\b(?:macos|osx)\b") { $platform = "macOS" } + + if ($normalizedRunName -match "(?i)\bdebug\b") { $configuration = "Debug" } + elseif ($normalizedRunName -match "(?i)\brelease\b") { $configuration = "Release" } + elseif ($hasKnownRunFamily) { $configuration = "not-encoded" } + + return [ordered]@{ TestRunIdentity = $testRunIdentity; Platform = $platform; Configuration = $configuration } +} + +function Get-CheckRunsForSha +{ + param([Parameter(Mandatory = $true)][string]$Sha) + + if ($isFixtureMode) + { + if (Test-HasProperty -Object $fixture.check_runs -Name $Sha) + { + return @($fixture.check_runs.$Sha) + } + return @() + } + + try + { + $headers = Get-GitHubHeaders + $result = Invoke-RestMethod -Uri "https://api.github.com/repos/$Repository/commits/$Sha/check-runs?check_name=Build%20Insights&per_page=100" -Headers $headers -Method Get -TimeoutSec 30 + return @($result.check_runs) + } + catch + { + return @() + } +} + +function Search-GitHubIssues +{ + # Returns @{ Complete; Numbers; TotalCount }. `Complete` requires BOTH that GitHub reported + # incomplete_results=false AND that every matching item (per total_count) was actually + # retrieved across the paginated fetch -- a `total_count` larger than what a single + # per_page=100 page returns previously went unnoticed and was still labeled complete. + param([Parameter(Mandatory = $true)][string]$Query) + + if ($isFixtureMode) + { + return $null + } + + $headers = Get-GitHubHeaders + $numbers = [System.Collections.Generic.List[int]]::new() + $totalCount = 0 + $complete = $true + $encoded = [System.Uri]::EscapeDataString($Query) + + for ($page = 1; $page -le $DuplicateSearchMaxPages; $page++) + { + try + { + $responseHeaders = $null + $result = Invoke-RestMethod -Uri "https://api.github.com/search/issues?q=$encoded&per_page=$DuplicateSearchPageSize&page=$page" -Headers $headers -Method Get -TimeoutSec 30 -ResponseHeadersVariable responseHeaders + Write-RateLimitDiagnostic -ResponseHeaders $responseHeaders + } + catch + { + $complete = $false + break + } + + $totalCount = [int]$result.total_count + if ([bool]$result.incomplete_results) + { + $complete = $false + } + foreach ($item in @($result.items)) + { + $null = $numbers.Add([int]$item.number) + } + + if ($numbers.Count -ge $totalCount) + { + break + } + if ($page -eq $DuplicateSearchMaxPages -and $numbers.Count -lt $totalCount) + { + $complete = $false + } + } + + return [ordered]@{ Complete = $complete; Numbers = @($numbers); TotalCount = $totalCount } +} + +function Get-DuplicateCandidateText +{ + # Fetches the searched-up issue or PR's title+body so its exact test identity can be + # verified before it is ever treated as a validated duplicate. `/issues/{number}` is a + # unified GitHub endpoint that also resolves pull requests. + param([Parameter(Mandatory = $true)][int]$Number) + + if ($isFixtureMode) + { + $key = "$Number" + if (Test-HasProperty -Object $fixture.duplicate_candidate_text -Name $key) + { + return [string]$fixture.duplicate_candidate_text.$key + } + return $null + } + + try + { + $headers = Get-GitHubHeaders + $result = Invoke-RestMethod -Uri "https://api.github.com/repos/$Repository/issues/$Number" -Headers $headers -Method Get -TimeoutSec 30 + return "$($result.title)`n$($result.body)" + } + catch + { + return $null + } +} + +function Test-ContainsExactTestName +{ + param( + [Parameter(Mandatory = $true)][string]$Text, + [Parameter(Mandatory = $true)][string]$TestName + ) + + $escapedTestName = [System.Text.RegularExpressions.Regex]::Escape($TestName) + return [regex]::IsMatch( + $Text, + "(^|[^A-Za-z0-9_.+])$escapedTestName($|[^A-Za-z0-9_.+])", + [System.Text.RegularExpressions.RegexOptions]::CultureInvariant) +} + +function Get-DocumentedSignature +{ + param([Parameter(Mandatory = $true)][string]$Text) + + $match = [regex]::Match( + $Text, + '##\s*Error Message.*?```json\s*(.*?)\s*```', + [System.Text.RegularExpressions.RegexOptions]::Singleline -bor + [System.Text.RegularExpressions.RegexOptions]::IgnoreCase) + if (-not $match.Success) + { + return $null + } + + try + { + $document = $match.Groups[1].Value | ConvertFrom-Json -Depth 16 + } + catch + { + return $null + } + + $messageValues = @() + if (Test-HasProperty -Object $document -Name "ErrorMessage") + { + $messageValues = @( + $document.ErrorMessage | + ForEach-Object { [string]$_ } | + Where-Object { -not [string]::IsNullOrWhiteSpace($_) } + ) + } + if ($messageValues.Count -gt 0) + { + return [ordered]@{ Kind = "ErrorMessage"; Values = $messageValues } + } + + if ((Test-HasProperty -Object $document -Name "ErrorPattern") -and + -not [string]::IsNullOrWhiteSpace([string]$document.ErrorPattern)) + { + return [ordered]@{ Kind = "ErrorPattern"; Values = @([string]$document.ErrorPattern) } + } + + return $null +} + +function Test-DocumentedSignatureCompatibility +{ + param( + [Parameter(Mandatory = $true)]$Signature, + [Parameter(Mandatory = $true)][object[]]$FailureLogs, + [Parameter(Mandatory = $true)][string]$TestName, + [Parameter(Mandatory = $true)][string]$Root + ) + + if ($FailureLogs.Count -eq 0) + { + return $false + } + + $values = @($Signature.Values | ForEach-Object { [string]$_ }) + $regexes = @() + if ([string]$Signature.Kind -eq "ErrorPattern") + { + try + { + $regexOptions = [System.Text.RegularExpressions.RegexOptions]::Singleline -bor + [System.Text.RegularExpressions.RegexOptions]::IgnoreCase -bor + [System.Text.RegularExpressions.RegexOptions]::NonBacktracking + $regexes = @($values | ForEach-Object { + [System.Text.RegularExpressions.Regex]::new($_, $regexOptions, [System.TimeSpan]::FromMilliseconds(50)) + }) + } + catch + { + return $false + } + } + + $escapedTestName = [System.Text.RegularExpressions.Regex]::Escape($TestName) + $testNameMatcher = [System.Text.RegularExpressions.Regex]::new( + "(^|[^A-Za-z0-9_.+])$escapedTestName($|[^A-Za-z0-9_.+])", + [System.Text.RegularExpressions.RegexOptions]::CultureInvariant -bor + [System.Text.RegularExpressions.RegexOptions]::NonBacktracking, + [System.TimeSpan]::FromMilliseconds(50)) + + foreach ($log in $FailureLogs) + { + $path = Join-Path $Root ([string]$log.path) + $failedTestLines = [System.Collections.Generic.List[int]]::new() + $matchedLines = [System.Collections.Generic.List[int]]::new() + $patternIndex = 0 + $lineNumber = 0 + $matched = $false + + foreach ($line in [System.IO.File]::ReadLines($path)) + { + $lineNumber++ + $normalizedLine = $line -replace "^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z\s+", "" + $lineContainsTest = $testNameMatcher.IsMatch($normalizedLine) + $lineIndicatesFailure = + $normalizedLine -match "(?i)^\s*\[FAIL(?:ED)?\]\s+" -or + $normalizedLine -match "(?i)^\s*Failed\s+" -or + $normalizedLine -match "(?i)^\s*\[[^\]\r\n]+\]\s+.+\s+\[FAIL(?:ED)?\]\s*$" + if ($lineContainsTest -and $lineIndicatesFailure) + { + $failedTestLines.Add($lineNumber) + continue + } + + $matchedThisLine = $false + try + { + $matchedThisLine = if ([string]$Signature.Kind -eq "ErrorPattern") + { + $regexes[$patternIndex].IsMatch($line) + } + else + { + $line.IndexOf($values[$patternIndex], [System.StringComparison]::Ordinal) -ge 0 + } + } + catch [System.Text.RegularExpressions.RegexMatchTimeoutException] + { + return $false + } + + if ($matchedThisLine) + { + $matchedLines.Add($lineNumber) + if ($values.Count -eq 1) + { + $matched = $true + } + else + { + $patternIndex++ + $matched = $patternIndex -eq $values.Count + } + } + if ($matched) + { + break + } + } + + $associated = $matched -and @( + $matchedLines | + Where-Object { + $matchedLine = $_ + @($failedTestLines | Where-Object { [System.Math]::Abs($_ - $matchedLine) -le 50 }).Count -gt 0 + } + ).Count -gt 0 + if (-not $associated) + { + return $false + } + } + + return $true +} + +# --------------------------------------------------------------------------- +# Step 1: validate the canonical, open quarantine issue. A label or copied marker is not proof; +# require the trusted workflow actor, both markers, and consistent structured workflow-run metadata. +# --------------------------------------------------------------------------- + +$missingEvidence = [System.Collections.Generic.List[object]]::new() +$reasonCodes = [System.Collections.Generic.List[string]]::new() + +$issue = Get-GitHubIssue -Number $IssueNumber +$issueUrl = "https://github.com/$Repository/issues/$IssueNumber" +$issueLabels = @($issue.labels | ForEach-Object { if ($_ -is [string]) { $_ } else { [string]$_.name } }) +$issueState = [string]$issue.state +$issueBody = [string]$issue.body +$issueActor = if ((Test-HasProperty -Object $issue -Name "user") -and $null -ne $issue.user) +{ + [string]$issue.user.login +} +else +{ + $null +} +$hasWorkflowIdMarker = $issueBody.Contains($workflowIdMarker, [System.StringComparison]::Ordinal) +$hasWorkflowCallMarker = $issueBody.Contains($workflowCallMarker, [System.StringComparison]::Ordinal) +$hasWorkflowMarker = $hasWorkflowIdMarker -and $hasWorkflowCallMarker +$workflowMetadataMatch = [regex]::Match( + $issueBody, + '', + [System.Text.RegularExpressions.RegexOptions]::Singleline -bor + [System.Text.RegularExpressions.RegexOptions]::CultureInvariant) +$workflowRunId = $null +$hasWorkflowMetadata = $false +if ($workflowMetadataMatch.Success) +{ + $metadata = $workflowMetadataMatch.Groups[1].Value + $metadataIdMatch = [regex]::Match($metadata, '(?:^|,\s*)id:\s*([1-9][0-9]*)(?:,|$)') + $metadataRunMatch = [regex]::Match($metadata, '(?:^|,\s*)run:\s*https://github\.com/dotnet/aspnetcore/actions/runs/([1-9][0-9]*)(?:,|$)') + $hasExpectedWorkflowId = [regex]::IsMatch($metadata, '(?:^|,\s*)workflow_id:\s*test-quarantine(?:,|$)') + if ($metadataIdMatch.Success -and + $metadataRunMatch.Success -and + $metadataIdMatch.Groups[1].Value -eq $metadataRunMatch.Groups[1].Value -and + $hasExpectedWorkflowId) + { + $workflowRunId = [long]$metadataIdMatch.Groups[1].Value + $hasWorkflowMetadata = $true + } +} +$hasTrustedIssueActor = $issueActor -in $trustedIssueActors + +if ($issueLabels -notcontains $canonicalQuarantineLabel -or + -not $hasWorkflowMarker -or + -not $hasTrustedIssueActor -or + -not $hasWorkflowMetadata) +{ + $reasonCodes.Add("issue-not-canonical-quarantine") + if ($issueLabels -notcontains $canonicalQuarantineLabel) + { + Add-MissingEvidence -List $missingEvidence -Kind "quarantine-label" -Detail "Issue #$IssueNumber does not carry the canonical '$canonicalQuarantineLabel' label." + } + if (-not $hasWorkflowMarker) + { + Add-MissingEvidence -List $missingEvidence -Kind "quarantine-workflow-marker" -Detail "Issue #$IssueNumber body does not contain both trusted test-quarantine workflow markers." + } + if (-not $hasTrustedIssueActor) + { + Add-MissingEvidence -List $missingEvidence -Kind "issue-actor" -Detail "Issue #$IssueNumber actor '$issueActor' is not a trusted GitHub Actions workflow actor." + } + if (-not $hasWorkflowMetadata) + { + Add-MissingEvidence -List $missingEvidence -Kind "workflow-metadata" -Detail "Issue #$IssueNumber lacks consistent structured test-quarantine workflow metadata and dotnet/aspnetcore Actions run provenance." + } +} + +if ($issueState -ne "open") +{ + $reasonCodes.Add("issue-not-open") + Add-MissingEvidence -List $missingEvidence -Kind "issue-state" -Detail "Issue #$IssueNumber is '$issueState', not 'open'." +} + +# --------------------------------------------------------------------------- +# Step 2: deterministically parse the issue body for the test name, referenced builds, and (when +# unambiguous) a failure signature. Every quarantine issue produced by the production workflow +# carries a '## Failing Test(s)' section and at least one 'buildId=' link; both formats in use +# (the strict 50_test_failure.md template and the freeform '## Details' variant) satisfy these two +# invariants, so this parsing does not depend on section ordering. +# +# A '## Failing Test(s)' section can legitimately name more than one concrete test identity (e.g. +# aspnetcore#68724 names both a base test and its server-execution subclass override, and live +# data shows only the override actually failed while the base identity passed). Silently picking +# the first one risks binding evidence to the wrong identity. This collector fails closed unless +# exactly one concrete identity can be unambiguously selected; evaluating every listed identity +# independently is a reasonable extension left for a follow-up, since this PR targets one +# issue/one root cause at a time. +# --------------------------------------------------------------------------- + +$testName = $null +$failingTestMatch = [regex]::Match($issueBody, "##\s*Failing Test\(s\)\s*\r?\n(.*?)(?=\r?\n##\s|\z)", [System.Text.RegularExpressions.RegexOptions]::Singleline) +$distinctIdentities = @() +if ($failingTestMatch.Success) +{ + $distinctIdentities = @( + [regex]::Matches($failingTestMatch.Groups[1].Value, '`([^`]+)`') | + ForEach-Object { $_.Groups[1].Value.Trim() } | + Where-Object { $_.Length -ge 3 -and $_.Length -le 1024 -and $_ -notmatch "[\r\n]" } | + Select-Object -Unique + ) +} + +if ($distinctIdentities.Count -eq 0) +{ + $reasonCodes.Add("test-name-unresolvable") + Add-MissingEvidence -List $missingEvidence -Kind "test-name" -Detail "Could not deterministically extract any backtick-quoted fully qualified test name from '## Failing Test(s)'." +} +elseif ($distinctIdentities.Count -gt 1) +{ + $reasonCodes.Add("multiple-test-identities-unresolved") + Add-MissingEvidence -List $missingEvidence -Kind "test-name" -Detail "'## Failing Test(s)' names $($distinctIdentities.Count) distinct test identities ($($distinctIdentities -join '; ')); this collector requires exactly one unambiguous identity per run rather than guessing which one actually failed." +} +else +{ + $testName = $distinctIdentities[0] +} + +$citedBuildIds = @( + [regex]::Matches($issueBody, "buildId=(\d+)") | + ForEach-Object { [int]$_.Groups[1].Value } | + Select-Object -Unique +) + +if ($citedBuildIds.Count -eq 0) +{ + $reasonCodes.Add("build-reference-unresolvable") + Add-MissingEvidence -List $missingEvidence -Kind "build-reference" -Detail "No 'buildId=' reference found in the issue body." +} + +$manualSignatureProvided = -not [string]::IsNullOrWhiteSpace($Signature) +$extractedSignature = $null +$errorMessageMatch = [regex]::Match($issueBody, '##\s*Error Message\s*\r?\n```(?:text)?\r?\n(.*?)```', [System.Text.RegularExpressions.RegexOptions]::Singleline) +if ($errorMessageMatch.Success) +{ + $firstLine = ($errorMessageMatch.Groups[1].Value -split "\r?\n" | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | Select-Object -First 1) + if (-not [string]::IsNullOrWhiteSpace($firstLine)) + { + $extractedSignature = $firstLine.Trim() + } +} + +$effectiveSignature = if ($manualSignatureProvided) { $Signature.Trim() } else { $extractedSignature } +if ([string]::IsNullOrWhiteSpace($effectiveSignature) -or + $effectiveSignature.Length -lt 8 -or $effectiveSignature.Length -gt 2048 -or + $effectiveSignature -match "[\r\n]") +{ + $reasonCodes.Add("signature-extraction-ambiguous") + Add-MissingEvidence -List $missingEvidence -Kind "signature" -Detail "No deterministic '## Error Message' code block was found and no valid -Signature override was supplied." + $effectiveSignature = $null +} + +# --------------------------------------------------------------------------- +# Step 2.5: confirm the immutable workflow-dispatch ref/SHA is a member of main and is exactly the +# commit checked out for collection. Main may legitimately advance after dispatch, so equality +# with the current tip is sufficient but not required. +# --------------------------------------------------------------------------- + +$repoHeadSha = (& git -C $RepositoryRoot rev-parse HEAD).Trim() +$effectiveEventRef = if ([string]::IsNullOrWhiteSpace($EventRef) -and $isFixtureMode) { "refs/heads/main" } else { $EventRef } +$effectiveEventSha = if ([string]::IsNullOrWhiteSpace($EventSha) -and $isFixtureMode) { $repoHeadSha } else { $EventSha } +$eventRefIsMain = $effectiveEventRef -eq "refs/heads/main" +$eventShaIsValid = $effectiveEventSha -match "^[0-9a-f]{40}$" +$checkoutMatchesEventSha = + $eventShaIsValid -and + $repoHeadSha.Equals($effectiveEventSha, [System.StringComparison]::OrdinalIgnoreCase) + +if (-not $eventRefIsMain) +{ + $reasonCodes.Add("workflow-dispatch-ref-not-main") + Add-MissingEvidence -List $missingEvidence -Kind "repository-ref" -Detail "Workflow dispatch ref '$effectiveEventRef' is not exactly 'refs/heads/main'." +} +if (-not $checkoutMatchesEventSha) +{ + $reasonCodes.Add("checkout-sha-not-dispatch-sha") + Add-MissingEvidence -List $missingEvidence -Kind "repository-ref" -Detail "Checked-out commit $repoHeadSha does not match workflow dispatch SHA '$effectiveEventSha'." +} + +if (-not $eventShaIsValid) +{ + $currentMainSha = $null + $dispatchShaOnMain = $false +} +else +{ + $mainRefResult = Get-TrustedMainRefResult -DispatchSha $effectiveEventSha + if (-not [bool]$mainRefResult.Checked) + { + $currentMainSha = $effectiveEventSha + $dispatchShaOnMain = $true + } + else + { + $currentMainSha = $mainRefResult.CurrentMainSha + $dispatchShaOnMain = [bool]$mainRefResult.DispatchShaOnMain + if (-not $dispatchShaOnMain) + { + $reasonCodes.Add("repository-ref-not-main") + $mainDisplay = if ($currentMainSha) { $currentMainSha } else { "(lookup failed)" } + Add-MissingEvidence -List $missingEvidence -Kind "repository-ref" -Detail "Workflow dispatch SHA '$effectiveEventSha' could not be confirmed as identical to or an ancestor/member of current dotnet/aspnetcore main SHA $mainDisplay." + } + } +} + +# --------------------------------------------------------------------------- +# Step 3: resolve Azure DevOps build metadata for every issue-cited build. Builds whose metadata +# has aged out of Azure DevOps retention are recorded as not-found, never fabricated (see +# aspnetcore#68945). +# --------------------------------------------------------------------------- + +$retrievedUtc = [System.DateTimeOffset]::UtcNow.ToString("O") +$azdoBuildRecords = [System.Collections.Generic.List[object]]::new() +$resolvedBuilds = [System.Collections.Generic.List[object]]::new() + +foreach ($buildId in $citedBuildIds) +{ + $build = Get-AzdoBuild -BuildId $buildId + if ($null -eq $build) + { + $null = $azdoBuildRecords.Add([ordered]@{ + id = $buildId + found = $false + retrieved_utc = $retrievedUtc + source = "issue-body-reference" + note = "Build metadata was not retrievable; it may have aged out of Azure DevOps retention." + }) + Add-MissingEvidence -List $missingEvidence -Kind "azdo-build" -Detail "Build $buildId metadata could not be retrieved." + continue + } + + $definitionId = [int]$build.definition.id + $sourceVersion = [string]$build.sourceVersion + $validation = Get-AzdoBuildValidation -Build $build -DefinitionId $definitionId -Role "failure" + $record = [ordered]@{ + id = $buildId + found = $true + retrieved_utc = $retrievedUtc + source = "issue-body-reference" + definition_id = $definitionId + source_branch = $validation.SourceBranch + source_version = $sourceVersion + started_utc = ConvertTo-Iso8601String -Value $build.startTime + finished_utc = ConvertTo-Iso8601String -Value $build.finishTime + status = $validation.Status + result = $validation.Result + } + $null = $azdoBuildRecords.Add($record) + if (-not [bool]$validation.Valid) + { + foreach ($invalidDimension in $validation.Reasons) + { + $reasonCode = switch ($invalidDimension) + { + "definition" { "azdo-build-definition-not-allowed"; break } + "source-branch" { "azdo-build-source-branch-not-main"; break } + "status" { "azdo-build-not-completed"; break } + default { "azdo-build-result-incompatible" } + } + $reasonCodes.Add($reasonCode) + } + Add-MissingEvidence -List $missingEvidence -Kind "azdo-build" -Detail "Cited build $buildId is ineligible: definition=$definitionId, sourceBranch='$($validation.SourceBranch)', status='$($validation.Status)', result='$($validation.Result)'." + continue + } + $null = $resolvedBuilds.Add(($record + @{ intended_role = "failure" })) +} + +# --------------------------------------------------------------------------- +# Step 4: if fewer than two distinct resolved builds are available, perform a capped +# supplementary recurrence scan across the same pipeline definition(s) on 'main' -- direct raw +# AzDO/VSTMR evidence. Build Insights is corroborating only and cannot establish exact recurrence. +# Signature matching +# uses ordinal substring containment, never `-like`/`-notlike`: a literal ErrorMessage containing +# `*`, `?`, or `[` would otherwise be misinterpreted as a wildcard pattern instead of literal text. +# --------------------------------------------------------------------------- + +function Test-SignatureMatch +{ + param( + [Parameter(Mandatory = $true)][AllowEmptyString()][string]$Haystack, + [Parameter(Mandatory = $true)][string]$Signature + ) + + return $Haystack.Contains($Signature, [System.StringComparison]::Ordinal) +} + +function Get-MatchingFailureDetail +{ + # Finds the first summary row for $BuildId/$TestName whose VSTMR detail errorMessage+stackTrace + # contains $Signature (ordinal substring). Returns $null when no such row exists. + param( + [Parameter(Mandatory = $true)]$BuildId, + [Parameter(Mandatory = $true)][string]$TestName, + [Parameter(Mandatory = $true)][string]$Signature + ) + + foreach ($row in @(Get-VstmrSummaryRows -BuildId $BuildId -TestName $TestName)) + { + if ([string]$row.outcome -ne "Failed") + { + continue + } + $detail = Get-VstmrDetail -RunId ([int]$row.runId) -ResultId ([int]$row.id) + if ($null -eq $detail) + { + continue + } + $haystack = "$($detail.errorMessage) $($detail.stackTrace)" + if (Test-SignatureMatch -Haystack $haystack -Signature $Signature) + { + return $row + } + } + + return $null +} + +if ($resolvedBuilds.Count -lt $minimumFailureBuilds -and $null -ne $testName -and $null -ne $effectiveSignature) +{ + $scanDefinitionIds = if ($resolvedBuilds.Count -gt 0) + { + @($resolvedBuilds | ForEach-Object { $_.definition_id } | Select-Object -Unique) + } + else + { + $pipelineDefinitionIds + } + + foreach ($definitionId in $scanDefinitionIds) + { + if ($resolvedBuilds.Count -ge $minimumFailureBuilds) + { + break + } + + $candidates = Get-AzdoRecurrenceCandidateBuilds -DefinitionId $definitionId + foreach ($candidate in $candidates) + { + if ($resolvedBuilds.Count -ge $minimumFailureBuilds) + { + break + } + + $candidateId = [int]$candidate.id + if ($citedBuildIds -contains $candidateId) + { + continue + } + + $candidateDefinitionId = if (Test-HasProperty -Object $candidate -Name "definition") { [int]$candidate.definition.id } else { $definitionId } + $validation = Get-AzdoBuildValidation -Build $candidate -DefinitionId $candidateDefinitionId -Role "failure" + $record = [ordered]@{ + id = $candidateId + found = $true + retrieved_utc = $retrievedUtc + source = "recurrence-scan" + definition_id = $candidateDefinitionId + source_branch = $validation.SourceBranch + source_version = [string]$candidate.sourceVersion + started_utc = ConvertTo-Iso8601String -Value $candidate.startTime + finished_utc = ConvertTo-Iso8601String -Value $candidate.finishTime + status = $validation.Status + result = $validation.Result + } + $null = $azdoBuildRecords.Add($record) + if (-not [bool]$validation.Valid) + { + continue + } + + $matchingRow = Get-MatchingFailureDetail -BuildId $candidateId -TestName $testName -Signature $effectiveSignature + if ($null -eq $matchingRow) + { + continue + } + + $null = $resolvedBuilds.Add(($record + @{ intended_role = "failure" })) + } + } +} + +if ($null -ne $testName -and $null -ne $effectiveSignature -and $resolvedBuilds.Count -lt $minimumFailureBuilds) +{ + $reasonCodes.Add("recurrence-single-build-only") + Add-MissingEvidence -List $missingEvidence -Kind "recurrence" -Detail "Only $($resolvedBuilds.Count) distinct build(s) with matching failure evidence were found; at least $minimumFailureBuilds are required and none could be added by the supplementary recurrence scan." +} + +# --------------------------------------------------------------------------- +# Step 4.5: gather at least one authoritative Passed occurrence of the same +# test on the same pipeline(s). This is what lets the evaluator confirm the failure is not a +# consistent regression -- Skipped results may be useful context but never satisfy this gate. +# A missing pass is recorded as insufficient evidence, never +# inferred as a pass. +# --------------------------------------------------------------------------- + +$negativeBuilds = [System.Collections.Generic.List[object]]::new() +$negativeBuildCap = [System.Math]::Max(0, [System.Math]::Min($RecurrenceScanBuildCap, 30 - $resolvedBuilds.Count)) +$failureWindowBuilds = @( + if ($null -ne $testName -and $null -ne $effectiveSignature) + { + $resolvedBuilds | + Where-Object { + $null -ne (Get-MatchingFailureDetail -BuildId $_.id -TestName $testName -Signature $effectiveSignature) + } + } +) +$minimumFailureStartTime = if ($failureWindowBuilds.Count -gt 0) +{ + @($failureWindowBuilds | ForEach-Object { + [System.DateTimeOffset]::Parse([string]$_.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) + } | Sort-Object)[0] +} +else +{ + $null +} +$maximumFailureStartTime = if ($failureWindowBuilds.Count -gt 0) +{ + @($failureWindowBuilds | ForEach-Object { + [System.DateTimeOffset]::Parse([string]$_.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) + } | Sort-Object -Descending)[0] +} +else +{ + $null +} + +if ($null -ne $testName -and + $null -ne $effectiveSignature -and + $null -ne $minimumFailureStartTime -and + $null -ne $maximumFailureStartTime) +{ + $negativeScanDefinitionIds = @($resolvedBuilds | ForEach-Object { $_.definition_id } | Select-Object -Unique) + if ($negativeScanDefinitionIds.Count -eq 0) + { + $negativeScanDefinitionIds = $pipelineDefinitionIds + } + + foreach ($definitionId in $negativeScanDefinitionIds) + { + if ($negativeBuilds.Count -ge $negativeBuildCap) + { + break + } + + $candidates = Get-AzdoNegativeCandidateBuilds ` + -DefinitionId $definitionId ` + -MinimumStartTime $minimumFailureStartTime ` + -MaximumStartTime $maximumFailureStartTime + foreach ($candidate in $candidates) + { + if ($negativeBuilds.Count -ge $negativeBuildCap) + { + break + } + + $candidateId = [int]$candidate.id + $candidateDefinitionId = if (Test-HasProperty -Object $candidate -Name "definition") { [int]$candidate.definition.id } else { $definitionId } + $validation = Get-AzdoBuildValidation -Build $candidate -DefinitionId $candidateDefinitionId -Role "negative" + $record = [ordered]@{ + id = $candidateId + found = $true + retrieved_utc = $retrievedUtc + source = "negative-scan" + definition_id = $candidateDefinitionId + source_branch = $validation.SourceBranch + source_version = [string]$candidate.sourceVersion + started_utc = ConvertTo-Iso8601String -Value $candidate.startTime + finished_utc = ConvertTo-Iso8601String -Value $candidate.finishTime + status = $validation.Status + result = $validation.Result + } + $null = $azdoBuildRecords.Add($record) + if (-not [bool]$validation.Valid) + { + continue + } + + $rows = @(Get-VstmrSummaryRows -BuildId $candidateId -TestName $testName | Where-Object { [string]$_.outcome -eq "Passed" }) + if ($rows.Count -eq 0) + { + continue + } + + $null = $negativeBuilds.Add(($record + @{ intended_role = "negative" })) + } + } +} + +# --------------------------------------------------------------------------- +# Step 5: for each resolved build, materialize authoritative raw evidence from the VSTMR detail +# result -- never a blindly-capped raw console log. Helix job/work-item coordinates are recorded +# as metadata only when a work item's own crash/'.WorkItemExecution' row happens to carry a +# `comment` field (rare for ordinary xUnit tests, confirmed live); otherwise `helix_unavailable` is +# recorded explicitly and the VSTMR detail text remains authoritative on its own, per aspnetcore's +# real API shape. Platform/configuration are parsed from the authoritative TestRun name, never +# fabricated; "unknown" is recorded when no recognized token is present. +# --------------------------------------------------------------------------- + +[System.IO.Directory]::CreateDirectory($EvidenceRoot) | Out-Null + +$rawEvidenceRecords = [System.Collections.Generic.List[object]]::new() +$rawLogs = [System.Collections.Generic.List[object]]::new() +$failureBuildIdSet = [System.Collections.Generic.HashSet[int]]::new() +$materializedFailureOccurrences = [System.Collections.Generic.List[object]]::new() +$eligiblePassedBuildIdsDuringCollection = [System.Collections.Generic.HashSet[int]]::new() +$evidenceIndex = 0 + +$evidenceBuilds = @($resolvedBuilds) + @($negativeBuilds) +foreach ($build in $evidenceBuilds) +{ + if ($null -eq $testName) + { + break + } + + $role = [string]$build.intended_role + if ($role -eq "negative" -and $eligiblePassedBuildIdsDuringCollection.Count -ge $minimumNegativeLogs) + { + break + } + + $expectedOutcome = if ($role -eq "failure") { @("Failed") } else { @("Passed") } + $matchedRow = $null + $matchedDetail = $null + $selectedRunName = $null + $selectedPlatformConfiguration = $null + $fallbackRow = $null + $fallbackDetail = $null + + foreach ($row in @(Get-VstmrSummaryRows -BuildId $build.id -TestName $testName)) + { + if ([string]$row.outcome -notin $expectedOutcome) + { + continue + } + $detail = Get-VstmrDetail -RunId ([int]$row.runId) -ResultId ([int]$row.id) + if ($null -eq $detail) + { + continue + } + if ($role -eq "negative") + { + if ($null -eq $fallbackRow) + { + $fallbackRow = $row + $fallbackDetail = $detail + } + + $candidateRunName = Get-VstmrRunName -RunId ([int]$row.runId) + $candidatePlatformConfiguration = Get-TestRunEnvironmentFromName -RunName $candidateRunName + $candidateEnvironmentKey = "$($build.definition_id)|$($candidatePlatformConfiguration.TestRunIdentity)|$($candidatePlatformConfiguration.Platform)|$($candidatePlatformConfiguration.Configuration)" + $candidateStartedUtc = [System.DateTimeOffset]::Parse([string]$build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) + $matchingFailureOccurrences = @($materializedFailureOccurrences | Where-Object { $_.EnvironmentKey -eq $candidateEnvironmentKey }) + $hasFailureBefore = @($matchingFailureOccurrences | Where-Object { $_.StartedUtc -lt $candidateStartedUtc }).Count -gt 0 + $hasFailureAfter = @($matchingFailureOccurrences | Where-Object { $_.StartedUtc -gt $candidateStartedUtc }).Count -gt 0 + if ($hasFailureBefore -and $hasFailureAfter) + { + $matchedRow = $row + $matchedDetail = $detail + $selectedRunName = $candidateRunName + $selectedPlatformConfiguration = $candidatePlatformConfiguration + break + } + continue + } + if ($role -eq "failure" -and $null -ne $effectiveSignature) + { + $haystack = "$($detail.errorMessage) $($detail.stackTrace)" + if (-not (Test-SignatureMatch -Haystack $haystack -Signature $effectiveSignature)) + { + continue + } + } + $matchedRow = $row + $matchedDetail = $detail + break + } + if ($role -eq "negative" -and $null -eq $matchedRow -and $null -ne $fallbackRow) + { + $matchedRow = $fallbackRow + $matchedDetail = $fallbackDetail + } + + if ($null -eq $matchedRow -or $null -eq $matchedDetail) + { + $null = $rawEvidenceRecords.Add([ordered]@{ + build_id = $build.id + role = $role + found = $false + captured_utc = $retrievedUtc + note = "No VSTMR result for build $($build.id) matched the expected outcome/signature for this test." + }) + Add-MissingEvidence -List $missingEvidence -Kind "vstmr-evidence" -Detail "Build $($build.id): no matching, retrievable VSTMR result detail." + continue + } + + $runId = [int]$matchedRow.runId + $resultId = [int]$matchedRow.id + + $helixJob = $null + $helixWorkItem = $null + if ((Test-HasProperty -Object $matchedDetail -Name "comment") -and -not [string]::IsNullOrEmpty($matchedDetail.comment)) + { + $commentObject = $matchedDetail.comment | ConvertFrom-Json -ErrorAction SilentlyContinue + if ($null -ne $commentObject -and (Test-HasProperty -Object $commentObject -Name "HelixJobId")) + { + $helixJob = [string]$commentObject.HelixJobId + $helixWorkItem = [string]$commentObject.HelixWorkItemName + } + } + $helixUnavailable = [string]::IsNullOrEmpty($helixJob) -or [string]::IsNullOrEmpty($helixWorkItem) + + $runName = if ($null -ne $selectedRunName) { $selectedRunName } else { Get-VstmrRunName -RunId $runId } + $platformConfiguration = if ($null -ne $selectedPlatformConfiguration) + { + $selectedPlatformConfiguration + } + else + { + Get-TestRunEnvironmentFromName -RunName $runName + } + if ($platformConfiguration.Platform -eq "unknown") + { + $reasonCodes.Add("evidence-platform-unknown") + Add-MissingEvidence -List $missingEvidence -Kind "environment" -Detail "Build $($build.id) $role evidence has unknown platform from TestRun '$runName'." + } + if ($platformConfiguration.Configuration -eq "unknown") + { + $reasonCodes.Add("evidence-configuration-unknown") + Add-MissingEvidence -List $missingEvidence -Kind "environment" -Detail "Build $($build.id) $role evidence has unknown configuration from TestRun '$runName'." + } + if ($platformConfiguration.TestRunIdentity -eq "unknown") + { + $reasonCodes.Add("evidence-test-run-identity-unknown") + Add-MissingEvidence -List $missingEvidence -Kind "environment" -Detail "Build $($build.id) $role evidence has no canonical TestRun identity from '$runName'." + } + + $evidenceIndex += 1 + $fileName = "issue-$IssueNumber-build-$($build.id)-$role.log" + $evidencePath = Join-Path $EvidenceRoot $fileName + + # The marker line (which Evaluate-TestQuarantineKbeCandidate.ps1 requires to associate a + # signature match with the declared test) and, for failures, the already-extracted signature + # are always placed first -- capping to $rawLogCap can then only ever truncate the tail of a + # long stack trace, never the lines the evaluator actually needs. + $markerLine = if ($role -eq "failure") { "Failed $testName [reported by VSTMR result $resultId]" } else { "$([string]$matchedRow.outcome) $testName [reported by VSTMR result $resultId]" } + $bodyText = if ($role -eq "failure") + { + "$($matchedDetail.errorMessage)`n$($matchedDetail.stackTrace)" + } + else + { + "(no error: VSTMR outcome was $([string]$matchedRow.outcome))" + } + # Build id is appended defensively so two builds that legitimately share identical evidence + # text (as small synthetic fixtures sometimes do) never collide to the same content hash. + $rawContent = "$markerLine`n$bodyText`n(observed in build $($build.id))" + $cappedContent = Get-CappedExcerpt -Value $rawContent -Cap $rawLogCap -ProtectedPhrases @($testName) + [System.IO.File]::WriteAllText($evidencePath, $cappedContent) + $sha256 = (Get-FileHash -LiteralPath $evidencePath -Algorithm SHA256).Hash.ToLowerInvariant() + + $evidenceRecord = [ordered]@{ + build_id = $build.id + role = $role + kind = "vstmr-detail" + run_id = $runId + result_id = $resultId + helix_unavailable = $helixUnavailable + test_run_identity = $platformConfiguration.TestRunIdentity + platform = $platformConfiguration.Platform + configuration = $platformConfiguration.Configuration + found = $true + captured_utc = $retrievedUtc + sha256 = $sha256 + evidence_path = $fileName + } + if (-not $helixUnavailable) + { + $evidenceRecord["helix_job"] = $helixJob + $evidenceRecord["helix_workitem"] = $helixWorkItem + } + $null = $rawEvidenceRecords.Add($evidenceRecord) + + if ($role -eq "failure") + { + $null = $failureBuildIdSet.Add([int]$build.id) + if ($platformConfiguration.TestRunIdentity -ne "unknown" -and + $platformConfiguration.Platform -ne "unknown" -and + $platformConfiguration.Configuration -ne "unknown") + { + $null = $materializedFailureOccurrences.Add([ordered]@{ + EnvironmentKey = "$($build.definition_id)|$($platformConfiguration.TestRunIdentity)|$($platformConfiguration.Platform)|$($platformConfiguration.Configuration)" + StartedUtc = [System.DateTimeOffset]::Parse([string]$build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) + }) + } + } + $outcomeValue = switch ([string]$matchedRow.outcome) + { + "Failed" { "failed"; break } + "Passed" { "passed"; break } + default { "skipped" } + } + + $null = $rawLogs.Add([ordered]@{ + id = "evidence-$evidenceIndex" + role = $role + outcome = $outcomeValue + path = $fileName + source_url = "https://dev.azure.com/dnceng-public/public/_build/results?buildId=$($build.id)&view=results" + sha256 = $sha256 + build = [ordered]@{ + id = [int]$build.id + pipeline_definition_id = [int]$build.definition_id + source_branch = [string]$build.source_branch + source_version = [string]$build.source_version + started_utc = [string]$build.started_utc + status = [string]$build.status + result = [string]$build.result + test_run_identity = $platformConfiguration.TestRunIdentity + platform = $platformConfiguration.Platform + configuration = $platformConfiguration.Configuration + } + }) + + if ($role -eq "negative") + { + $passStartedUtc = [System.DateTimeOffset]::Parse([string]$build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) + $passEnvironmentKey = "$($build.definition_id)|$($platformConfiguration.TestRunIdentity)|$($platformConfiguration.Platform)|$($platformConfiguration.Configuration)" + $matchingFailureOccurrences = @($materializedFailureOccurrences | Where-Object { $_.EnvironmentKey -eq $passEnvironmentKey }) + $hasFailureBefore = @($matchingFailureOccurrences | Where-Object { $_.StartedUtc -lt $passStartedUtc }).Count -gt 0 + $hasFailureAfter = @($matchingFailureOccurrences | Where-Object { $_.StartedUtc -gt $passStartedUtc }).Count -gt 0 + if ($hasFailureBefore -and $hasFailureAfter) + { + $null = $eligiblePassedBuildIdsDuringCollection.Add([int]$build.id) + } + } +} + +if ($null -ne $testName -and $failureBuildIdSet.Count -lt $minimumFailureBuilds) +{ + if ($reasonCodes -notcontains "recurrence-single-build-only") + { + $reasonCodes.Add("raw-evidence-insufficient") + Add-MissingEvidence -List $missingEvidence -Kind "raw-evidence" -Detail "Only $($failureBuildIdSet.Count) distinct build(s) produced retrievable failure evidence; at least $minimumFailureBuilds are required." + } +} + +$failureLogsForPassEligibility = @($rawLogs | Where-Object { $_.role -eq "failure" }) +$passedLogsForEligibility = @($rawLogs | Where-Object { $_.role -eq "negative" -and $_.outcome -eq "passed" }) +$eligiblePassedBuildIds = [System.Collections.Generic.HashSet[int]]::new() +$passesWithMatchingEnvironment = 0 +foreach ($passLog in $passedLogsForEligibility) +{ + $passStartedUtc = [System.DateTimeOffset]::Parse([string]$passLog.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) + $matchingFailureLogs = @( + $failureLogsForPassEligibility | + Where-Object { + [int]$_.build.pipeline_definition_id -eq [int]$passLog.build.pipeline_definition_id -and + [string]$_.build.test_run_identity -eq [string]$passLog.build.test_run_identity -and + [string]$_.build.platform -eq [string]$passLog.build.platform -and + [string]$_.build.configuration -eq [string]$passLog.build.configuration + } + ) + if ($matchingFailureLogs.Count -eq 0) + { + continue + } + + $passesWithMatchingEnvironment++ + $hasFailureBefore = @($matchingFailureLogs | Where-Object { + [System.DateTimeOffset]::Parse([string]$_.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) -lt $passStartedUtc + }).Count -gt 0 + $hasFailureAfter = @($matchingFailureLogs | Where-Object { + [System.DateTimeOffset]::Parse([string]$_.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) -gt $passStartedUtc + }).Count -gt 0 + if ($hasFailureBefore -and $hasFailureAfter) + { + $null = $eligiblePassedBuildIds.Add([int]$passLog.build.id) + } +} + +if ($null -ne $testName -and $eligiblePassedBuildIds.Count -lt $minimumNegativeLogs) +{ + $reasonCodes.Add("raw-evidence-insufficient") + if ($failureLogsForPassEligibility.Count -gt 0 -and + $passedLogsForEligibility.Count -gt 0 -and + $passesWithMatchingEnvironment -eq 0) + { + $reasonCodes.Add("passed-evidence-environment-mismatch") + Add-MissingEvidence -List $missingEvidence -Kind "pass-evidence" -Detail "No authoritative Passed occurrence shared pipeline definition, canonical TestRun identity, platform, and configuration with any collected failure." + } + elseif ($passesWithMatchingEnvironment -gt 0) + { + $reasonCodes.Add("passed-evidence-not-interleaved") + Add-MissingEvidence -List $missingEvidence -Kind "pass-evidence" -Detail "No environment-matched Passed occurrence was strictly between an earlier and a later authoritative failure." + } + else + { + Add-MissingEvidence -List $missingEvidence -Kind "raw-evidence" -Detail "No retrievable authoritative Passed evidence was found." + } +} + +# --------------------------------------------------------------------------- +# Step 6: fetch Build Insights check-run snapshots. Advisory/corroborating only: recorded +# regardless of outcome, and a missing or generic snapshot never overrides raw evidence gathered +# above. `exact_test_referenced` requires the FULL fully-qualified test name, never a bare method +# name (which commonly collides with unrelated tests); `known_issue_referenced` requires a +# concrete dotnet/aspnetcore issue number/URL near the phrase, not the bare phrase alone (a +# heading or table column label reading "Known Issue" with no associated reference must not set +# this true). +# --------------------------------------------------------------------------- + +$buildInsightsSnapshots = [System.Collections.Generic.List[object]]::new() +$distinctShas = @($resolvedBuilds | ForEach-Object { $_.source_version } | Where-Object { $_ } | Select-Object -Unique) +$corroboratingContext = [System.Collections.Generic.List[object]]::new() + +foreach ($sha in $distinctShas) +{ + $checkRuns = Get-CheckRunsForSha -Sha $sha + $buildInsights = @( + $checkRuns | + Where-Object { + if ([string]$_.name -ne "Build Insights") + { + return $false + } + $appSlug = if ((Test-HasProperty -Object $_ -Name "app") -and + $null -ne $_.app -and + (Test-HasProperty -Object $_.app -Name "slug")) + { + [string]$_.app.slug + } + else + { + $null + } + return [string]::IsNullOrEmpty($appSlug) -or $appSlug -eq "build-insights" + } | + Select-Object -First 1 + ) + + if ($buildInsights.Count -eq 0) + { + $null = $buildInsightsSnapshots.Add([ordered]@{ + source_version = $sha + found = $false + retrieved_utc = $retrievedUtc + exact_test_referenced = $false + short_name_referenced = $false + known_issue_referenced = $false + known_issue_numbers = @() + }) + continue + } + $buildInsights = $buildInsights[0] + + $text = [string]$buildInsights.output.text + $textSha256 = Get-Sha256String -Value $text + $shortMethodName = if ($testName) { ($testName -split '\.')[-1] } else { $null } + $exactTestReferenced = ($null -ne $testName) -and $text.Contains($testName, [System.StringComparison]::Ordinal) + $shortNameReferenced = (-not $exactTestReferenced) -and ($null -ne $shortMethodName) -and $text.Contains($shortMethodName, [System.StringComparison]::Ordinal) + + $knownIssueNumbers = [System.Collections.Generic.List[int]]::new() + foreach ($phraseMatch in [regex]::Matches($text, "(?i)known issue")) + { + $windowStart = $phraseMatch.Index + $windowLength = [System.Math]::Min(200, $text.Length - $windowStart) + $window = $text.Substring($windowStart, $windowLength) + foreach ($numberMatch in [regex]::Matches($window, "dotnet/aspnetcore(?:#|/issues/)(\d+)|(?]*?)\s*-->') + $snapshotId = if ($snapshotIdMatch.Success) { $snapshotIdMatch.Groups[1].Value.Trim() } else { $null } + $appSlug = if ((Test-HasProperty -Object $buildInsights -Name "app") -and + $null -ne $buildInsights.app -and + (Test-HasProperty -Object $buildInsights.app -Name "slug")) + { + [string]$buildInsights.app.slug + } + else + { + $null + } + $detailsUrl = if (Test-HasProperty -Object $buildInsights -Name "details_url") + { + [string]$buildInsights.details_url + } + else + { + $null + } + + $null = $buildInsightsSnapshots.Add([ordered]@{ + source_version = $sha + found = $true + retrieved_utc = $retrievedUtc + check_id = [long]$buildInsights.id + app_slug = $appSlug + conclusion = [string]$buildInsights.conclusion + title = Get-CappedExcerpt -Value ([string]$buildInsights.output.title) -Cap 512 + text_sha256 = $textSha256 + text_excerpt = Get-CappedExcerpt -Value $text -Cap $excerptCap + details_url = $detailsUrl + html_url = [string]$buildInsights.html_url + snapshot_id = $snapshotId + exact_test_referenced = $exactTestReferenced + short_name_referenced = $shortNameReferenced + known_issue_referenced = $knownIssueNumbers.Count -gt 0 + known_issue_numbers = $knownIssueNumbers + }) + $corroboratingUrl = if (-not [string]::IsNullOrWhiteSpace($detailsUrl)) + { + $detailsUrl + } + else + { + [string]$buildInsights.html_url + } + $null = $corroboratingContext.Add([ordered]@{ source = "build-insights"; url = $corroboratingUrl }) +} + +# --------------------------------------------------------------------------- +# Step 7: categorized duplicate KBE / fix-PR search. Search hits are discovery only. Existing KBEs +# require the exact FQN and a documented ErrorMessage/ErrorPattern that matches every authoritative +# failure log with the evaluator's semantics. Fix PRs remain unvalidated until a later collector can +# prove closing-link and changed-file relevance. A failed candidate-detail fetch makes the query +# incomplete. +# --------------------------------------------------------------------------- + +$shortName = if ($testName) { ($testName -split '\.')[-1] } else { $IssueNumber.ToString() } +$recentWindowDate = [System.DateTimeOffset]::UtcNow.AddDays(-$DuplicateSearchWindowDays).ToString("yyyy-MM-dd") +$duplicateQueries = @( + @{ category = "open-kbe"; query = "repo:$Repository is:issue is:open label:`"Known Build Error`" $shortName" } + @{ category = "recently-closed-kbe"; query = "repo:$Repository is:issue is:closed closed:>=$recentWindowDate label:`"Known Build Error`" $shortName" } + @{ category = "open-fix-pr"; query = "repo:$Repository is:pr is:open $shortName" } + @{ category = "recently-merged-fix-pr"; query = "repo:$Repository is:pr is:merged merged:>=$recentWindowDate $shortName" } +) + +$duplicateQueryResults = [System.Collections.Generic.List[object]]::new() +$duplicateReferences = [System.Collections.Generic.List[string]]::new() +$unvalidatedCandidates = [System.Collections.Generic.List[object]]::new() +$kbeNumbers = [System.Collections.Generic.List[int]]::new() +$allQueriesComplete = $true + +foreach ($q in $duplicateQueries) +{ + $searchResult = if ($isFixtureMode) + { + $categoryName = [string]$q.category + if (Test-HasProperty -Object $fixture.duplicate_search -Name $categoryName) + { + $entry = $fixture.duplicate_search.$categoryName + [ordered]@{ Complete = [bool]$entry.complete; Numbers = @($entry.result_numbers); TotalCount = [int]$(if (Test-HasProperty -Object $entry -Name "total_count") { $entry.total_count } else { @($entry.result_numbers).Count }) } + } + else + { + [ordered]@{ Complete = $false; Numbers = @(); TotalCount = 0 } + } + } + else + { + Search-GitHubIssues -Query $q.query + } + + $queryComplete = [bool]$searchResult.Complete + if (-not $queryComplete) + { + $allQueriesComplete = $false + } + + $isKbeCategory = $q.category -in @("open-kbe", "recently-closed-kbe") + foreach ($n in @($searchResult.Numbers)) + { + $candidateText = Get-DuplicateCandidateText -Number $n + if ($null -eq $candidateText) + { + $queryComplete = $false + $allQueriesComplete = $false + $reasonCodes.Add("duplicate-detail-fetch-incomplete") + Add-MissingEvidence -List $missingEvidence -Kind "duplicate-search" -Detail "Search returned $($q.category) #$n, but its issue/PR detail could not be fetched." + $null = $unvalidatedCandidates.Add([ordered]@{ + category = $q.category + number = $n + reason = "could not fetch issue/PR detail; duplicate coverage is incomplete" + }) + continue + } + + $validated = $false + $reason = "issue/PR #$n does not contain the exact fully-qualified test name" + if ($null -ne $testName -and (Test-ContainsExactTestName -Text $candidateText -TestName $testName)) + { + if ($isKbeCategory) + { + $documentedSignature = Get-DocumentedSignature -Text $candidateText + $validated = + $null -ne $documentedSignature -and + (Test-DocumentedSignatureCompatibility ` + -Signature $documentedSignature ` + -FailureLogs @($rawLogs | Where-Object { $_.role -eq "failure" }) ` + -TestName $testName ` + -Root $EvidenceRoot) + $reason = if ($null -eq $documentedSignature) + { + "exact FQN found, but no deterministic documented ErrorMessage/ErrorPattern was parseable" + } + else + { + "exact FQN found, but the documented signature is incompatible with authoritative failure evidence" + } + } + else + { + $validated = $false + $reason = "fix PR validation is disabled until closing-link and changed-file relevance can be proven" + } + } + elseif ($null -eq $testName) + { + $reason = "no resolved test identity to validate against" + } + + if ($validated) + { + $null = $kbeNumbers.Add($n) + $null = $duplicateReferences.Add("issue:$n") + } + else + { + $null = $unvalidatedCandidates.Add([ordered]@{ category = $q.category; number = $n; reason = $reason }) + } + } + + $null = $duplicateQueryResults.Add([ordered]@{ + category = $q.category + query = $q.query + complete = $queryComplete + result_numbers = @($searchResult.Numbers) + total_count = [int]$searchResult.TotalCount + }) +} + +$duplicateStatus = if (-not $allQueriesComplete) +{ + "not-evaluated" +} +elseif ($kbeNumbers.Count -gt 0) +{ + "existing-kbe" +} +else +{ + "none" +} + +if (-not $allQueriesComplete) +{ + $reasonCodes.Add("duplicate-search-incomplete") + Add-MissingEvidence -List $missingEvidence -Kind "duplicate-search" -Detail "At least one duplicate KBE/fix-PR category had incomplete search pagination or candidate-detail coverage." +} + +$duplicateQueriesForCandidate = @($duplicateQueryResults | ForEach-Object { + [ordered]@{ + category = $_.category + query = $_.query + complete = $_.complete + result_numbers = $_.result_numbers + } +}) + +$duplicateCheck = [ordered]@{ + status = $duplicateStatus + checked_utc = $retrievedUtc + coverage = [ordered]@{ + open_kbes = [bool](@($duplicateQueryResults | Where-Object { $_.category -eq "open-kbe" -and $_.complete }).Count -eq 1) + recently_closed_kbes = [bool](@($duplicateQueryResults | Where-Object { $_.category -eq "recently-closed-kbe" -and $_.complete }).Count -eq 1) + open_fix_prs = [bool](@($duplicateQueryResults | Where-Object { $_.category -eq "open-fix-pr" -and $_.complete }).Count -eq 1) + recently_merged_fix_prs = [bool](@($duplicateQueryResults | Where-Object { $_.category -eq "recently-merged-fix-pr" -and $_.complete }).Count -eq 1) + } + references = @($duplicateReferences | Select-Object -Unique) + queries = $duplicateQueriesForCandidate +} + +# The dossier's duplicate_search additionally carries total_count and unvalidated_candidates. +# candidate.duplicate_check keeps the exact candidate-schema shape (additionalProperties: false; +# no total_count/unvalidated_candidates there). +$duplicateCheckWithUnvalidated = [ordered]@{ + status = $duplicateStatus + checked_utc = $retrievedUtc + coverage = $duplicateCheck.coverage + references = $duplicateCheck.references + queries = @($duplicateQueryResults) + unvalidated_candidates = @($unvalidatedCandidates) +} + +# --------------------------------------------------------------------------- +# Step 8: assemble the dossier -- and the candidate, when every gate passed. +# --------------------------------------------------------------------------- + +$reasonCodes = @($reasonCodes | Select-Object -Unique) +$outcome = if ($reasonCodes.Count -gt 0) { "incomplete" } else { "candidate" } + +$candidate = $null +if ($outcome -eq "candidate") +{ + $proposedClassification = switch ($duplicateStatus) + { + "existing-kbe" { "reuse-existing-kbe"; break } + default + { + if ($effectiveSignature -match "(?i)timeout|WebDriverException|TaskCanceledException") + { + "timeout-needs-classification" + } + else + { + "new-kbe-candidate" + } + } + } + + $null = $corroboratingContext.Add([ordered]@{ source = "quarantine-issue"; url = $issueUrl }) + + $candidate = [ordered]@{ + schema_version = 1 + repository = "dotnet/aspnetcore" + repository_ref = [ordered]@{ + branch = "main" + commit_sha = $repoHeadSha + } + issue = [ordered]@{ + number = $IssueNumber + url = $issueUrl + } + test = [ordered]@{ + fully_qualified_name = $testName + } + signature = [ordered]@{ + kind = "ErrorMessage" + values = @($effectiveSignature) + build_retry = $false + exclude_console_log = $false + } + policy = [ordered]@{ + minimum_failure_logs = $minimumFailureBuilds + minimum_negative_logs = $minimumNegativeLogs + } + evidence = [ordered]@{ + raw_logs = @($rawLogs) + corroborating_context = @($corroboratingContext) + } + duplicate_check = $duplicateCheck + proposed_classification = $proposedClassification + } + + $candidateJson = $candidate | ConvertTo-Json -Depth 32 + if (-not ($candidateJson | Test-Json -SchemaFile $CandidateSchemaFile)) + { + throw "Collector produced a candidate that does not satisfy the versioned candidate schema." + } + + $candidateDirectory = Split-Path -Parent $CandidateFile + if ($candidateDirectory) + { + [System.IO.Directory]::CreateDirectory($candidateDirectory) | Out-Null + } + [System.IO.File]::WriteAllText($CandidateFile, $candidateJson + [System.Environment]::NewLine) +} + +$incomplete = $null +if ($outcome -eq "incomplete") +{ + $incomplete = [ordered]@{ + reason_codes = @($reasonCodes) + message = "Collector could not produce a validated candidate for issue #$IssueNumber : " + ($reasonCodes -join ", ") + "." + missing_evidence = @($missingEvidence) + } +} + +$dossier = [ordered]@{ + schema_version = 1 + repository = "dotnet/aspnetcore" + collector = [ordered]@{ + name = "Collect-TestQuarantineKbeEvidence.ps1" + version = 1 + generated_utc = $retrievedUtc + fixture_mode = $isFixtureMode + manual_signature_provided = $manualSignatureProvided + } + issue = [ordered]@{ + number = $IssueNumber + url = $issueUrl + state = $issueState + labels = @($issueLabels) + actor = $issueActor + has_workflow_marker = $hasWorkflowMarker + has_workflow_metadata = $hasWorkflowMetadata + workflow_run_id = $workflowRunId + } + outcome = $outcome + provenance = [ordered]@{ + repository_ref_verification = [ordered]@{ + event_ref = $effectiveEventRef + event_sha = if ($eventShaIsValid) { $effectiveEventSha } else { $null } + checkout_sha = $repoHeadSha + current_main_sha = $currentMainSha + checkout_matches_event_sha = $checkoutMatchesEventSha + event_ref_is_main = $eventRefIsMain + dispatch_sha_on_main = $dispatchShaOnMain + matches_main = $eventRefIsMain -and $checkoutMatchesEventSha -and $dispatchShaOnMain + } + azdo_builds = @($azdoBuildRecords) + build_insights_snapshots = @($buildInsightsSnapshots) + raw_evidence_sources = @($rawEvidenceRecords) + duplicate_search = $duplicateCheckWithUnvalidated + } + candidate = $candidate + incomplete = $incomplete +} + +$dossierJson = $dossier | ConvertTo-Json -Depth 32 +if (-not ($dossierJson | Test-Json -SchemaFile $DossierSchemaFile)) +{ + throw "Generated dossier does not satisfy the versioned dossier schema." +} + +$outputDirectory = Split-Path -Parent $OutputFile +if ($outputDirectory) +{ + [System.IO.Directory]::CreateDirectory($outputDirectory) | Out-Null +} +[System.IO.File]::WriteAllText($OutputFile, $dossierJson + [System.Environment]::NewLine) + +Write-Host "Wrote '$outcome' dossier for issue #$IssueNumber to $OutputFile" diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 new file mode 100644 index 000000000000..f42fe3b52f5c --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 @@ -0,0 +1,847 @@ +#!/usr/bin/env pwsh + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$CandidateFile, + + [Parameter(Mandatory = $true)] + [string]$EvidenceRoot, + + [Parameter(Mandatory = $true)] + [string]$OutputFile, + + [string]$RepositoryRoot = "$PSScriptRoot/../../../..", + + [string]$CandidateSchemaFile = "$PSScriptRoot/test-quarantine-kbe-shadow-candidate.schema.json", + + [string]$ReceiptSchemaFile = "$PSScriptRoot/test-quarantine-kbe-shadow-receipt.schema.json" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$minimumFailureEvidenceFloor = 2 +$minimumNegativeEvidenceFloor = 1 +$failureAssociationWindowLines = 50 + +function Get-Sha256String +{ + param([Parameter(Mandatory = $true)][string]$Value) + + $bytes = [System.Text.Encoding]::UTF8.GetBytes($Value) + $hash = [System.Security.Cryptography.SHA256]::HashData($bytes) + + return [System.Convert]::ToHexString($hash).ToLowerInvariant() +} + +function Get-SafeExcerpt +{ + param([Parameter(Mandatory = $true)][string]$Value) + + $excerpt = [System.Text.RegularExpressions.Regex]::Replace($Value, "[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "?").Trim() + if ($excerpt.Length -gt 300) + { + $excerpt = $excerpt.Substring(0, 300) + } + + return $excerpt +} + +function Resolve-EvidencePath +{ + param( + [Parameter(Mandatory = $true)][string]$Root, + [Parameter(Mandatory = $true)][string]$RelativePath + ) + + if ([System.IO.Path]::IsPathRooted($RelativePath)) + { + throw "Evidence path must be relative: $RelativePath" + } + + $rootInfo = [System.IO.DirectoryInfo]::new($Root) + if ($null -ne $rootInfo.LinkTarget) + { + throw "Evidence root must not be a symbolic link." + } + + $segments = @($RelativePath -split "[\\/]") + if ($segments.Count -eq 0 -or + @($segments | Where-Object { [string]::IsNullOrWhiteSpace($_) -or $_ -in @(".", "..") }).Count -gt 0) + { + throw "Evidence path contains an invalid path segment: $RelativePath" + } + + $normalizedRelativePath = [string]::Join([System.IO.Path]::DirectorySeparatorChar, $segments) + $candidatePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($Root, $normalizedRelativePath)) + $rootPrefix = $Root.TrimEnd([System.IO.Path]::DirectorySeparatorChar) + [System.IO.Path]::DirectorySeparatorChar + if (-not $candidatePath.StartsWith($rootPrefix, [System.StringComparison]::Ordinal)) + { + throw "Evidence path escapes the evidence root: $RelativePath" + } + + $currentPath = $Root + foreach ($segment in $segments) + { + $currentPath = [System.IO.Path]::Combine($currentPath, $segment) + if ([System.IO.Directory]::Exists($currentPath) -or [System.IO.File]::Exists($currentPath)) + { + $itemInfo = Get-Item -LiteralPath $currentPath -Force + if ($null -ne $itemInfo.LinkTarget) + { + throw "Evidence path must not traverse a symbolic link: $RelativePath" + } + } + } + + if (-not [System.IO.File]::Exists($candidatePath)) + { + throw "Evidence file does not exist: $RelativePath" + } + + return $candidatePath +} + +function New-MatchedLine +{ + param( + [Parameter(Mandatory = $true)][int]$LineNumber, + [Parameter(Mandatory = $true)][int]$PatternIndex, + [Parameter(Mandatory = $true)][string]$Line + ) + + return [ordered]@{ + line_number = $LineNumber + pattern_index = $PatternIndex + line_sha256 = Get-Sha256String -Value $Line + excerpt = Get-SafeExcerpt -Value $Line + } +} + +function Test-Log +{ + param( + [Parameter(Mandatory = $true)][string]$Path, + [Parameter(Mandatory = $true)][string]$Kind, + [Parameter(Mandatory = $true)][string[]]$Values, + [Parameter(Mandatory = $true)][string]$TestName, + [Parameter(Mandatory = $true)][int]$FailureAssociationWindowLines, + [System.Text.RegularExpressions.Regex[]]$Regexes + ) + + $matchedLines = [System.Collections.Generic.List[object]]::new() + $matchedLineNumbers = [System.Collections.Generic.List[int]]::new() + $failedTestLineNumbers = [System.Collections.Generic.List[int]]::new() + $lineNumber = 0 + $matchCount = 0 + $passOrSkipMatchCount = 0 + $regexTimeoutCount = 0 + $patternIndex = 0 + $matched = $false + $escapedTestName = [System.Text.RegularExpressions.Regex]::Escape($TestName) + $testNameMatcher = [System.Text.RegularExpressions.Regex]::new( + "(^|[^A-Za-z0-9_.+])$escapedTestName($|[^A-Za-z0-9_.+])", + [System.Text.RegularExpressions.RegexOptions]::CultureInvariant -bor + [System.Text.RegularExpressions.RegexOptions]::NonBacktracking, + [System.TimeSpan]::FromMilliseconds(50)) + + foreach ($line in [System.IO.File]::ReadLines($Path)) + { + $lineNumber++ + + $normalizedLine = $line -replace "^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z\s+", "" + $lineContainsTest = $testNameMatcher.IsMatch($normalizedLine) + $lineIndicatesFailure = + $normalizedLine -match "(?i)^\s*\[FAIL(?:ED)?\]\s+" -or + $normalizedLine -match "(?i)^\s*Failed\s+" -or + $normalizedLine -match "(?i)^\s*\[[^\]\r\n]+\]\s+.+\s+\[FAIL(?:ED)?\]\s*$" + $isFailedTestMarker = $lineContainsTest -and $lineIndicatesFailure + if ($isFailedTestMarker) + { + $failedTestLineNumbers.Add($lineNumber) + } + + if ($normalizedLine -match "(?i)^\s*(?:\[(?:PASS|SKIP)\]\s+|Passed\s+|Skipped\s+)") + { + for ($index = 0; $index -lt $Values.Count; $index++) + { + try + { + $passOrSkipMatched = if ($Kind -eq "ErrorPattern") + { + $Regexes[$index].IsMatch($line) + } + else + { + $line.IndexOf($Values[$index], [System.StringComparison]::Ordinal) -ge 0 + } + } + catch [System.Text.RegularExpressions.RegexMatchTimeoutException] + { + $regexTimeoutCount++ + $passOrSkipMatched = $false + } + + if ($passOrSkipMatched) + { + $passOrSkipMatchCount++ + break + } + } + } + + if ($isFailedTestMarker) + { + continue + } + + if ($Values.Count -eq 1) + { + try + { + $lineMatched = if ($Kind -eq "ErrorPattern") + { + $Regexes[0].IsMatch($line) + } + else + { + $line.IndexOf($Values[0], [System.StringComparison]::Ordinal) -ge 0 + } + } + catch [System.Text.RegularExpressions.RegexMatchTimeoutException] + { + $regexTimeoutCount++ + $lineMatched = $false + } + + if ($lineMatched) + { + $matched = $true + $matchCount++ + $matchedLineNumbers.Add($lineNumber) + + if ($matchedLines.Count -lt 20) + { + $matchedLines.Add((New-MatchedLine -LineNumber $lineNumber -PatternIndex 0 -Line $line)) + } + } + + continue + } + + if ($matched) + { + continue + } + + try + { + $lineMatched = if ($Kind -eq "ErrorPattern") + { + $Regexes[$patternIndex].IsMatch($line) + } + else + { + $line.IndexOf($Values[$patternIndex], [System.StringComparison]::Ordinal) -ge 0 + } + } + catch [System.Text.RegularExpressions.RegexMatchTimeoutException] + { + $regexTimeoutCount++ + $lineMatched = $false + } + + if ($lineMatched) + { + $matchedLines.Add((New-MatchedLine -LineNumber $lineNumber -PatternIndex $patternIndex -Line $line)) + $matchedLineNumbers.Add($lineNumber) + + $patternIndex++ + if ($patternIndex -eq $Values.Count) + { + $matched = $true + $matchCount = 1 + } + } + } + + $signatureAssociatedWithFailedTest = + $matched -and + $failedTestLineNumbers.Count -gt 0 -and + @( + $matchedLineNumbers | + Where-Object { + $matchedLineNumber = $_ + @( + $failedTestLineNumbers | + Where-Object { [System.Math]::Abs($_ - $matchedLineNumber) -le $FailureAssociationWindowLines } + ).Count -gt 0 + } + ).Count -gt 0 + + return [ordered]@{ + line_count = $lineNumber + matched = $matched + match_count = $matchCount + pass_or_skip_match_count = $passOrSkipMatchCount + regex_timeout_count = $regexTimeoutCount + failed_test_detected = $failedTestLineNumbers.Count -gt 0 + failed_test_line_numbers = @($failedTestLineNumbers | Select-Object -First 20) + signature_associated_with_failed_test = $signatureAssociatedWithFailedTest + matched_lines = @($matchedLines) + } +} + +$candidatePath = (Resolve-Path -LiteralPath $CandidateFile).Path +$candidateSchemaPath = (Resolve-Path -LiteralPath $CandidateSchemaFile).Path +$receiptSchemaPath = (Resolve-Path -LiteralPath $ReceiptSchemaFile).Path +$evidenceRootPath = (Resolve-Path -LiteralPath $EvidenceRoot).Path +$repositoryRootPath = (Resolve-Path -LiteralPath $RepositoryRoot).Path +$outputPath = [System.IO.Path]::GetFullPath($OutputFile) + +if ($outputPath -in @($candidatePath, $candidateSchemaPath, $receiptSchemaPath)) +{ + throw "OutputFile must not overwrite an evaluator input." +} + +if ([System.IO.File]::Exists($outputPath)) +{ + [System.IO.File]::Delete($outputPath) +} + +$candidateJson = [System.IO.File]::ReadAllText($candidatePath) +if (-not ($candidateJson | Test-Json -SchemaFile $candidateSchemaPath)) +{ + throw "Candidate JSON does not satisfy the versioned schema." +} + +$candidateDocument = [System.Text.Json.JsonDocument]::Parse($candidateJson) +try +{ + $checkedUtcText = $candidateDocument.RootElement.GetProperty("duplicate_check").GetProperty("checked_utc").GetString() +} +finally +{ + $candidateDocument.Dispose() +} + +$candidate = $candidateJson | ConvertFrom-Json -Depth 32 +$repositoryHead = (& git -C $repositoryRootPath rev-parse HEAD 2>$null).Trim() +if ($LASTEXITCODE -ne 0 -or $repositoryHead -notmatch "^[0-9a-f]{40}$") +{ + throw "RepositoryRoot is not a readable Git repository." +} + +if (-not $repositoryHead.Equals([string]$candidate.repository_ref.commit_sha, [System.StringComparison]::Ordinal)) +{ + throw "Candidate repository commit does not match the checked-out repository." +} + +$expectedIssueUrl = "https://github.com/dotnet/aspnetcore/issues/$($candidate.issue.number)" +if (-not $expectedIssueUrl.Equals([string]$candidate.issue.url, [System.StringComparison]::Ordinal)) +{ + throw "Candidate issue URL does not match the issue number." +} + +$candidateSha256 = (Get-FileHash -LiteralPath $candidatePath -Algorithm SHA256).Hash.ToLowerInvariant() +$candidateSchemaSha256 = (Get-FileHash -LiteralPath $candidateSchemaPath -Algorithm SHA256).Hash.ToLowerInvariant() +$receiptSchemaSha256 = (Get-FileHash -LiteralPath $receiptSchemaPath -Algorithm SHA256).Hash.ToLowerInvariant() +$kind = [string]$candidate.signature.kind +$values = @($candidate.signature.values | ForEach-Object { [string]$_ }) +$qualityFailures = [System.Collections.Generic.List[string]]::new() +$incompleteReasons = [System.Collections.Generic.List[string]]::new() + +$testName = [string]$candidate.test.fully_qualified_name +$testLeafName = ($testName -split "\.")[-1] +if ($kind -eq "ErrorMessage") +{ + foreach ($value in $values) + { + $literalWithoutResultPrefix = $value -replace "^\s*\[(FAIL|PASS|SKIP)\]\s*", "" + if ($testName.IndexOf($literalWithoutResultPrefix, [System.StringComparison]::Ordinal) -ge 0 -or + $testLeafName.IndexOf($literalWithoutResultPrefix, [System.StringComparison]::Ordinal) -ge 0) + { + $qualityFailures.Add("The literal signature contains only the test identifier or a fragment of it.") + } + } +} + +$duplicateStatus = [string]$candidate.duplicate_check.status +$proposedClassification = [string]$candidate.proposed_classification +if ($duplicateStatus -eq "none" -and $proposedClassification -eq "reuse-existing-kbe") +{ + $qualityFailures.Add("The proposed classification requires an existing KBE, but the duplicate check reports none.") +} + +$issueReferences = @($candidate.duplicate_check.references | Where-Object { $_ -match "^issue:[1-9][0-9]*$" }) +$pullRequestReferences = @($candidate.duplicate_check.references | Where-Object { $_ -match "^pull-request:[1-9][0-9]*$" }) +if ($duplicateStatus -eq "existing-kbe" -and $issueReferences.Count -eq 0) +{ + $qualityFailures.Add("The duplicate check reports an existing KBE without an issue reference.") +} + +if ($duplicateStatus -eq "existing-fix-pr" -and $pullRequestReferences.Count -eq 0) +{ + $qualityFailures.Add("The duplicate check reports an existing fix PR without a pull request reference.") +} +elseif ($duplicateStatus -eq "existing-fix-pr") +{ + $incompleteReasons.Add("Existing fix PR classification is unsupported until closing-link and changed-file relevance are proven.") +} + +$checkedUtc = [System.DateTimeOffset]::MinValue +if (-not [System.DateTimeOffset]::TryParse( + $checkedUtcText, + [System.Globalization.CultureInfo]::InvariantCulture, + [System.Globalization.DateTimeStyles]::None, + [ref]$checkedUtc)) +{ + throw "Duplicate check timestamp is invalid." +} + +$now = [System.DateTimeOffset]::UtcNow +$checkedUtc = $checkedUtc.ToUniversalTime() +if ($checkedUtc -gt $now.AddMinutes(5)) +{ + $qualityFailures.Add("The duplicate check timestamp is in the future.") +} +elseif ($checkedUtc -lt $now.AddHours(-24)) +{ + $incompleteReasons.Add("The duplicate check is older than 24 hours.") +} + +$regexes = @() +if ($kind -eq "ErrorPattern") +{ + $regexOptions = [System.Text.RegularExpressions.RegexOptions]::Singleline -bor + [System.Text.RegularExpressions.RegexOptions]::IgnoreCase -bor + [System.Text.RegularExpressions.RegexOptions]::NonBacktracking + $regexTimeout = [System.TimeSpan]::FromMilliseconds(50) + + foreach ($value in $values) + { + if ($value -in @(".*", ".+", "^.*$", "^.+$")) + { + $qualityFailures.Add("The regex signature is unbounded and matches arbitrary text.") + } + + try + { + $regexes += [System.Text.RegularExpressions.Regex]::new($value, $regexOptions, $regexTimeout) + } + catch + { + $qualityFailures.Add("The regex is not compatible with Build Insights KBE matching: $($_.Exception.Message)") + } + } + + foreach ($regex in $regexes) + { + $bareTestProbes = @( + $testName, + $testLeafName, + "[FAIL] $testName", + "[PASS] $testName", + "[SKIP] $testName" + ) + foreach ($probe in $bareTestProbes) + { + try + { + if ($regex.IsMatch($probe)) + { + $qualityFailures.Add("The regex signature matches the test identifier or a fragment of it.") + break + } + } + catch [System.Text.RegularExpressions.RegexMatchTimeoutException] + { + $qualityFailures.Add("The regex exceeded the Build Insights KBE timeout while checking signature specificity.") + break + } + } + } +} + +$logResults = [System.Collections.Generic.List[object]]::new() +$failureLogCount = 0 +$negativeLogCount = 0 +$failureLogsMatched = 0 +$negativeCollisionCount = 0 +$passOrSkipCollisionCount = 0 +$failureHashes = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) +$negativeHashes = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) +$failureBuildIds = [System.Collections.Generic.HashSet[int]]::new() +$negativeBuildIds = [System.Collections.Generic.HashSet[int]]::new() +$seenLogIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) +$seenLogPaths = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + +foreach ($log in $candidate.evidence.raw_logs) +{ + if (-not $seenLogIds.Add([string]$log.id)) + { + throw "Duplicate evidence id: $($log.id)." + } + + if (-not $seenLogPaths.Add([string]$log.path)) + { + throw "Duplicate evidence path: $($log.path)." + } + + $resolvedLogPath = Resolve-EvidencePath -Root $evidenceRootPath -RelativePath ([string]$log.path) + $actualHash = (Get-FileHash -LiteralPath $resolvedLogPath -Algorithm SHA256).Hash.ToLowerInvariant() + if (-not $actualHash.Equals([string]$log.sha256, [System.StringComparison]::Ordinal)) + { + throw "Evidence hash mismatch for $($log.id)." + } + + $match = if ($regexes.Count -eq $values.Count -or $kind -eq "ErrorMessage") + { + Test-Log ` + -Path $resolvedLogPath ` + -Kind $kind ` + -Values $values ` + -TestName ([string]$candidate.test.fully_qualified_name) ` + -FailureAssociationWindowLines $failureAssociationWindowLines ` + -Regexes $regexes + } + else + { + [ordered]@{ + line_count = @([System.IO.File]::ReadLines($resolvedLogPath)).Count + matched = $false + match_count = 0 + pass_or_skip_match_count = 0 + regex_timeout_count = 0 + failed_test_detected = $false + failed_test_line_numbers = @() + signature_associated_with_failed_test = $false + matched_lines = @() + } + } + + if ($log.role -eq "failure") + { + $failureLogCount++ + $null = $failureHashes.Add($actualHash) + $null = $failureBuildIds.Add([int]$log.build.id) + if ($match.matched) + { + $failureLogsMatched++ + } + + if (-not $match.failed_test_detected) + { + $qualityFailures.Add("Failure log '$($log.id)' does not contain a supported failed-test marker for the declared test.") + } + elseif ($match.matched -and -not $match.signature_associated_with_failed_test) + { + $qualityFailures.Add("The signature match in failure log '$($log.id)' is not within $failureAssociationWindowLines lines of the declared test failure.") + } + } + else + { + $negativeLogCount++ + if ($match.matched) + { + $negativeCollisionCount++ + } + } + + if ([string]$log.build.platform -eq "unknown") + { + $incompleteReasons.Add("Evidence log '$($log.id)' has unknown platform; exact environment dimensions are required.") + } + if ([string]$log.build.configuration -eq "unknown") + { + $incompleteReasons.Add("Evidence log '$($log.id)' has unknown configuration; exact environment dimensions are required.") + } + if ([string]$log.build.test_run_identity -eq "unknown") + { + $incompleteReasons.Add("Evidence log '$($log.id)' has unknown canonical TestRun identity; exact environment dimensions are required.") + } + + $passOrSkipCollisionCount += [int]$match.pass_or_skip_match_count + + $logResults.Add([ordered]@{ + id = [string]$log.id + role = [string]$log.role + outcome = [string]$log.outcome + path = [string]$log.path + source_url = [string]$log.source_url + sha256 = $actualHash + build = $log.build + line_count = [int]$match.line_count + matched = [bool]$match.matched + match_count = [int]$match.match_count + pass_or_skip_match_count = [int]$match.pass_or_skip_match_count + regex_timeout_count = [int]$match.regex_timeout_count + failed_test_detected = [bool]$match.failed_test_detected + failed_test_line_numbers = @($match.failed_test_line_numbers) + signature_associated_with_failed_test = [bool]$match.signature_associated_with_failed_test + matched_lines = @($match.matched_lines) + }) +} + +$failureLogsForPassEligibility = @($logResults | Where-Object { $_.role -eq "failure" }) +$passedLogsForEligibility = @($logResults | Where-Object { $_.role -eq "negative" -and $_.outcome -eq "passed" }) +$passesWithMatchingEnvironment = 0 +foreach ($passLog in $passedLogsForEligibility) +{ + $passStartedUtc = [System.DateTimeOffset]::Parse([string]$passLog.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) + $matchingFailureLogs = @( + $failureLogsForPassEligibility | + Where-Object { + [int]$_.build.pipeline_definition_id -eq [int]$passLog.build.pipeline_definition_id -and + [string]$_.build.test_run_identity -eq [string]$passLog.build.test_run_identity -and + [string]$_.build.platform -eq [string]$passLog.build.platform -and + [string]$_.build.configuration -eq [string]$passLog.build.configuration + } + ) + if ($matchingFailureLogs.Count -eq 0) + { + continue + } + + $passesWithMatchingEnvironment++ + $hasFailureBefore = @($matchingFailureLogs | Where-Object { + [System.DateTimeOffset]::Parse([string]$_.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) -lt $passStartedUtc + }).Count -gt 0 + $hasFailureAfter = @($matchingFailureLogs | Where-Object { + [System.DateTimeOffset]::Parse([string]$_.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) -gt $passStartedUtc + }).Count -gt 0 + if ($hasFailureBefore -and $hasFailureAfter) + { + $null = $negativeHashes.Add([string]$passLog.sha256) + $null = $negativeBuildIds.Add([int]$passLog.build.id) + } +} + +$requiredFailureLogs = [System.Math]::Max( + $minimumFailureEvidenceFloor, + [int]$candidate.policy.minimum_failure_logs) +$requiredNegativeLogs = [System.Math]::Max( + $minimumNegativeEvidenceFloor, + [int]$candidate.policy.minimum_negative_logs) + +if ($failureHashes.Count -lt $requiredFailureLogs) +{ + $incompleteReasons.Add("Only $($failureHashes.Count) distinct failure log(s) were supplied; $requiredFailureLogs are required.") +} + +if ($failureBuildIds.Count -lt $requiredFailureLogs) +{ + $incompleteReasons.Add("Only $($failureBuildIds.Count) distinct failure build(s) were supplied; $requiredFailureLogs are required.") +} + +if ($negativeHashes.Count -lt $requiredNegativeLogs) +{ + $incompleteReasons.Add("Only $($negativeHashes.Count) distinct authoritative Passed log(s) were supplied; $requiredNegativeLogs are required.") + if ($failureLogsForPassEligibility.Count -gt 0 -and + $passedLogsForEligibility.Count -gt 0 -and + $passesWithMatchingEnvironment -eq 0) + { + $incompleteReasons.Add("No authoritative Passed occurrence matched a failure's pipeline definition, canonical TestRun identity, platform, and configuration.") + } + elseif ($passesWithMatchingEnvironment -gt 0) + { + $incompleteReasons.Add("No environment-matched Passed occurrence was strictly between an earlier and a later authoritative failure.") + } +} + +if ($negativeBuildIds.Count -lt $requiredNegativeLogs) +{ + $incompleteReasons.Add("Only $($negativeBuildIds.Count) distinct authoritative Passed build(s) were supplied; $requiredNegativeLogs are required.") +} + +if (@($failureHashes | Where-Object { $negativeHashes.Contains($_) }).Count -gt 0) +{ + $qualityFailures.Add("The same log content was supplied as both failure and negative evidence.") +} + +if ($failureLogsMatched -ne $failureLogCount) +{ + $qualityFailures.Add("The signature did not match every supplied failure log.") +} + +if ($negativeCollisionCount -gt 0) +{ + $qualityFailures.Add("The signature matched $negativeCollisionCount negative log(s).") +} + +if ($passOrSkipCollisionCount -gt 0) +{ + $qualityFailures.Add("The signature matched $passOrSkipCollisionCount pass or skip line(s).") +} + +$totalRegexTimeoutCount = 0 +foreach ($logResult in $logResults) +{ + $totalRegexTimeoutCount += [int]$logResult.regex_timeout_count +} + +if ($totalRegexTimeoutCount -gt 0) +{ + $qualityFailures.Add("The regex exceeded the Build Insights KBE timeout on at least one line.") +} + +$coverage = $candidate.duplicate_check.coverage +$requiredDuplicateCategories = @( + "open-kbe", + "recently-closed-kbe", + "open-fix-pr", + "recently-merged-fix-pr" +) +$completeDuplicateCategories = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) +foreach ($query in $candidate.duplicate_check.queries) +{ + if ([bool]$query.complete) + { + $null = $completeDuplicateCategories.Add([string]$query.category) + } +} + +$kbeQueryNumbers = @( + $candidate.duplicate_check.queries | + Where-Object { $_.category -in @("open-kbe", "recently-closed-kbe") } | + ForEach-Object { $_.result_numbers } +) +$fixPrQueryNumbers = @( + $candidate.duplicate_check.queries | + Where-Object { $_.category -in @("open-fix-pr", "recently-merged-fix-pr") } | + ForEach-Object { $_.result_numbers } +) + +if ($duplicateStatus -eq "existing-kbe") +{ + $referencedKbeNumbers = @($issueReferences | ForEach-Object { [int]($_ -replace "^issue:", "") }) + if (@($referencedKbeNumbers | Where-Object { $_ -in $kbeQueryNumbers }).Count -eq 0) + { + $qualityFailures.Add("The existing KBE reference is absent from the recorded KBE query results.") + } +} + +if ($duplicateStatus -eq "existing-fix-pr") +{ + $referencedFixPrNumbers = @($pullRequestReferences | ForEach-Object { [int]($_ -replace "^pull-request:", "") }) + if (@($referencedFixPrNumbers | Where-Object { $_ -in $fixPrQueryNumbers }).Count -eq 0) + { + $qualityFailures.Add("The existing fix PR reference is absent from the recorded PR query results.") + } +} + +if ($duplicateStatus -eq "none" -and + ($candidate.duplicate_check.references.Count -gt 0 -or + $kbeQueryNumbers.Count -gt 0 -or + $fixPrQueryNumbers.Count -gt 0)) +{ + $qualityFailures.Add("The duplicate check reports no duplicate despite recorded issue or pull request results.") +} + +$duplicateCoverageComplete = + [bool]$coverage.open_kbes -and + [bool]$coverage.recently_closed_kbes -and + [bool]$coverage.open_fix_prs -and + [bool]$coverage.recently_merged_fix_prs -and + (@($candidate.duplicate_check.queries | Where-Object { -not $_.complete }).Count -eq 0) -and + (@($requiredDuplicateCategories | Where-Object { -not $completeDuplicateCategories.Contains($_) }).Count -eq 0) + +if (-not $duplicateCoverageComplete -or $candidate.duplicate_check.status -eq "not-evaluated") +{ + $incompleteReasons.Add("Duplicate KBE and fix PR coverage is incomplete.") +} + +$deterministicStatus = if ($qualityFailures.Count -gt 0) +{ + "rejected" +} +elseif ($incompleteReasons.Count -gt 0) +{ + "incomplete" +} +else +{ + "validated" +} + +$shadowRecommendation = "human-review" +if ($deterministicStatus -eq "incomplete") +{ + $shadowRecommendation = "insufficient-evidence" +} +elseif ($deterministicStatus -eq "validated") +{ + $shadowRecommendation = switch ($duplicateStatus) + { + "existing-kbe" { "reuse-existing-kbe"; break } + "existing-fix-pr" { "existing-fix-pr"; break } + "ambiguous" { "human-review"; break } + "integrity-filtered" { "human-review"; break } + default { $proposedClassification } + } +} + +$eligibleForKbeEnrichment = $false + +$reasons = @($qualityFailures) + @($incompleteReasons) +if ($reasons.Count -eq 0) +{ + $reasons = @("The candidate passed deterministic signature, evidence, and duplicate-coverage gates.") +} + +$receipt = [ordered]@{ + schema_version = 1 + repository = "dotnet/aspnetcore" + repository_ref = $candidate.repository_ref + generated_utc = [System.DateTimeOffset]::UtcNow.ToString("O") + evaluator = [ordered]@{ + name = "Evaluate-TestQuarantineKbeCandidate.ps1" + version = 1 + matcher = "Build Insights KBE ErrorMessage/ErrorPattern semantics with failed-test association" + failure_association_window_lines = $failureAssociationWindowLines + candidate_sha256 = $candidateSha256 + candidate_schema_sha256 = $candidateSchemaSha256 + receipt_schema_sha256 = $receiptSchemaSha256 + } + issue = $candidate.issue + test = $candidate.test + signature = $candidate.signature + policy = $candidate.policy + evidence = [ordered]@{ + failure_log_count = $failureLogCount + negative_log_count = $negativeLogCount + distinct_failure_log_count = $failureHashes.Count + distinct_negative_log_count = $negativeHashes.Count + distinct_failure_build_count = $failureBuildIds.Count + distinct_negative_build_count = $negativeBuildIds.Count + all_failure_logs_matched = $failureLogCount -gt 0 -and $failureLogsMatched -eq $failureLogCount + negative_collision_count = $negativeCollisionCount + pass_or_skip_collision_count = $passOrSkipCollisionCount + logs = @($logResults) + corroborating_context = @($candidate.evidence.corroborating_context) + } + duplicate_check = $candidate.duplicate_check + agent_proposed_classification = $proposedClassification + deterministic_status = $deterministicStatus + shadow_recommendation = $shadowRecommendation + eligible_for_kbe_enrichment = $eligibleForKbeEnrichment + evidence_provenance_verified = $false + human_review_required = $true + zero_remote_writes = $true + reasons = @($reasons | Select-Object -Unique) +} + +$receiptJson = $receipt | ConvertTo-Json -Depth 32 +if (-not ($receiptJson | Test-Json -SchemaFile $receiptSchemaPath)) +{ + throw "Generated receipt does not satisfy the versioned schema." +} + +$outputDirectory = Split-Path -Parent $outputPath +if ($outputDirectory) +{ + [System.IO.Directory]::CreateDirectory($outputDirectory) | Out-Null +} + +[System.IO.File]::WriteAllText($outputPath, $receiptJson + [System.Environment]::NewLine) +Write-Host "Wrote $deterministicStatus shadow receipt to $outputPath" diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/New-TestQuarantineKbeSummary.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/New-TestQuarantineKbeSummary.ps1 new file mode 100644 index 000000000000..2075638dc64f --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/New-TestQuarantineKbeSummary.ps1 @@ -0,0 +1,174 @@ +#!/usr/bin/env pwsh +<# +.SYNOPSIS + Renders a short, human-readable Markdown summary of one test-quarantine-kbe-shadow run. + +.DESCRIPTION + Reads the collector's dossier (and, when present, the evaluator's receipt) and writes a + Markdown summary intended for the workflow's step summary and as a small uploaded artifact. + Purely a read-only presentation layer: it does not re-derive or override any collector or + evaluator decision. + +.PARAMETER DossierFile + Path to the dossier JSON produced by Collect-TestQuarantineKbeEvidence.ps1. + +.PARAMETER ReceiptFile + Optional path to the receipt JSON produced by Evaluate-TestQuarantineKbeCandidate.ps1. Only + present when the dossier's outcome was 'candidate'. + +.PARAMETER OutputFile + Path to write the rendered Markdown summary to. +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$DossierFile, + + [string]$ReceiptFile, + + [Parameter(Mandatory = $true)] + [string]$OutputFile +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +function Get-PropertyOrDefault +{ + param($Object, [Parameter(Mandatory = $true)][string]$Name, $Default = $null) + + if ($null -eq $Object) + { + return $Default + } + $property = $Object.PSObject.Properties[$Name] + if ($null -eq $property) + { + return $Default + } + return $property.Value +} + +function ConvertTo-DisplayTimestamp +{ + # ConvertFrom-Json auto-parses ISO-8601 strings into [datetime]; render explicitly as + # round-trip UTC text instead of relying on the current culture's default ToString(). + param($Value) + + if ($null -eq $Value) + { + return "" + } + if ($Value -is [datetime]) + { + return $Value.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ") + } + return [string]$Value +} + +$dossier = Get-Content -LiteralPath $DossierFile -Raw | ConvertFrom-Json -Depth 32 + +$lines = [System.Collections.Generic.List[string]]::new() +$null = $lines.Add("# Test quarantine KBE shadow report") +$null = $lines.Add("") +$null = $lines.Add("This is a **read-only, non-authoritative shadow evaluation**. It never labels, comments on, or otherwise mutates any issue, pull request, branch, or repository file.") +$null = $lines.Add("") +$null = $lines.Add("| Field | Value |") +$null = $lines.Add("|---|---|") +$null = $lines.Add("| Issue | [#$($dossier.issue.number)]($($dossier.issue.url)) |") +$null = $lines.Add("| Collector outcome | ``$($dossier.outcome)`` |") +$null = $lines.Add("| Fixture mode | $($dossier.collector.fixture_mode) |") +$null = $lines.Add("| Manual signature supplied | $($dossier.collector.manual_signature_provided) |") +$null = $lines.Add("| Generated (UTC) | $(ConvertTo-DisplayTimestamp -Value $dossier.collector.generated_utc) |") +$null = $lines.Add("") + +if ($dossier.outcome -eq "incomplete") +{ + $null = $lines.Add("## Incomplete") + $null = $lines.Add("") + $null = $lines.Add($dossier.incomplete.message) + $null = $lines.Add("") + $null = $lines.Add("| Reason code | ") + $null = $lines.Add("|---|") + foreach ($code in @($dossier.incomplete.reason_codes)) + { + $null = $lines.Add("| ``$code`` |") + } + $null = $lines.Add("") + if (@($dossier.incomplete.missing_evidence).Count -gt 0) + { + $null = $lines.Add("
Missing evidence detail") + $null = $lines.Add("") + foreach ($item in @($dossier.incomplete.missing_evidence)) + { + $null = $lines.Add("- **$($item.kind)**: $($item.detail)") + } + $null = $lines.Add("") + $null = $lines.Add("
") + $null = $lines.Add("") + } +} +else +{ + $candidate = $dossier.candidate + $null = $lines.Add("## Candidate") + $null = $lines.Add("") + $null = $lines.Add("- **Test**: ``$($candidate.test.fully_qualified_name)``") + $null = $lines.Add("- **Proposed classification**: ``$($candidate.proposed_classification)``") + $null = $lines.Add("- **Duplicate check status**: ``$($candidate.duplicate_check.status)``") + if (@($candidate.duplicate_check.references).Count -gt 0) + { + $formattedReferences = (@($candidate.duplicate_check.references) | ForEach-Object { '`' + $_ + '`' }) -join ', ' + $null = $lines.Add("- **Duplicate references**: $formattedReferences") + } + $failureCount = @($candidate.evidence.raw_logs | Where-Object { $_.role -eq "failure" }).Count + $negativeCount = @($candidate.evidence.raw_logs | Where-Object { $_.role -eq "negative" }).Count + $null = $lines.Add("- **Failure evidence logs**: $failureCount") + $null = $lines.Add("- **Negative evidence logs**: $negativeCount") + $null = $lines.Add("") + + if (-not [string]::IsNullOrEmpty($ReceiptFile) -and (Test-Path -LiteralPath $ReceiptFile)) + { + $receipt = Get-Content -LiteralPath $ReceiptFile -Raw | ConvertFrom-Json -Depth 32 + $null = $lines.Add("## Evaluator receipt") + $null = $lines.Add("") + $null = $lines.Add("| Field | Value |") + $null = $lines.Add("|---|---|") + $null = $lines.Add("| Deterministic status | ``$($receipt.deterministic_status)`` |") + $null = $lines.Add("| Shadow recommendation | ``$($receipt.shadow_recommendation)`` |") + $null = $lines.Add("| Eligible for KBE enrichment | $($receipt.eligible_for_kbe_enrichment) |") + $null = $lines.Add("| Evidence provenance verified | $($receipt.evidence_provenance_verified) |") + $null = $lines.Add("| Human review required | $($receipt.human_review_required) |") + $null = $lines.Add("") + $null = $lines.Add("**Reasons:**") + foreach ($reason in @($receipt.reasons)) + { + $null = $lines.Add("- $reason") + } + $null = $lines.Add("") + } +} + +if (@($dossier.provenance.build_insights_snapshots).Count -gt 0) +{ + $null = $lines.Add("## Build Insights snapshots (corroborating only, never authoritative)") + $null = $lines.Add("") + $null = $lines.Add("| Commit | Found | Conclusion | Exact test referenced | Known issue referenced |") + $null = $lines.Add("|---|---|---|---|---|") + foreach ($snapshot in @($dossier.provenance.build_insights_snapshots)) + { + $shortSha = $snapshot.source_version.Substring(0, 7) + $conclusion = Get-PropertyOrDefault -Object $snapshot -Name "conclusion" -Default "(none)" + $null = $lines.Add("| ``$shortSha`` | $($snapshot.found) | $conclusion | $($snapshot.exact_test_referenced) | $($snapshot.known_issue_referenced) |") + } + $null = $lines.Add("") +} + +$outputDirectory = Split-Path -Parent $OutputFile +if ($outputDirectory) +{ + [System.IO.Directory]::CreateDirectory($outputDirectory) | Out-Null +} +[System.IO.File]::WriteAllText($OutputFile, ($lines -join [System.Environment]::NewLine) + [System.Environment]::NewLine) +Write-Host "Wrote summary to $OutputFile" diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md b/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md new file mode 100644 index 000000000000..9b20efa597a0 --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md @@ -0,0 +1,310 @@ +# Test quarantine KBE shadow evaluation + +This directory implements a **read-only, non-authoritative shadow evaluation** of whether a +single, already-open dotnet/aspnetcore test-quarantine issue has enough evidence for a +Runtime-style Known Build Error (KBE) signature. It is deliberately narrow in scope: it never +authorizes an automated fix, never mutates repository state, and never promotes itself to +production without an explicit, separate decision by a maintainer. + +## Components + +| File | Role | +|---|---| +| `Collect-TestQuarantineKbeEvidence.ps1` | Deterministic collector. Given one issue number (and, optionally, a manual signature override), gathers public evidence and emits a **dossier**: either a `candidate` ready for the evaluator, or a structured `incomplete` outcome. | +| `Evaluate-TestQuarantineKbeCandidate.ps1` | Deterministic evaluator. Validates a candidate's signature, build/environment provenance, and authoritative Passed evidence, then emits a **receipt**. | +| `New-TestQuarantineKbeSummary.ps1` | Renders a short Markdown summary of a dossier (+ receipt, when present) for the workflow step summary and as an artifact. | +| `test-quarantine-kbe-shadow-candidate.schema.json` | Versioned schema for the evaluator's input. | +| `test-quarantine-kbe-shadow-receipt.schema.json` | Versioned schema for the evaluator's output. | +| `test-quarantine-kbe-shadow-dossier.schema.json` | Schema for the collector's output envelope (provenance + either `candidate` or `incomplete`). Independently versioned; it wraps and reuses the candidate schema rather than replacing or competing with it. | +| `fixtures//` | Compact, sanitized, offline fixtures for three real pilot issues, each with a golden `expected-dossier.json`. | +| `Test-Evaluate-TestQuarantineKbeCandidate.ps1`, `Test-Collect-TestQuarantineKbeEvidence.ps1` | Deterministic, offline test suites (no network access). | +| `Test-WorkflowScriptInjectionSafety.ps1` | Static regression test asserting that no `run:` script body in the two workflows below interpolates a `${{ ... }}` GitHub Actions expression (see "Script-injection safety" below). | + +The companion `.github/workflows/test-quarantine-kbe-shadow.yml` (maintainer dispatch) and +`.github/workflows/test-quarantine-kbe-shadow-tests.yml` (CI) workflows are described below. + +## Trust boundary + +* **Build Insights is the only GitHub check consumed, and remains corroborating rather than + authoritative.** The collector selects only check name `Build Insights`, requiring app slug + `build-insights` when the app metadata is present; the GitHub API request also uses the exact + `check_name=Build Insights` filter. It records check/app identity, dashboard and + GitHub URLs, snapshot ID, conclusion, a SHA-256 of its full text, a capped/redacted excerpt, and + two *conservative* substring checks -- + `exact_test_referenced` (the quarantined test's **full fully-qualified name** appears verbatim; + never set from a bare method name, which commonly collides with unrelated tests -- a + `short_name_referenced` field records that weaker match separately, for transparency only) and + `known_issue_referenced` (a **concrete** dotnet/aspnetcore issue number/URL appears near a + "Known Issue" style label; the bare phrase alone, e.g. a heading or table column reading "Known + Issues" with no associated reference, does not set this true -- `known_issue_numbers` records + exactly which issue(s) were found). A missing or generic snapshot is recorded and surfaced, but + it never overrides raw evidence and never by itself makes a candidate valid or invalid. The + historical pilot commits predate Build Insights and therefore honestly record it as absent. + Current captured/sanitized coverage uses the Build Insights check/app identity, a + `build-insights.dot.net/pull-requests/...` details URL, and its `SnapshotId` marker. Build Insights + is used because it is the supported product/control plane for dashboard history and KBE workflow, + not because its raw failure text is assumed to be more authoritative. The current captured + payload contains report-new-issue links but no matched known issue, so + `known_issue_referenced` remains false; concrete GitHub issue parsing is retained as + corroborating-only support for an observed future payload shape and never gates validity. +* **Authoritative VSTMR test-result detail, not a GitHub check or raw Helix console-log fetch, is + what proves an exact test failure and its recurrence.** Azure DevOps' `resultsbyBuild` + *summary* rows carry only identity and outcome (`id`, `runId`, `automatedTestName`, `outcome`) -- + confirmed live against aspnetcore#68947's own cited build: no `comment`/`errorMessage`/ + `stackTrace` field is present on an ordinary xUnit test's summary row. The collector instead + calls the *detailed* per-result endpoint (`GET .../test/Runs/{runId}/results/{resultId}`) to + retrieve the authoritative `errorMessage`/`stackTrace`, and materializes evidence from that text + directly. Helix job/work-item coordinates are recorded as metadata only when a `comment` field + happens to be present (in practice, only a Helix work item's own crash/`.WorkItemExecution` + pseudo-test row carries one); otherwise `helix_unavailable: true` is recorded explicitly and the + VSTMR detail text remains authoritative on its own -- never silently degraded. Recurrence + requires evidence from **at least two distinct builds** (a single build producing two separate + artifacts is not recurrence), and at least one authoritative **Passed** occurrence strictly + between an earlier and a later failure in the same pipeline definition, canonical TestRun + identity, platform, and configuration. A pass before all failures or after the last failure could represent + a not-yet-failing or already-fixed regression, so neither proves active intermittency. Ordering + currently uses strict Azure DevOps build `started_utc` timestamps; commit ancestry would be + stronger but is not available in the bounded offline evidence contract. The live Passed scan is + bounded to the earliest/latest authoritative failure start times with + `queryOrder=startTimeDescending`, rather than sampling newer builds that cannot be interleaved. + Signature matching against raw text uses ordinal, + case-sensitive substring containment (`[string]::Contains(..., Ordinal)`) throughout -- never + PowerShell's `-like`/`-notlike` operators, whose `*`, `?`, and `[...]` wildcard semantics would otherwise + silently misinterpret a literal ErrorMessage containing any of those characters. +* **Azure DevOps build recurrence spans both `failed` and `partiallySucceeded` results.** A + `resultFilter=failed`-only recurrence query misses real evidence: aspnetcore#68947's own cited + build 1551326 is itself `partiallySucceeded`, not `failed`, confirmed live. Azure DevOps' + `resultFilter` does not support a comma-separated multi-value combination (verified live: + `resultFilter=failed,partiallySucceeded` does not behave as their union), so the collector issues + one request per result value and merges/dedupes by build id (`Merge-AzdoBuildLists`, covered by a + direct, network-free unit test). +* **A `## Failing Test(s)` section naming more than one concrete test identity fails closed.** + aspnetcore#68724 names both a base test and its server-execution subclass override in one + section; live data shows only the override actually failed while the base identity passed. + Silently picking the first backtick-quoted name risks binding evidence to the wrong test. This + collector requires exactly one unambiguous identity per run and fails closed + (`multiple-test-identities-unresolved`) otherwise, rather than guessing. Evaluating every listed + identity independently is a reasonable follow-up left for later, since this PR targets one + issue/one root cause at a time -- documented here as an accepted simplification, not an + oversight. +* **A label or copied workflow marker is not canonical issue provenance.** The collector requires + the `test-failure` label, actor `app/github-actions` or `github-actions[bot]`, both expected + test-quarantine markers, and structured `gh-aw-agentic-workflow` metadata whose `workflow_id` is + `test-quarantine` and whose numeric `id` matches a + `https://github.com/dotnet/aspnetcore/actions/runs/{id}` run URL. A user-authored issue with copied + static strings fails closed. +* **A duplicate-search hit is discovery only, not a validated duplicate.** Every numeric result + returned by the four categorized GitHub searches is fetched. An existing KBE requires the exact + FQN plus a documented `ErrorMessage`/`ErrorPattern` that matches every authoritative failure log + with evaluator-compatible semantics. Fix PR search hits are intentionally never classified as + `existing-fix-pr` yet: title/body mentions cannot prove a closing relationship or changed-file + relevance, so every fetched PR remains an explicit `unvalidated_candidate`. A failed + candidate-detail fetch makes that query and the overall duplicate coverage incomplete. The + "recently" closed/merged categories carry an explicit 90-day `closed:>=`/`merged:>=` time-window + qualifier so the label matches what the query actually searches, and a query is only marked + `complete` when GitHub reports `incomplete_results: false` **and** every matching item (per + `total_count`, across up to 3 paginated pages of 100) was actually retrieved -- a `total_count` + larger than a single page previously went unnoticed. +* **The immutable workflow-dispatch ref is verified, not assumed.** Trusted `github.ref` and + `github.sha` values are passed through `env:` bindings. The ref must be exactly `refs/heads/main`, + the checkout must equal the dispatch SHA, and the dispatch SHA must be identical to or an + ancestor/member of current main. The dossier records event ref/SHA, checkout SHA, and current + main SHA separately, so main advancing after dispatch is valid while non-main dispatches fail + closed. +* **Every countable build is validated before use.** Pipeline definition must be 83 or 87, + `sourceBranch` must be exactly `refs/heads/main`, status must be `completed`, and failure builds + must be `failed` or `partiallySucceeded` (`succeeded` is required for the Passed scan). These + dimensions are recorded in build provenance and candidate evidence. +* **Canonical TestRun identity/platform/configuration are derived from authoritative metadata, + never fabricated.** Azure + DevOps' `buildConfiguration.platform`/`.flavor` fields are empty strings on every real run + observed live; the only authoritative, cheaply-available signal is the VSTMR TestRun's own + `name`. Real families include `Quarantine-Mono-Linux-Release-xunit`, + `Linux-Release-xunit`, sharded `Linux-xunit_1` (observed in build 1538879, run 42642946), + `Ubuntu.2404.Amd64.Open`, and `Windows.Amd64.VS2026.Open`. The collector lowercases the full name + and removes only a known volatile terminal shard suffix (`_N` after `xunit`, `js`, or `Open`); + meaningful family/runtime/browser, OS, architecture, toolchain, and `Open` tokens remain part of + `test_run_identity`. + Platform recognizes Linux/Ubuntu, Windows, and macOS/OSX forms. Configuration is `Debug` or + `Release` only when encoded by the family; otherwise recognized `xunit`/`js`/`Open` families use + the deliberate stable value `not-encoded`. An unrecognized TestRun identity or platform/ + configuration emits explicit missing-evidence and prevents a candidate/validated receipt. + This identity represents a TestRun **name family**, not a unique run: definition 83 can publish + multiple distinct runs with the same name in one build, and those are deliberately treated as + the same environment family. +* **Never infer a pass, a recurrence, a signature, a TestRun identity/platform/configuration, or + a validated duplicate from missing or unverifiable evidence.** Every gap -- a build whose Azure DevOps + metadata has aged out of retention, historical VSTMR test-result data no longer queryable for a + build, an ambiguous or absent signature, an incomplete duplicate search, an unconfirmed + repository ref -- is recorded as an explicit reason code and fails the run closed + (`outcome: "incomplete"`) rather than silently degrading. Real Azure DevOps retention data + confirms this is not hypothetical: build 1537561 (cited by aspnetcore#68947) has already aged out + of the public `dnceng-public` project. +* **No repository-state mutation.** The workflow's permissions are `contents: read`, + `issues: read`, `pull-requests: read`, `checks: read` -- read-only across the board. It uses only + the ambient `GITHUB_TOKEN`, sent as a real `Authorization: Bearer ` header (never a + literal placeholder), never a PAT or other secret. Authenticated calls are load-bearing for + practical use, not cosmetic: GitHub's unauthenticated search-API rate limit (10 requests/minute) + is exhausted by a single run's four duplicate searches plus one retry, and the collector logs the + authenticated `X-RateLimit-Remaining`/`X-RateLimit-Limit` headers (non-blocking, informational + only) after each GitHub call. The workflow uploads artifacts; it never calls any write API (no + labels, comments, commits, branches, or files). +* **Evidence provenance remains unverified in this PR.** Even when the collector's candidate is + independently valid, the evaluator still emits `evidence_provenance_verified: false`, + `eligible_for_kbe_enrichment: false`, and `human_review_required: true`. Flipping + `evidence_provenance_verified` to `true` is a deliberate, separate promotion decision (see + "Promotion gates" below) -- not something this collector claims for itself. + +## Workflow: `test-quarantine-kbe-shadow.yml` (maintainer dispatch) + +* **Trigger**: `workflow_dispatch` only, with a required `issue_number` input and an optional + `signature` input (used only when the issue body has no fenced `## Error Message` block that the + collector can extract deterministically). +* **Fork guard**: the job is gated on `github.repository == 'dotnet/aspnetcore'`, so a forked copy + of this file is inert. +* **Concurrency**: one run per issue number (`test-quarantine-kbe-shadow-`), + cancelling any still-running dispatch for the same issue. +* **Permissions**: `contents: read`, `issues: read`, `pull-requests: read`, `checks: read` -- the + last two are the "only demonstrated needs" beyond the baseline: `checks: read` for the Build + Insights snapshot, `pull-requests: read` for the duplicate fix-PR search. +* **Actions are pinned by commit SHA** (`actions/checkout`, `actions/upload-artifact`), matching + repository convention. +* **Steps**: validate the input, checkout, run the collector, run the evaluator only when the + collector's outcome is `candidate`, render the human-readable summary (to the job's step summary + and as a file), then upload artifacts. +* **Artifacts** (uploaded with `retention-days: 7`, i.e. short-lived by design): `dossier.json`, + `candidate.json` (candidate outcome only), `receipt.json` (candidate outcome only), + `summary.md`, and the capped/redacted evidence text files under `evidence/`. + +## CI: `test-quarantine-kbe-shadow-tests.yml` + +Runs both deterministic, offline PowerShell test suites (`Test-Evaluate-TestQuarantineKbeCandidate.ps1` +and `Test-Collect-TestQuarantineKbeEvidence.ps1`), plus the script-injection safety guard described +below, on every pull request that touches this directory or either workflow file, plus on manual +dispatch. Every fixture is local and offline; no network access or secrets are required, so this is +a low-risk, standard `pull_request`-triggered check -- no self-test mode inside the shadow workflow +itself was needed to close this CI gap. + +## Script-injection safety + +`test-quarantine-kbe-shadow.yml` accepts two `workflow_dispatch` inputs, `issue_number` and +`signature`, both of which are attacker-influenceable text (anyone who can dispatch the workflow +controls their exact content). Neither is ever interpolated directly into a `run:` script body via +`${{ inputs.issue_number }}` / `${{ inputs.signature }}`: a signature value containing a quote, +backtick, or newline embedded directly into script text could otherwise execute arbitrary commands +on the runner. Instead, every input flows through a step (or job) `env:` binding -- for example +`env: { SIGNATURE_INPUT: ${{ inputs.signature }} }` -- and is read back inside the script as an +opaque environment variable (`$env:SIGNATURE_INPUT`), which the PowerShell parser never re-parses +as script text regardless of its content. `issue_number` is additionally validated against +`^[1-9][0-9]*$` before being persisted to `$GITHUB_ENV`, and only that already-validated +`env.ISSUE_NUMBER` value (not the raw input) is used in the uploaded artifact's name -- a +non-`run:` context where Actions itself handles the substitution, not a shell. + +`Test-WorkflowScriptInjectionSafety.ps1` is a static, offline regression test for exactly this +invariant: it parses every `run:` block (block-scalar and single-line forms) in both workflow files +using a minimal, indentation-based reader and fails if any of them contains a `${{` token. It +deliberately does not inspect `if:`, `env:`, `with:`, or `concurrency:` values, since a `${{ }}` +expression there is evaluated by the Actions engine itself and is not a script-injection vector. + +## Fixture mode (`-FixtureRoot`) + +Both `Collect-TestQuarantineKbeEvidence.ps1` and its tests support a `-FixtureRoot ` +parameter. When set, the collector reads one consolidated `fixture.json` document from that +directory instead of making any live GitHub/Azure DevOps call. `fixture.json` mirrors the shape of +the real endpoints captured live during development: + +Fixture tests always pass an explicit `EventRef = refs/heads/main` and `EventSha` equal to the +checked-out repository SHA. The collector does not default these parameters from ambient +`GITHUB_REF`/`GITHUB_SHA`, so a pull-request test job cannot change fixture output. Live workflow +callers pass trusted `github.ref`/`github.sha` through step environment bindings. + +| Key | Mirrors | +|---|---| +| `issue` | `GET /repos/{repo}/issues/{number}` (number, state, labels, actor, body with structured workflow-run metadata) | +| `main_branch` *(optional)* | Current main `.sha`, plus optional `contains_event_sha` to model ancestry/membership after main advances; omit for legacy pilot fixtures that do not exercise this guard | +| `azdo_builds` | `GET .../build/builds/{id}` keyed by build id | +| `recurrence_scan`, `negative_scan` | `GET .../build/builds?resultFilter=...` keyed by pipeline definition id | +| `vstmr_summary` | `GET .../testresults/resultsbyBuild?buildId=...` keyed by build id (array of summary rows) | +| `vstmr_detail` | `GET .../test/Runs/{runId}/results/{resultId}` keyed by `"{runId}:{resultId}"` | +| `vstmr_runs` | `GET .../test/runs/{runId}` keyed by run id (only `.name` is used) | +| `check_runs` | `GET /repos/{repo}/commits/{sha}/check-runs` keyed by commit SHA | +| `duplicate_search` | `GET /search/issues?q=...` keyed by category, pre-paginated (`complete`, `result_numbers`, `total_count`) | +| `duplicate_candidate_text` | `GET /repos/{repo}/issues/{number}` (title + body) keyed by issue/PR number, used to validate a duplicate-search hit | + +## Pilot fixtures + +| Issue | Fixture invocation | Outcome | +|---|---|---| +| [#68724](https://github.com/dotnet/aspnetcore/issues/68724) | no `-Signature` | `incomplete`: `## Failing Test(s)` names two distinct concrete identities (a base test and its server-execution subclass override); live data shows only the override actually failed, so the collector fails closed (`multiple-test-identities-unresolved`) rather than guess | +| [#68947](https://github.com/dotnet/aspnetcore/issues/68947) | `-Signature "OpenQA.Selenium.WebDriverException : The HTTP request to the remote WebDriver server"` (the real issue body has no fenced `## Error Message`, so extraction is deterministically ambiguous without the override -- verified by a second, signature-less invocation of the same fixture in the test suite) | `incomplete`: two failures recur, but the only authoritative pass is older than both failures, so no pass is strictly interleaved between failures (`passed-evidence-not-interleaved`) | +| [#68945](https://github.com/dotnet/aspnetcore/issues/68945) | `-Signature "System.Threading.Tasks.TaskCanceledException: The operation was canceled."` | `incomplete`: the second cited build's historical VSTMR result is unavailable, leaving one usable failure; the remaining pass is not strictly interleaved around two failures (`raw-evidence-insufficient`, `passed-evidence-not-interleaved`) | + +Each fixture directory also has an `expected-dossier.json` golden file used for deep-equality +comparison in the test suite. Golden comparisons exclude the collector's `generated_utc` / +`retrieved_utc` / `captured_utc` / `checked_utc` timestamps and the running checkout's +`commit_sha` / `event_sha` / `checkout_sha` / `current_main_sha` -- all of which are expected to differ +run-to-run and commit-to-commit -- replacing them with the literal sentinel `` on both +sides before comparing. + +The test suite additionally covers, via small synthetic (non-pilot) fixtures: a closed issue, an +issue missing the `test-failure` label, missing or forged workflow provenance, immutable dispatch +ancestry and non-main rejection, strict build definition/ref/status/result gates, skip-only, +pre-first, post-last, and TestRun-identity/environment-mismatched pass evidence, valid interleaving, +unknown environment dimensions, Build Insights snapshot precision, compatible and incompatible same-FQN KBE signatures, failed +duplicate-detail fetches, deliberately unvalidated fix-PR mentions, wildcard-shaped literal +signatures, and build-list merge/dedupe. + +## Reconciling with the existing evaluator contract + +The collector does **not** introduce a third, competing candidate contract. Its `candidate` output, +when present, is validated against the same versioned +`test-quarantine-kbe-shadow-candidate.schema.json` used by the evaluator and is fed to +`Evaluate-TestQuarantineKbeCandidate.ps1` exactly as-is. `test-quarantine-kbe-shadow-dossier.schema.json` is a +new, independently versioned (`schema_version: 1`) envelope that carries collector-specific +provenance (repository-ref verification, Azure DevOps build resolution, Build Insights snapshots, +raw-evidence retrieval, unvalidated duplicate-search candidates) alongside that same +`candidate` object, or a structured `incomplete` outcome when any evidence gate fails. Because the +candidate schema's `duplicate_check.queries[]` items are `additionalProperties: false` (and +correctly so -- it must stay byte-for-byte compatible with the evaluator), the +dossier-only `total_count` field is carried on `provenance.duplicate_search.queries[]` only, never +on `candidate.duplicate_check.queries[]`. + +## Eventual extraction of the production deterministic collector + +`.github/workflows/test-quarantine.md` already embeds a much larger deterministic, pre-activation +collector (Azure DevOps `resultsbyBuild`/build-timeline aggregation across ~200 builds, Helix +console-log `[FAIL]` block extraction, secret redaction) that gathers failure evidence for the +*entire* quarantine-management workflow, not a single issue. **This PR does not modify that +workflow or its production quarantine behavior.** The long-term path is to factor the shared +pieces -- the Azure DevOps/VSTMR fetch helpers, the secret-redaction patterns -- out of +`test-quarantine.md`'s Python step and this directory's PowerShell collector into one common, +tested module that both consume, rather than maintaining two independent implementations of the +same evidence-gathering logic indefinitely. Until that extraction happens, this collector +intentionally duplicates only the narrow, single-issue subset of that logic it needs (build +resolution, VSTMR summary/detail lookup, the same secret-redaction pattern family) and +cross-references the production workflow's real, observed issue-body formats (both the +`50_test_failure.md` template and the freeform `## Details` variant) so it does not invent a third, +incompatible issue-body convention. + +## Promotion gates + +None of the following are implemented by this PR. They are the measurable conditions a future, +separate change would need to satisfy before this collector's output could ever be trusted enough +to flip `evidence_provenance_verified` to `true` or to feed a maintainer-triggered fix workflow: + +1. **Signed/attested provenance for every raw evidence file** -- e.g. a hash chain from the Azure + DevOps API response actually observed at collection time, not merely a locally computed SHA-256 + of whatever bytes were written to disk. +2. **A second, independent collector run reaching the same `candidate` (or `incomplete`) result** + for the same issue, to bound single-run collection errors (rate limiting, partial API + responses, transient Azure DevOps outages). +3. **A maintainer explicitly reviewing and approving** the specific candidate/receipt pair -- this + PR's receipt already always sets `human_review_required: true` for exactly this reason. +4. **The shared extraction described above landing** so the single-issue collector and the + production quarantine workflow's evidence gathering can no longer silently diverge. +5. **A dedicated, maintainer-triggered fix workflow being designed and reviewed separately** -- this + PR intentionally stops at read-only evaluation and does not implement or wire up any automated + fix path. +6. **Evaluating every identity named in a multi-identity `## Failing Test(s)` section + independently**, rather than failing the whole run closed the moment more than one concrete + test is named (the current, deliberately conservative behavior for this PR). diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 new file mode 100644 index 000000000000..432ab51e8f4a --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 @@ -0,0 +1,965 @@ +#!/usr/bin/env pwsh +<# +.SYNOPSIS + Deterministic, offline tests for Collect-TestQuarantineKbeEvidence.ps1. + +.DESCRIPTION + Exercises the collector entirely in -FixtureRoot mode (zero network access) against the + three real pilot quarantine issues recorded in fixtures/, against a battery of synthetic + edge-case fixtures for gates not represented by those three issues, and against + Merge-AzdoBuildLists directly as a pure unit. Golden dossier comparisons exclude + collector-generated timestamps and the running repository's HEAD commit SHA, both of which + are expected to differ between runs/commits; every other field must match exactly. + + Where the collector's outcome is 'candidate', the resulting candidate.json is also fed + through Evaluate-TestQuarantineKbeCandidate.ps1 to prove the two scripts reconcile: the + collector's output is accepted as-is by the versioned evaluator contract. +#> + +[CmdletBinding()] +param() + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$collector = "$PSScriptRoot/Collect-TestQuarantineKbeEvidence.ps1" +$evaluator = "$PSScriptRoot/Evaluate-TestQuarantineKbeCandidate.ps1" +$summaryGenerator = "$PSScriptRoot/New-TestQuarantineKbeSummary.ps1" +$dossierSchema = "$PSScriptRoot/test-quarantine-kbe-shadow-dossier.schema.json" +$candidateSchema = "$PSScriptRoot/test-quarantine-kbe-shadow-candidate.schema.json" +$fixturesRoot = "$PSScriptRoot/fixtures" +$tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) "aspnetcore-kbe-shadow-collector-$([System.Guid]::NewGuid().ToString('N'))" +$repositoryRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path +$repositoryHead = (& git -C $repositoryRoot rev-parse HEAD).Trim() +$fixtureEventRef = "refs/heads/main" +$fixtureEventSha = $repositoryHead +$originalGitHubRef = $env:GITHUB_REF +$originalGitHubSha = $env:GITHUB_SHA + +function Assert-Equal +{ + param( + [Parameter(Mandatory = $true)][AllowNull()]$Actual, + [Parameter(Mandatory = $true)][AllowNull()]$Expected, + [Parameter(Mandatory = $true)][string]$Message + ) + + if ("$Actual" -ne "$Expected") + { + throw "$Message Expected '$Expected', actual '$Actual'." + } +} + +function Assert-Contains +{ + param( + [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$Collection, + [Parameter(Mandatory = $true)]$Value, + [Parameter(Mandatory = $true)][string]$Message + ) + + if (@($Collection | Where-Object { "$_" -eq "$Value" }).Count -eq 0) + { + throw "$Message Expected collection to contain '$Value'; actual: $($Collection -join ', ')." + } +} + +function Assert-NotContains +{ + param( + [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$Collection, + [Parameter(Mandatory = $true)]$Value, + [Parameter(Mandatory = $true)][string]$Message + ) + + if (@($Collection | Where-Object { "$_" -eq "$Value" }).Count -gt 0) + { + throw "$Message Expected collection NOT to contain '$Value'; actual: $($Collection -join ', ')." + } +} + +# The collector-generated timestamps and the running checkout's HEAD commit SHA are expected to +# differ run-to-run and commit-to-commit; every golden fixture below was captured with these +# fields already replaced by this same sentinel. +$volatileKeys = @("generated_utc", "retrieved_utc", "captured_utc", "checked_utc", "commit_sha", "event_sha", "checkout_sha", "current_main_sha") +$volatileSentinel = "" + +function ConvertTo-NormalizedObject +{ + param($Value) + + if ($null -eq $Value) + { + return $null + } + if ($Value -is [System.Management.Automation.PSCustomObject]) + { + $result = [ordered]@{} + foreach ($prop in $Value.PSObject.Properties) + { + $result[$prop.Name] = if ($volatileKeys -contains $prop.Name) { $volatileSentinel } else { ConvertTo-NormalizedObject -Value $prop.Value } + } + return [PSCustomObject]$result + } + if ($Value -is [array]) + { + return ,@($Value | ForEach-Object { ConvertTo-NormalizedObject -Value $_ }) + } + return $Value +} + +function Test-DeepEqual +{ + param($Expected, $Actual, [string]$Path = "`$") + + if ($null -eq $Expected -or $null -eq $Actual) + { + if ($null -ne $Expected -or $null -ne $Actual) + { + throw "Golden mismatch at ${Path}: expected '$Expected', actual '$Actual'." + } + return + } + + if ($Expected -is [System.Management.Automation.PSCustomObject] -and $Actual -is [System.Management.Automation.PSCustomObject]) + { + $expectedNames = @($Expected.PSObject.Properties.Name | Sort-Object) + $actualNames = @($Actual.PSObject.Properties.Name | Sort-Object) + $diff = Compare-Object -ReferenceObject $expectedNames -DifferenceObject $actualNames + if ($null -ne $diff) + { + throw "Golden mismatch at ${Path}: property set differs (expected: $($expectedNames -join ','); actual: $($actualNames -join ','))." + } + foreach ($name in $expectedNames) + { + Test-DeepEqual -Expected $Expected.$name -Actual $Actual.$name -Path "$Path.$name" + } + return + } + + if ($Expected -is [array] -and $Actual -is [array]) + { + if ($Expected.Count -ne $Actual.Count) + { + throw "Golden mismatch at ${Path}: array length differs (expected $($Expected.Count), actual $($Actual.Count))." + } + for ($i = 0; $i -lt $Expected.Count; $i++) + { + Test-DeepEqual -Expected $Expected[$i] -Actual $Actual[$i] -Path "$Path[$i]" + } + return + } + + if ("$Expected" -ne "$Actual") + { + throw "Golden mismatch at ${Path}: expected '$Expected', actual '$Actual'." + } +} + +function Invoke-Collector +{ + param( + [Parameter(Mandatory = $true)][int]$IssueNumber, + [Parameter(Mandatory = $true)][string]$FixtureRoot, + [Parameter(Mandatory = $true)][string]$WorkDirectory, + [string]$Signature, + [string]$EventRef, + [string]$EventSha + ) + + [System.IO.Directory]::CreateDirectory($WorkDirectory) | Out-Null + $dossierPath = Join-Path $WorkDirectory "dossier.json" + $candidatePath = Join-Path $WorkDirectory "candidate.json" + $evidenceRoot = Join-Path $WorkDirectory "evidence" + + $params = @{ + IssueNumber = $IssueNumber + OutputFile = $dossierPath + CandidateFile = $candidatePath + EvidenceRoot = $evidenceRoot + FixtureRoot = $FixtureRoot + RepositoryRoot = $repositoryRoot + EventRef = if ([string]::IsNullOrEmpty($EventRef)) { $fixtureEventRef } else { $EventRef } + EventSha = if ([string]::IsNullOrEmpty($EventSha)) { $fixtureEventSha } else { $EventSha } + DossierSchemaFile = $dossierSchema + CandidateSchemaFile = $candidateSchema + } + if (-not [string]::IsNullOrEmpty($Signature)) + { + $params["Signature"] = $Signature + } + & $collector @params | Out-Null + + return [ordered]@{ + DossierPath = $dossierPath + CandidatePath = $candidatePath + EvidenceRoot = $evidenceRoot + Dossier = (Get-Content -LiteralPath $dossierPath -Raw | ConvertFrom-Json -Depth 32) + } +} + +function Assert-GoldenDossier +{ + param( + [Parameter(Mandatory = $true)][string]$IssueDirectory, + [Parameter(Mandatory = $true)]$ActualDossier + ) + + $goldenPath = Join-Path $IssueDirectory "expected-dossier.json" + $golden = Get-Content -LiteralPath $goldenPath -Raw | ConvertFrom-Json -Depth 32 + $normalizedActual = ConvertTo-NormalizedObject -Value $ActualDossier + Test-DeepEqual -Expected $golden -Actual $normalizedActual +} + +function New-SyntheticFixture +{ + # Writes a minimal-but-complete fixture.json (every top-level key the collector expects to + # be able to read present, even if empty) to a fresh directory under $tempRoot and returns + # its path. + param( + [Parameter(Mandatory = $true)][string]$Name, + [Parameter(Mandatory = $true)][hashtable]$Overrides + ) + + $base = [ordered]@{ + issue = $null + main_branch = $null + azdo_builds = [ordered]@{} + recurrence_scan = [ordered]@{} + negative_scan = [ordered]@{} + vstmr_summary = [ordered]@{} + vstmr_detail = [ordered]@{} + vstmr_runs = [ordered]@{} + check_runs = [ordered]@{} + duplicate_search = [ordered]@{} + duplicate_candidate_text = [ordered]@{} + } + foreach ($key in $Overrides.Keys) + { + $base[$key] = $Overrides[$key] + } + if ($null -ne $base.issue -and -not $base.issue.Contains("user")) + { + $base.issue["user"] = [ordered]@{ login = "app/github-actions" } + } + if ($null -eq $base["main_branch"]) + { + $base.Remove("main_branch") + } + + $fixtureBuilds = @($base.azdo_builds.Values) + foreach ($scan in @($base.recurrence_scan, $base.negative_scan)) + { + foreach ($entry in $scan.Values) + { + $fixtureBuilds += @($entry) + } + } + foreach ($build in $fixtureBuilds) + { + if (-not $build.Contains("sourceBranch")) + { + $build["sourceBranch"] = "refs/heads/main" + } + if (-not $build.Contains("status")) + { + $build["status"] = "completed" + } + } + + $directory = Join-Path $tempRoot "fixture-$Name" + [System.IO.Directory]::CreateDirectory($directory) | Out-Null + $base | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath (Join-Path $directory "fixture.json") + return $directory +} + +function New-DerivedFixture +{ + param( + [Parameter(Mandatory = $true)][string]$Name, + [Parameter(Mandatory = $true)][string]$Source, + [Parameter(Mandatory = $true)][scriptblock]$Mutate + ) + + $fixtureObject = Get-Content -LiteralPath (Join-Path $Source "fixture.json") -Raw | ConvertFrom-Json -Depth 32 + & $Mutate $fixtureObject + $directory = Join-Path $tempRoot "fixture-$Name" + [System.IO.Directory]::CreateDirectory($directory) | Out-Null + $fixtureObject | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath (Join-Path $directory "fixture.json") + return $directory +} + +$workflowMarker = @" + + + +"@ +# Keep the ignored check name split so the Build Insights-only source audit has zero legacy +# product-name references while still exercising exact check-name rejection. +$ignoredLegacyCheckName = "Build " + "Analysis" +# Built from a single-quoted literal (no backtick-escape processing) rather than embedding +# repeated backtick-escape sequences directly in double-quoted synthetic issue bodies below, +# which is easy to miscount (a stray extra backtick silently becomes an unrelated `t`/`n`/etc. +# escape sequence instead of a literal backtick). +$codeFence = '```' +$defaultDuplicateSearch = [ordered]@{ + "open-kbe" = [ordered]@{ complete = $true; result_numbers = @(); total_count = 0 } + "recently-closed-kbe" = [ordered]@{ complete = $true; result_numbers = @(); total_count = 0 } + "open-fix-pr" = [ordered]@{ complete = $true; result_numbers = @(); total_count = 0 } + "recently-merged-fix-pr" = [ordered]@{ complete = $true; result_numbers = @(); total_count = 0 } +} + +try +{ + [System.IO.Directory]::CreateDirectory($tempRoot) | Out-Null + $env:GITHUB_REF = "refs/pull/69021/merge" + $env:GITHUB_SHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + # ------------------------------------------------------------------ + # Pilot 1 -- aspnetcore#68724: '## Failing Test(s)' names two distinct concrete test + # identities (a base test and its server-execution subclass override). Live data shows only + # the override actually failed while the base identity passed -- the collector must fail + # closed rather than silently bind evidence to the first name it sees. + # ------------------------------------------------------------------ + $result68724 = Invoke-Collector -IssueNumber 68724 -FixtureRoot "$fixturesRoot/68724" -WorkDirectory "$tempRoot/68724" + Assert-Equal -Actual $result68724.Dossier.outcome -Expected "incomplete" -Message "#68724 outcome mismatch." + Assert-Contains -Collection @($result68724.Dossier.incomplete.reason_codes) -Value "multiple-test-identities-unresolved" -Message "#68724 reason codes mismatch." + Assert-NotContains -Collection @($result68724.Dossier.incomplete.reason_codes) -Value "workflow-dispatch-ref-not-main" -Message "Ambient GITHUB_REF must not affect fixture collection." + Assert-Equal -Actual $result68724.Dossier.provenance.repository_ref_verification.event_ref -Expected $fixtureEventRef -Message "Fixture event ref must be explicit and deterministic." + Assert-Equal -Actual $result68724.Dossier.provenance.repository_ref_verification.event_sha -Expected $fixtureEventSha -Message "Fixture event SHA must be explicit and deterministic." + Assert-GoldenDossier -IssueDirectory "$fixturesRoot/68724" -ActualDossier $result68724.Dossier + + # ------------------------------------------------------------------ + # Pilot 2 -- aspnetcore#68947: the issue body has no fenced '## Error Message' block, so + # deterministic extraction is ambiguous without a manual signature. Its own cited second + # build (1537561) has aged out of Azure DevOps retention; recurrence is instead established + # by the collector's supplementary recurrence scan, which must also query + # resultFilter=partiallySucceeded (the cited build 1551326's own real result value) in + # addition to resultFilter=failed. + # ------------------------------------------------------------------ + $result68947NoSig = Invoke-Collector -IssueNumber 68947 -FixtureRoot "$fixturesRoot/68947" -WorkDirectory "$tempRoot/68947-no-signature" + Assert-Equal -Actual $result68947NoSig.Dossier.outcome -Expected "incomplete" -Message "#68947 (no signature) outcome mismatch." + Assert-Contains -Collection @($result68947NoSig.Dossier.incomplete.reason_codes) -Value "signature-extraction-ambiguous" -Message "#68947 (no signature) reason codes mismatch." + + $signature68947 = "OpenQA.Selenium.WebDriverException : The HTTP request to the remote WebDriver server" + $result68947 = Invoke-Collector -IssueNumber 68947 -FixtureRoot "$fixturesRoot/68947" -WorkDirectory "$tempRoot/68947" -Signature $signature68947 + Assert-Equal -Actual $result68947.Dossier.outcome -Expected "incomplete" -Message "#68947 outcome mismatch." + Assert-Contains -Collection @($result68947.Dossier.incomplete.reason_codes) -Value "passed-evidence-not-interleaved" -Message "#68947 must not count its pass that predates both collected failures." + Assert-Equal -Actual $result68947.Dossier.candidate -Expected $null -Message "#68947 must not emit a candidate without a contemporaneous environment-matched pass." + Assert-GoldenDossier -IssueDirectory "$fixturesRoot/68947" -ActualDossier $result68947.Dossier + + $summaryPath68947 = Join-Path "$tempRoot/68947" "summary.md" + & $summaryGenerator -DossierFile $result68947.DossierPath -OutputFile $summaryPath68947 + $summaryText68947 = Get-Content -LiteralPath $summaryPath68947 -Raw + if (-not $summaryText68947.Contains("passed-evidence-not-interleaved")) + { + throw "#68947 summary must mention the chronology failure." + } + + # ------------------------------------------------------------------ + # Pilot 3 -- aspnetcore#68945: both cited builds' Azure DevOps build-level metadata still + # resolves, but the second build's historical VSTMR test-result data is no longer queryable, + # leaving only one usable failure log below the two-build recurrence floor. The collector + # must fail closed rather than infer a pass or fabricate evidence. + # ------------------------------------------------------------------ + $result68945 = Invoke-Collector -IssueNumber 68945 -FixtureRoot "$fixturesRoot/68945" -WorkDirectory "$tempRoot/68945" -Signature "System.Threading.Tasks.TaskCanceledException: The operation was canceled." + Assert-Equal -Actual $result68945.Dossier.outcome -Expected "incomplete" -Message "#68945 outcome mismatch." + Assert-Contains -Collection @($result68945.Dossier.incomplete.reason_codes) -Value "raw-evidence-insufficient" -Message "#68945 reason codes must record the insufficient evidence." + Assert-Equal -Actual $result68945.Dossier.candidate -Expected $null -Message "#68945 must not emit a candidate." + Assert-GoldenDossier -IssueDirectory "$fixturesRoot/68945" -ActualDossier $result68945.Dossier + + $summaryPath68945 = Join-Path "$tempRoot/68945" "summary.md" + & $summaryGenerator -DossierFile $result68945.DossierPath -OutputFile $summaryPath68945 + $summaryText68945 = Get-Content -LiteralPath $summaryPath68945 -Raw + if (-not $summaryText68945.Contains("raw-evidence-insufficient")) + { + throw "#68945 summary must mention the raw-evidence-insufficient reason code." + } + + # ------------------------------------------------------------------ + # Edge case: a closed issue must fail closed regardless of label/marker. + # ------------------------------------------------------------------ + $closedDir = New-SyntheticFixture -Name "closed-issue" -Overrides @{ + issue = [ordered]@{ + number = 1 + state = "closed" + labels = @("test-failure") + body = "## Failing Test(s)`n`` Sample.Tests.Closed ``$([System.Environment]::NewLine)$workflowMarker" + } + duplicate_search = $defaultDuplicateSearch + } + $resultClosed = Invoke-Collector -IssueNumber 1 -FixtureRoot $closedDir -WorkDirectory (Join-Path $tempRoot "closed-issue") + Assert-Equal -Actual $resultClosed.Dossier.outcome -Expected "incomplete" -Message "Closed-issue outcome mismatch." + Assert-Contains -Collection @($resultClosed.Dossier.incomplete.reason_codes) -Value "issue-not-open" -Message "Closed-issue reason codes mismatch." + + # ------------------------------------------------------------------ + # Edge case: an issue missing the canonical 'test-failure' label must fail closed. + # ------------------------------------------------------------------ + $unlabeledDir = New-SyntheticFixture -Name "unlabeled-issue" -Overrides @{ + issue = [ordered]@{ + number = 2 + state = "open" + labels = @("area-blazor") + body = "## Failing Test(s)`n`` Sample.Tests.Unlabeled ``$([System.Environment]::NewLine)$workflowMarker" + } + duplicate_search = $defaultDuplicateSearch + } + $resultUnlabeled = Invoke-Collector -IssueNumber 2 -FixtureRoot $unlabeledDir -WorkDirectory (Join-Path $tempRoot "unlabeled-issue") + Assert-Equal -Actual $resultUnlabeled.Dossier.outcome -Expected "incomplete" -Message "Unlabeled-issue outcome mismatch." + Assert-Contains -Collection @($resultUnlabeled.Dossier.incomplete.reason_codes) -Value "issue-not-canonical-quarantine" -Message "Unlabeled-issue reason codes mismatch." + + # ------------------------------------------------------------------ + # Edge case (item 5): the 'test-failure' label alone is not proof of quarantine automation -- + # an issue carrying the label and open state, but with no trusted gh-aw workflow marker in + # its body, must also fail closed. + # ------------------------------------------------------------------ + $missingMarkerDir = New-SyntheticFixture -Name "missing-marker" -Overrides @{ + issue = [ordered]@{ + number = 10 + state = "open" + labels = @("test-failure") + body = "## Failing Test(s)`n`` Sample.Tests.NoMarker ``" + } + duplicate_search = $defaultDuplicateSearch + } + $resultMissingMarker = Invoke-Collector -IssueNumber 10 -FixtureRoot $missingMarkerDir -WorkDirectory (Join-Path $tempRoot "missing-marker") + Assert-Equal -Actual $resultMissingMarker.Dossier.issue.has_workflow_marker -Expected $false -Message "Missing-marker issue.has_workflow_marker mismatch." + Assert-Equal -Actual $resultMissingMarker.Dossier.outcome -Expected "incomplete" -Message "Missing-marker outcome mismatch." + Assert-Contains -Collection @($resultMissingMarker.Dossier.incomplete.reason_codes) -Value "issue-not-canonical-quarantine" -Message "Missing-marker reason codes mismatch: the 'test-failure' label alone must not be treated as proof of quarantine automation." + + $forgedMarkerDir = New-SyntheticFixture -Name "forged-marker" -Overrides @{ + issue = [ordered]@{ + number = 15 + state = "open" + labels = @("test-failure") + user = [ordered]@{ login = "octocat" } + body = "## Failing Test(s)`n`` Sample.Tests.ForgedMarker ``$([System.Environment]::NewLine)$workflowMarker" + } + duplicate_search = $defaultDuplicateSearch + } + $resultForgedMarker = Invoke-Collector -IssueNumber 15 -FixtureRoot $forgedMarkerDir -WorkDirectory (Join-Path $tempRoot "forged-marker") + Assert-Equal -Actual $resultForgedMarker.Dossier.outcome -Expected "incomplete" -Message "A user-authored issue with copied workflow markers must fail closed." + Assert-Contains -Collection @($resultForgedMarker.Dossier.incomplete.reason_codes) -Value "issue-not-canonical-quarantine" -Message "Forged-marker reason code mismatch." + Assert-Equal -Actual $resultForgedMarker.Dossier.issue.has_workflow_marker -Expected $true -Message "Forged-marker fixture must prove copied static markers were present." + Assert-Equal -Actual $resultForgedMarker.Dossier.issue.has_workflow_metadata -Expected $true -Message "Forged-marker fixture must prove copied structured metadata was present." + Assert-Equal -Actual $resultForgedMarker.Dossier.issue.actor -Expected "octocat" -Message "Forged-marker actor provenance mismatch." + + $mismatchedMetadata = @" + + + +"@ + $mismatchedMetadataDir = New-SyntheticFixture -Name "mismatched-workflow-metadata" -Overrides @{ + issue = [ordered]@{ + number = 16 + state = "open" + labels = @("test-failure") + body = "## Failing Test(s)`n`` Sample.Tests.MismatchedMetadata ``$([System.Environment]::NewLine)$mismatchedMetadata" + } + duplicate_search = $defaultDuplicateSearch + } + $resultMismatchedMetadata = Invoke-Collector -IssueNumber 16 -FixtureRoot $mismatchedMetadataDir -WorkDirectory (Join-Path $tempRoot "mismatched-workflow-metadata") + Assert-Equal -Actual $resultMismatchedMetadata.Dossier.outcome -Expected "incomplete" -Message "Mismatched workflow metadata must fail closed." + Assert-Equal -Actual $resultMismatchedMetadata.Dossier.issue.has_workflow_metadata -Expected $false -Message "Mismatched workflow run IDs must not validate." + + # ------------------------------------------------------------------ + # The immutable dispatch SHA must be confirmed as a member of main. A deliberately unrelated + # current-main SHA must fail closed rather than mislabel the checkout. + # ------------------------------------------------------------------ + $repoRefMismatchDir = New-SyntheticFixture -Name "repo-ref-mismatch" -Overrides @{ + issue = [ordered]@{ + number = 13 + state = "open" + labels = @("test-failure") + body = "## Failing Test(s)`n`` Sample.Tests.RepoRefMismatch ``$([System.Environment]::NewLine)$workflowMarker" + } + main_branch = [ordered]@{ sha = "0000000000000000000000000000000000000000" } + duplicate_search = $defaultDuplicateSearch + } + $resultRepoRefMismatch = Invoke-Collector -IssueNumber 13 -FixtureRoot $repoRefMismatchDir -WorkDirectory (Join-Path $tempRoot "repo-ref-mismatch") + Assert-Equal -Actual $resultRepoRefMismatch.Dossier.outcome -Expected "incomplete" -Message "Repository-ref-mismatch outcome mismatch." + Assert-Contains -Collection @($resultRepoRefMismatch.Dossier.incomplete.reason_codes) -Value "repository-ref-not-main" -Message "Repository-ref-mismatch reason codes mismatch." + Assert-Equal -Actual $resultRepoRefMismatch.Dossier.provenance.repository_ref_verification.matches_main -Expected $false -Message "Repository-ref-mismatch matches_main mismatch." + + # ------------------------------------------------------------------ + # Build Insights capture plus exact-vs-short-name and concrete-vs-generic-known-issue flag + # precision. Only the full FQN and a concrete issue reference may set the stronger flags. + # ------------------------------------------------------------------ + $flagsTestName = "Sample.Tests.ExactMatchCase" + $flagsSignature = "System.InvalidOperationException: Sample failure for exact-match testing." + $flagsShaA = "f4d9777d7b9a3d45c88e1ca1b10609e412cc4ade" + $flagsShaB = "e06ef94591aaa5a8dc3f84926f8664b2964bf0ea" + $flagsShaC = "51e066210e1643430dccb9986c42500d3e706638" + $flagsDir = New-SyntheticFixture -Name "check-run-flags" -Overrides @{ + main_branch = [ordered]@{ sha = ("b" * 40); contains_event_sha = $true } + issue = [ordered]@{ + number = 11 + state = "open" + labels = @("test-failure") + body = "## Failing Test(s)`n`` $flagsTestName ``$([System.Environment]::NewLine)## Error Message$([System.Environment]::NewLine)${codeFence}text$([System.Environment]::NewLine)$flagsSignature$([System.Environment]::NewLine)$codeFence$([System.Environment]::NewLine)## Build$([System.Environment]::NewLine)https://dev.azure.com/dnceng-public/public/_build/results?buildId=6100001$([System.Environment]::NewLine)$workflowMarker" + } + azdo_builds = [ordered]@{ + "6100001" = [ordered]@{ definition = [ordered]@{ id = 83 }; sourceVersion = $flagsShaA; startTime = "2026-08-01T00:00:00Z"; finishTime = "2026-08-01T01:00:00Z"; result = "failed" } + } + recurrence_scan = [ordered]@{ + "83" = @([ordered]@{ id = 6100002; sourceVersion = $flagsShaB; startTime = "2026-07-30T00:00:00Z"; finishTime = "2026-07-30T01:00:00Z"; result = "failed" }) + } + negative_scan = [ordered]@{ + "83" = @([ordered]@{ id = 6100003; sourceVersion = $flagsShaC; startTime = "2026-07-31T00:00:00Z"; finishTime = "2026-07-31T01:00:00Z"; result = "succeeded" }) + } + vstmr_summary = [ordered]@{ + "6100001" = @([ordered]@{ id = 1; runId = 7100001; outcome = "Failed"; automatedTestName = $flagsTestName }) + "6100002" = @([ordered]@{ id = 2; runId = 7100002; outcome = "Failed"; automatedTestName = $flagsTestName }) + "6100003" = @([ordered]@{ id = 3; runId = 7100003; outcome = "Passed"; automatedTestName = $flagsTestName }) + } + vstmr_detail = [ordered]@{ + "7100001:1" = [ordered]@{ outcome = "Failed"; errorMessage = $flagsSignature; stackTrace = "at $flagsTestName.Run() (build a)" } + "7100002:2" = [ordered]@{ outcome = "Failed"; errorMessage = $flagsSignature; stackTrace = "at $flagsTestName.Run() (build b)" } + "7100003:3" = [ordered]@{ outcome = "Passed"; errorMessage = $null; stackTrace = $null } + } + vstmr_runs = [ordered]@{ + "7100001" = [ordered]@{ name = "Quarantine-Mono-Windows-Debug-xunit" } + "7100002" = [ordered]@{ name = "Quarantine-Mono-Windows-Debug-xunit" } + "7100003" = [ordered]@{ name = "Quarantine-Mono-Windows-Debug-xunit" } + } + check_runs = [ordered]@{ + # Check identity, URLs, title, failure shape, and SnapshotId are captured/sanitized + # from the current Build Insights payload on dotnet/aspnetcore#69027. + $flagsShaA = @([ordered]@{ + name = "Build Insights"; id = 100745972738; conclusion = "failure" + details_url = "https://build-insights.dot.net/pull-requests/3922" + html_url = "https://github.com/dotnet/aspnetcore/runs/100745972738" + app = [ordered]@{ slug = "build-insights"; name = "Build Insights" } + output = [ordered]@{ + title = ".NET Result Analysis" + text = "
`n

Build Failures

`nReport repository issue`n:x:eng/targets/Java.Common.targets(59,5): error MSB3073: a sanitized build command exited with code 1.`n" + } + }) + $flagsShaB = @([ordered]@{ + name = "Build Insights"; id = 100745972739; conclusion = "failure" + details_url = "https://build-insights.dot.net/pull-requests/3923" + html_url = "https://github.com/dotnet/aspnetcore/runs/100745972739" + app = [ordered]@{ slug = "build-insights"; name = "Build Insights" } + # This sanitized future-compatible shape exercises exact FQN and concrete GitHub + # issue parsing independently of the captured current payload above. + output = [ordered]@{ + title = ".NET Result Analysis" + text = "$flagsTestName failed. This matches a Known Issue: dotnet/aspnetcore#70000.`n" + } + }) + } + duplicate_search = $defaultDuplicateSearch + } + $resultFlags = Invoke-Collector -IssueNumber 11 -FixtureRoot $flagsDir -WorkDirectory (Join-Path $tempRoot "check-run-flags") + Assert-Equal -Actual $resultFlags.Dossier.outcome -Expected "candidate" -Message "check-run-flags outcome mismatch." + Assert-Equal -Actual $resultFlags.Dossier.provenance.repository_ref_verification.dispatch_sha_on_main -Expected $true -Message "A dispatch SHA that is an ancestor/member of advanced main must validate." + Assert-Equal -Actual $resultFlags.Dossier.provenance.repository_ref_verification.current_main_sha -Expected ("b" * 40) -Message "Current main SHA provenance mismatch." + Assert-Equal -Actual $resultFlags.Dossier.provenance.repository_ref_verification.checkout_sha -Expected $repositoryHead -Message "Dispatch checkout SHA provenance mismatch." + $snapshotA = @($resultFlags.Dossier.provenance.build_insights_snapshots | Where-Object { $_.source_version -eq $flagsShaA })[0] + $snapshotB = @($resultFlags.Dossier.provenance.build_insights_snapshots | Where-Object { $_.source_version -eq $flagsShaB })[0] + Assert-Equal -Actual $snapshotA.app_slug -Expected "build-insights" -Message "Build Insights app slug mismatch." + Assert-Equal -Actual $snapshotA.details_url -Expected "https://build-insights.dot.net/pull-requests/3922" -Message "Build Insights details URL mismatch." + Assert-Equal -Actual $snapshotA.snapshot_id -Expected "2026-09-03T17-26-04" -Message "Build Insights SnapshotId mismatch." + Assert-Equal -Actual $snapshotA.exact_test_referenced -Expected $false -Message "Captured current Build Insights payload must not invent an exact test reference." + Assert-Equal -Actual $snapshotA.known_issue_referenced -Expected $false -Message "A Build Insights report-new-issue link is not an existing known issue." + Assert-Equal -Actual $snapshotB.exact_test_referenced -Expected $true -Message "exact_test_referenced must be true when the full FQN appears verbatim." + Assert-Equal -Actual $snapshotB.known_issue_referenced -Expected $true -Message "known_issue_referenced must be true when a concrete issue number follows 'Known Issue'." + Assert-Contains -Collection @($snapshotB.known_issue_numbers) -Value 70000 -Message "known_issue_numbers must record the referenced issue." + + $legacyCheckOnlyDir = New-DerivedFixture -Name "ignored-legacy-check" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.check_runs.$flagsShaA[0].name = $ignoredLegacyCheckName + } + $resultLegacyCheckOnly = Invoke-Collector -IssueNumber 11 -FixtureRoot $legacyCheckOnlyDir -WorkDirectory (Join-Path $tempRoot "ignored-legacy-check") + $ignoredSnapshot = @($resultLegacyCheckOnly.Dossier.provenance.build_insights_snapshots | Where-Object { $_.source_version -eq $flagsShaA })[0] + Assert-Equal -Actual $ignoredSnapshot.found -Expected $false -Message "A differently named legacy check must never be selected as Build Insights." + + $wrongBuildInsightsAppDir = New-DerivedFixture -Name "wrong-build-insights-app" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.check_runs.$flagsShaA[0].app.slug = "unrelated-check-app" + } + $resultWrongBuildInsightsApp = Invoke-Collector -IssueNumber 11 -FixtureRoot $wrongBuildInsightsAppDir -WorkDirectory (Join-Path $tempRoot "wrong-build-insights-app") + $wrongAppSnapshot = @($resultWrongBuildInsightsApp.Dossier.provenance.build_insights_snapshots | Where-Object { $_.source_version -eq $flagsShaA })[0] + Assert-Equal -Actual $wrongAppSnapshot.found -Expected $false -Message "A Build Insights-named check from another app must be ignored." + + $missingDetailsUrlDir = New-DerivedFixture -Name "build-insights-without-details-url" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.check_runs.$flagsShaA[0].PSObject.Properties.Remove("details_url") + } + $resultMissingDetailsUrl = Invoke-Collector -IssueNumber 11 -FixtureRoot $missingDetailsUrlDir -WorkDirectory (Join-Path $tempRoot "build-insights-without-details-url") + $missingDetailsSnapshot = @($resultMissingDetailsUrl.Dossier.provenance.build_insights_snapshots | Where-Object { $_.source_version -eq $flagsShaA })[0] + Assert-Equal -Actual $missingDetailsSnapshot.found -Expected $true -Message "A nullable Build Insights details URL must not discard the snapshot." + Assert-Equal -Actual $missingDetailsSnapshot.details_url -Expected $null -Message "Missing Build Insights details URL provenance mismatch." + + $flagsReceiptPath = Join-Path "$tempRoot/check-run-flags" "receipt.json" + & $evaluator -CandidateFile $resultFlags.CandidatePath -EvidenceRoot $resultFlags.EvidenceRoot -OutputFile $flagsReceiptPath -RepositoryRoot $repositoryRoot -CandidateSchemaFile $candidateSchema + $flagsReceipt = Get-Content -LiteralPath $flagsReceiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $flagsReceipt.deterministic_status -Expected "validated" -Message "Collector/evaluator pass-eligibility rules must reconcile." + + $multiplePassRowsDir = New-DerivedFixture -Name "multiple-pass-environments" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.vstmr_summary.'6100003' = @( + [PSCustomObject]@{ id = 4; runId = 7100004; outcome = "Passed"; automatedTestName = $flagsTestName }, + $fixtureObject.vstmr_summary.'6100003'[0] + ) + $fixtureObject.vstmr_detail | Add-Member -NotePropertyName '7100004:4' -NotePropertyValue ([PSCustomObject]@{ outcome = "Passed"; errorMessage = $null; stackTrace = $null }) + $fixtureObject.vstmr_runs | Add-Member -NotePropertyName '7100004' -NotePropertyValue ([PSCustomObject]@{ name = "Quarantine-Mono-Linux-Release-xunit" }) + } + $resultMultiplePassRows = Invoke-Collector -IssueNumber 11 -FixtureRoot $multiplePassRowsDir -WorkDirectory (Join-Path $tempRoot "multiple-pass-environments") + Assert-Equal -Actual $resultMultiplePassRows.Dossier.outcome -Expected "candidate" -Message "Collector must prefer an environment-matched pass row from a multi-run build." + $selectedPassSource = @($resultMultiplePassRows.Dossier.provenance.raw_evidence_sources | Where-Object { $_.role -eq "negative" })[0] + Assert-Equal -Actual $selectedPassSource.run_id -Expected 7100003 -Message "Collector selected the wrong pass TestRun environment." + + $nonMainResult = Invoke-Collector ` + -IssueNumber 11 ` + -FixtureRoot $flagsDir ` + -WorkDirectory (Join-Path $tempRoot "non-main-dispatch") ` + -EventRef "refs/heads/feature/quarantine" ` + -EventSha $repositoryHead + Assert-Equal -Actual $nonMainResult.Dossier.outcome -Expected "incomplete" -Message "A non-main workflow dispatch must fail closed." + Assert-Contains -Collection @($nonMainResult.Dossier.incomplete.reason_codes) -Value "workflow-dispatch-ref-not-main" -Message "Non-main dispatch reason code mismatch." + + $skipOnlyDir = New-DerivedFixture -Name "skip-only-negative" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.vstmr_summary.'6100003'[0].outcome = "Skipped" + $fixtureObject.vstmr_detail.'7100003:3'.outcome = "Skipped" + } + $resultSkipOnly = Invoke-Collector -IssueNumber 11 -FixtureRoot $skipOnlyDir -WorkDirectory (Join-Path $tempRoot "skip-only-negative") + Assert-Equal -Actual $resultSkipOnly.Dossier.outcome -Expected "incomplete" -Message "Skip-only evidence must not satisfy intermittency eligibility." + Assert-Contains -Collection @($resultSkipOnly.Dossier.incomplete.reason_codes) -Value "raw-evidence-insufficient" -Message "Skip-only evidence reason code mismatch." + + $unknownEnvironmentDir = New-DerivedFixture -Name "unknown-environment" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.vstmr_runs.'7100001'.name = "Quarantine-Mono-xunit" + } + $resultUnknownEnvironment = Invoke-Collector -IssueNumber 11 -FixtureRoot $unknownEnvironmentDir -WorkDirectory (Join-Path $tempRoot "unknown-environment") + Assert-Equal -Actual $resultUnknownEnvironment.Dossier.outcome -Expected "incomplete" -Message "Unknown required environment dimensions must fail closed." + Assert-Contains -Collection @($resultUnknownEnvironment.Dossier.incomplete.reason_codes) -Value "evidence-platform-unknown" -Message "Unknown platform reason code mismatch." + $unknownEnvironmentSource = @($resultUnknownEnvironment.Dossier.provenance.raw_evidence_sources | Where-Object { $_.build_id -eq 6100001 })[0] + Assert-Equal -Actual $unknownEnvironmentSource.configuration -Expected "not-encoded" -Message "Recognized TestRun families without Debug/Release must use the stable not-encoded configuration policy." + + $unknownExecutionLegDir = New-DerivedFixture -Name "unknown-execution-leg" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.vstmr_runs.'7100001'.name = "Windows" + } + $resultUnknownExecutionLeg = Invoke-Collector -IssueNumber 11 -FixtureRoot $unknownExecutionLegDir -WorkDirectory (Join-Path $tempRoot "unknown-execution-leg") + Assert-Equal -Actual $resultUnknownExecutionLeg.Dossier.outcome -Expected "incomplete" -Message "Unknown required TestRun identity must fail closed." + Assert-Contains -Collection @($resultUnknownExecutionLeg.Dossier.incomplete.reason_codes) -Value "evidence-test-run-identity-unknown" -Message "Unknown TestRun identity reason code mismatch." + Assert-Contains -Collection @($resultUnknownExecutionLeg.Dossier.incomplete.reason_codes) -Value "evidence-configuration-unknown" -Message "Unknown configuration reason code mismatch." + + $unknownPassExecutionLegDir = New-DerivedFixture -Name "unknown-pass-execution-leg" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.vstmr_runs.'7100003'.name = "Windows-Debug" + } + $resultUnknownPassExecutionLeg = Invoke-Collector -IssueNumber 11 -FixtureRoot $unknownPassExecutionLegDir -WorkDirectory (Join-Path $tempRoot "unknown-pass-execution-leg") + Assert-Equal -Actual $resultUnknownPassExecutionLeg.Dossier.outcome -Expected "incomplete" -Message "Unknown pass TestRun identity must fail closed." + Assert-Contains -Collection @($resultUnknownPassExecutionLeg.Dossier.incomplete.reason_codes) -Value "evidence-test-run-identity-unknown" -Message "Unknown pass TestRun identity reason code mismatch." + + $differentPassEnvironmentDir = New-DerivedFixture -Name "different-pass-environment" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.vstmr_runs.'7100003'.name = "Quarantine-Mono-Linux-Release-xunit" + } + $resultDifferentPassEnvironment = Invoke-Collector -IssueNumber 11 -FixtureRoot $differentPassEnvironmentDir -WorkDirectory (Join-Path $tempRoot "different-pass-environment") + Assert-Equal -Actual $resultDifferentPassEnvironment.Dossier.outcome -Expected "incomplete" -Message "A pass from a different environment must not prove intermittency." + Assert-Contains -Collection @($resultDifferentPassEnvironment.Dossier.incomplete.reason_codes) -Value "passed-evidence-environment-mismatch" -Message "Different pass environment reason code mismatch." + + $differentExecutionLegDir = New-DerivedFixture -Name "different-execution-leg" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.vstmr_runs.'7100003'.name = "Quarantine-CoreCLR-Windows-Debug-xunit" + } + $resultDifferentExecutionLeg = Invoke-Collector -IssueNumber 11 -FixtureRoot $differentExecutionLegDir -WorkDirectory (Join-Path $tempRoot "different-execution-leg") + Assert-Equal -Actual $resultDifferentExecutionLeg.Dossier.outcome -Expected "incomplete" -Message "A CoreCLR TestRun must not prove Mono failures intermittent." + Assert-Contains -Collection @($resultDifferentExecutionLeg.Dossier.incomplete.reason_codes) -Value "passed-evidence-environment-mismatch" -Message "Different TestRun identity reason code mismatch." + + $compositeExecutionLegDir = New-DerivedFixture -Name "composite-execution-leg" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.vstmr_runs.'7100001'.name = "Quarantine-Mono-WebAssembly-Windows-Debug-xunit" + $fixtureObject.vstmr_runs.'7100002'.name = "Quarantine-Mono-WebAssembly-Windows-Debug-xunit" + } + $resultCompositeExecutionLeg = Invoke-Collector -IssueNumber 11 -FixtureRoot $compositeExecutionLegDir -WorkDirectory (Join-Path $tempRoot "composite-execution-leg") + Assert-Equal -Actual $resultCompositeExecutionLeg.Dossier.outcome -Expected "incomplete" -Message "A Mono pass must not match a distinct Mono+WebAssembly TestRun identity." + Assert-Contains -Collection @($resultCompositeExecutionLeg.Dossier.incomplete.reason_codes) -Value "passed-evidence-environment-mismatch" -Message "Composite TestRun identity reason code mismatch." + + $passAfterLastDir = New-DerivedFixture -Name "pass-after-last-failure" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.negative_scan.'83'[0].startTime = "2026-08-02T00:00:00Z" + $fixtureObject.negative_scan.'83'[0].finishTime = "2026-08-02T01:00:00Z" + } + $resultPassAfterLast = Invoke-Collector -IssueNumber 11 -FixtureRoot $passAfterLastDir -WorkDirectory (Join-Path $tempRoot "pass-after-last-failure") + Assert-Equal -Actual $resultPassAfterLast.Dossier.outcome -Expected "incomplete" -Message "A pass after the last failure must not prove active intermittency." + Assert-Contains -Collection @($resultPassAfterLast.Dossier.incomplete.reason_codes) -Value "passed-evidence-not-interleaved" -Message "Pass-after-last reason code mismatch." + + $invalidBuildCases = @( + [ordered]@{ + Name = "wrong-definition" + ReasonCode = "azdo-build-definition-not-allowed" + Mutate = { param($fixtureObject) $fixtureObject.azdo_builds.'6100001'.definition.id = 999 } + }, + [ordered]@{ + Name = "wrong-branch" + ReasonCode = "azdo-build-source-branch-not-main" + Mutate = { param($fixtureObject) $fixtureObject.azdo_builds.'6100001'.sourceBranch = "refs/pull/123/merge" } + }, + [ordered]@{ + Name = "incomplete-build" + ReasonCode = "azdo-build-not-completed" + Mutate = { param($fixtureObject) $fixtureObject.azdo_builds.'6100001'.status = "inProgress" } + }, + [ordered]@{ + Name = "wrong-result" + ReasonCode = "azdo-build-result-incompatible" + Mutate = { param($fixtureObject) $fixtureObject.azdo_builds.'6100001'.result = "succeeded" } + } + ) + foreach ($invalidBuildCase in $invalidBuildCases) + { + $invalidBuildDir = New-DerivedFixture -Name $invalidBuildCase.Name -Source $flagsDir -Mutate $invalidBuildCase.Mutate + $invalidBuildResult = Invoke-Collector -IssueNumber 11 -FixtureRoot $invalidBuildDir -WorkDirectory (Join-Path $tempRoot $invalidBuildCase.Name) + Assert-Equal -Actual $invalidBuildResult.Dossier.outcome -Expected "incomplete" -Message "$($invalidBuildCase.Name) build must fail closed." + Assert-Contains -Collection @($invalidBuildResult.Dossier.incomplete.reason_codes) -Value $invalidBuildCase.ReasonCode -Message "$($invalidBuildCase.Name) reason code mismatch." + } + + # ------------------------------------------------------------------ + # A duplicate-search hit is discovery only. Even the same FQN must remain unvalidated when the + # documented KBE signature is incompatible with the authoritative failure evidence. + # ------------------------------------------------------------------ + $dupTestName = "Sample.Tests.DuplicateValidationCase" + $dupSignature = "System.InvalidOperationException: Sample failure for duplicate validation testing." + $dupShaA = "c78cc5badc905159a96a0d4bb0686acadaddc5c3" + $dupShaB = "439036f2881b7046fe9b9c3953bff60ed45dda6a" + $dupShaC = "48bf6fbc5f5aa8484a773f612851e95a4f52973a" + $dupDir = New-SyntheticFixture -Name "duplicate-unvalidated" -Overrides @{ + issue = [ordered]@{ + number = 12 + state = "open" + labels = @("test-failure") + body = "## Failing Test(s)`n`` $dupTestName ``$([System.Environment]::NewLine)## Error Message$([System.Environment]::NewLine)${codeFence}text$([System.Environment]::NewLine)$dupSignature$([System.Environment]::NewLine)$codeFence$([System.Environment]::NewLine)## Build$([System.Environment]::NewLine)https://dev.azure.com/dnceng-public/public/_build/results?buildId=6200001$([System.Environment]::NewLine)$workflowMarker" + } + azdo_builds = [ordered]@{ + "6200001" = [ordered]@{ definition = [ordered]@{ id = 83 }; sourceVersion = $dupShaA; startTime = "2026-08-01T00:00:00Z"; finishTime = "2026-08-01T01:00:00Z"; result = "failed" } + } + recurrence_scan = [ordered]@{ + "83" = @([ordered]@{ id = 6200002; sourceVersion = $dupShaB; startTime = "2026-07-30T00:00:00Z"; finishTime = "2026-07-30T01:00:00Z"; result = "failed" }) + } + negative_scan = [ordered]@{ + "83" = @([ordered]@{ id = 6200003; sourceVersion = $dupShaC; startTime = "2026-07-31T00:00:00Z"; finishTime = "2026-07-31T01:00:00Z"; result = "succeeded" }) + } + vstmr_summary = [ordered]@{ + "6200001" = @([ordered]@{ id = 1; runId = 7200001; outcome = "Failed"; automatedTestName = $dupTestName }) + "6200002" = @([ordered]@{ id = 2; runId = 7200002; outcome = "Failed"; automatedTestName = $dupTestName }) + "6200003" = @([ordered]@{ id = 3; runId = 7200003; outcome = "Passed"; automatedTestName = $dupTestName }) + } + vstmr_detail = [ordered]@{ + "7200001:1" = [ordered]@{ outcome = "Failed"; errorMessage = $dupSignature; stackTrace = "at $dupTestName.Run() (build 1)" } + "7200002:2" = [ordered]@{ outcome = "Failed"; errorMessage = $dupSignature; stackTrace = "at $dupTestName.Run() (build 2)" } + "7200003:3" = [ordered]@{ outcome = "Passed"; errorMessage = $null; stackTrace = $null } + } + vstmr_runs = [ordered]@{ + "7200001" = [ordered]@{ name = "Quarantine-Mono-Linux-Release-xunit" } + "7200002" = [ordered]@{ name = "Quarantine-Mono-Linux-Release-xunit" } + "7200003" = [ordered]@{ name = "Quarantine-Mono-Linux-Release-xunit" } + } + duplicate_search = [ordered]@{ + "open-kbe" = [ordered]@{ complete = $true; result_numbers = @(99999); total_count = 1 } + "recently-closed-kbe" = [ordered]@{ complete = $true; result_numbers = @(); total_count = 0 } + "open-fix-pr" = [ordered]@{ complete = $true; result_numbers = @(); total_count = 0 } + "recently-merged-fix-pr" = [ordered]@{ complete = $true; result_numbers = @(); total_count = 0 } + } + duplicate_candidate_text = [ordered]@{ + "99999" = "Known Build Error for $dupTestName`n## Error Message$([System.Environment]::NewLine)${codeFence}json$([System.Environment]::NewLine){ `"ErrorMessage`": `"System.InvalidOperationException: A different root cause.`", `"BuildRetry`": false, `"ExcludeConsoleLog`": false }$([System.Environment]::NewLine)$codeFence" + } + } + $resultDup = Invoke-Collector -IssueNumber 12 -FixtureRoot $dupDir -WorkDirectory (Join-Path $tempRoot "duplicate-unvalidated") + Assert-Equal -Actual $resultDup.Dossier.outcome -Expected "candidate" -Message "duplicate-unvalidated outcome mismatch." + Assert-Equal -Actual $resultDup.Dossier.candidate.duplicate_check.status -Expected "none" -Message "A same-FQN KBE with an incompatible signature must never set duplicate_check.status to existing-kbe." + Assert-Equal -Actual (@($resultDup.Dossier.candidate.duplicate_check.references)).Count -Expected 0 -Message "An unvalidated search hit must never appear in duplicate_check.references." + $unvalidated = @($resultDup.Dossier.provenance.duplicate_search.unvalidated_candidates) + if ($unvalidated.Count -eq 0 -or -not (@($unvalidated | Where-Object { $_.number -eq 99999 }).Count -gt 0)) + { + throw "Expected an unvalidated_candidates entry for issue #99999." + } + + $compatibleKbeDir = New-DerivedFixture -Name "duplicate-compatible-kbe" -Source $dupDir -Mutate { + param($fixtureObject) + $fixtureObject.duplicate_candidate_text.'99999' = "Known Build Error for $dupTestName`n## Error Message$([System.Environment]::NewLine)${codeFence}json$([System.Environment]::NewLine){ `"ErrorMessage`": `"$dupSignature`", `"BuildRetry`": false, `"ExcludeConsoleLog`": false }$([System.Environment]::NewLine)$codeFence" + } + $resultCompatibleKbe = Invoke-Collector -IssueNumber 12 -FixtureRoot $compatibleKbeDir -WorkDirectory (Join-Path $tempRoot "duplicate-compatible-kbe") + Assert-Equal -Actual $resultCompatibleKbe.Dossier.candidate.duplicate_check.status -Expected "existing-kbe" -Message "An exact-FQN KBE with a compatible documented signature should validate." + + $detailFetchFailureDir = New-DerivedFixture -Name "duplicate-detail-fetch-failure" -Source $dupDir -Mutate { + param($fixtureObject) + $fixtureObject.duplicate_candidate_text.PSObject.Properties.Remove("99999") + } + $resultDetailFetchFailure = Invoke-Collector -IssueNumber 12 -FixtureRoot $detailFetchFailureDir -WorkDirectory (Join-Path $tempRoot "duplicate-detail-fetch-failure") + Assert-Equal -Actual $resultDetailFetchFailure.Dossier.outcome -Expected "incomplete" -Message "A failed duplicate candidate-detail fetch must make the collector incomplete." + Assert-Contains -Collection @($resultDetailFetchFailure.Dossier.incomplete.reason_codes) -Value "duplicate-detail-fetch-incomplete" -Message "Candidate-detail fetch failure reason code mismatch." + Assert-Equal -Actual $resultDetailFetchFailure.Dossier.provenance.duplicate_search.coverage.open_kbes -Expected $false -Message "The affected duplicate query coverage must be incomplete." + $failedDetailQuery = @($resultDetailFetchFailure.Dossier.provenance.duplicate_search.queries | Where-Object { $_.category -eq "open-kbe" })[0] + Assert-Equal -Actual $failedDetailQuery.complete -Expected $false -Message "The affected duplicate query must be marked incomplete." + + $fixPrUnvalidatedDir = New-DerivedFixture -Name "fix-pr-unvalidated" -Source $dupDir -Mutate { + param($fixtureObject) + $fixtureObject.duplicate_search.'open-kbe'.result_numbers = @() + $fixtureObject.duplicate_search.'open-kbe'.total_count = 0 + $fixtureObject.duplicate_search.'open-fix-pr'.result_numbers = @(99999) + $fixtureObject.duplicate_search.'open-fix-pr'.total_count = 1 + $fixtureObject.duplicate_candidate_text.'99999' = "Fix $dupTestName without a compatible signature or linked issue." + } + $resultFixPrUnvalidated = Invoke-Collector -IssueNumber 12 -FixtureRoot $fixPrUnvalidatedDir -WorkDirectory (Join-Path $tempRoot "fix-pr-unvalidated") + Assert-Equal -Actual $resultFixPrUnvalidated.Dossier.candidate.duplicate_check.status -Expected "none" -Message "An exact-FQN fix PR without compatible association must remain unvalidated." + + $fixPrMentionOnlyDir = New-DerivedFixture -Name "fix-pr-mention-only" -Source $dupDir -Mutate { + param($fixtureObject) + $fixtureObject.duplicate_search.'open-kbe'.result_numbers = @() + $fixtureObject.duplicate_search.'open-kbe'.total_count = 0 + $fixtureObject.duplicate_search.'open-fix-pr'.result_numbers = @(99999) + $fixtureObject.duplicate_search.'open-fix-pr'.total_count = 1 + $fixtureObject.duplicate_candidate_text.'99999' = "Fix $dupTestName`nRoot cause: $dupSignature" + } + $resultFixPrMentionOnly = Invoke-Collector -IssueNumber 12 -FixtureRoot $fixPrMentionOnlyDir -WorkDirectory (Join-Path $tempRoot "fix-pr-mention-only") + Assert-Equal -Actual $resultFixPrMentionOnly.Dossier.candidate.duplicate_check.status -Expected "none" -Message "FQN/signature co-occurrence alone must not classify a search hit as a fix PR." + $fixPrReason = [string](@($resultFixPrMentionOnly.Dossier.provenance.duplicate_search.unvalidated_candidates | Where-Object { $_.number -eq 99999 })[0].reason) + if (-not $fixPrReason.Contains("closing-link and changed-file relevance")) + { + throw "Unvalidated fix PR must record the unsupported proof limitation." + } + + # ------------------------------------------------------------------ + # Edge case (item 11): a literal ErrorMessage containing '*', '?', and '[' must be matched via + # ordinal substring containment, never PowerShell -like/-notlike wildcard semantics. A decoy + # build sharing only the literal '[' character (but not the full signature text) must NOT be + # picked up as a second recurrence match. + # ------------------------------------------------------------------ + $wildTestName = "Sample.Tests.WildcardSignatureCase" + $wildSignature = "Assert.Equal() Failure: Array index [0] was *unexpected*, value? did not match." + $wildShaA = "11c785d7c74de87dc8dabb59c500da8af9254f81" + $wildShaB = "1d386dc40f508a46c0b76768cdbb1226cb6fe626" + $wildShaDecoy = "7ab2248eea6ca9ec8fa5c10cabd3f5c520edd126" + $wildShaNeg = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + $wildDir = New-SyntheticFixture -Name "wildcard-signature" -Overrides @{ + issue = [ordered]@{ + number = 14 + state = "open" + labels = @("test-failure") + body = "## Failing Test(s)`n`` $wildTestName ``$([System.Environment]::NewLine)## Error Message$([System.Environment]::NewLine)${codeFence}text$([System.Environment]::NewLine)$wildSignature$([System.Environment]::NewLine)$codeFence$([System.Environment]::NewLine)## Build$([System.Environment]::NewLine)https://dev.azure.com/dnceng-public/public/_build/results?buildId=6400001$([System.Environment]::NewLine)$workflowMarker" + } + azdo_builds = [ordered]@{ + "6400001" = [ordered]@{ definition = [ordered]@{ id = 83 }; sourceVersion = $wildShaA; startTime = "2026-08-01T00:00:00Z"; finishTime = "2026-08-01T01:00:00Z"; result = "failed" } + } + recurrence_scan = [ordered]@{ + "83" = @( + [ordered]@{ id = 6400099; sourceVersion = $wildShaDecoy; startTime = "2026-07-31T00:00:00Z"; finishTime = "2026-07-31T01:00:00Z"; result = "failed" }, + [ordered]@{ id = 6400002; sourceVersion = $wildShaB; startTime = "2026-07-30T00:00:00Z"; finishTime = "2026-07-30T01:00:00Z"; result = "failed" } + ) + } + negative_scan = [ordered]@{ + "83" = @([ordered]@{ id = 6400003; sourceVersion = $wildShaNeg; startTime = "2026-07-31T00:00:00Z"; finishTime = "2026-07-31T01:00:00Z"; result = "succeeded" }) + } + vstmr_summary = [ordered]@{ + "6400001" = @([ordered]@{ id = 1; runId = 7400001; outcome = "Failed"; automatedTestName = $wildTestName }) + "6400099" = @([ordered]@{ id = 99; runId = 7400099; outcome = "Failed"; automatedTestName = $wildTestName }) + "6400002" = @([ordered]@{ id = 2; runId = 7400002; outcome = "Failed"; automatedTestName = $wildTestName }) + "6400003" = @([ordered]@{ id = 3; runId = 7400003; outcome = "Passed"; automatedTestName = $wildTestName }) + } + vstmr_detail = [ordered]@{ + "7400001:1" = [ordered]@{ outcome = "Failed"; errorMessage = $wildSignature; stackTrace = "at $wildTestName.Run() (build 1)" } + "7400099:99" = [ordered]@{ outcome = "Failed"; errorMessage = "Some unrelated failure referencing array index [5] elsewhere."; stackTrace = "at Sample.Tests.Unrelated.Run()" } + "7400002:2" = [ordered]@{ outcome = "Failed"; errorMessage = $wildSignature; stackTrace = "at $wildTestName.Run() (build 2)" } + "7400003:3" = [ordered]@{ outcome = "Passed"; errorMessage = $null; stackTrace = $null } + } + vstmr_runs = [ordered]@{ + "7400001" = [ordered]@{ name = "Quarantine-Mono-Linux-Debug-xunit" } + "7400099" = [ordered]@{ name = "Quarantine-Mono-Linux-Debug-xunit" } + "7400002" = [ordered]@{ name = "Quarantine-Mono-Linux-Debug-xunit" } + "7400003" = [ordered]@{ name = "Quarantine-Mono-Linux-Debug-xunit" } + } + duplicate_search = $defaultDuplicateSearch + } + $resultWildcard = Invoke-Collector -IssueNumber 14 -FixtureRoot $wildDir -WorkDirectory (Join-Path $tempRoot "wildcard-signature") + Assert-Equal -Actual $resultWildcard.Dossier.outcome -Expected "candidate" -Message "wildcard-signature outcome mismatch." + $wildFailureBuildIds = @($resultWildcard.Dossier.candidate.evidence.raw_logs | Where-Object { $_.role -eq "failure" } | ForEach-Object { $_.build.id }) + Assert-Equal -Actual $wildFailureBuildIds.Count -Expected 2 -Message "wildcard-signature must gather exactly two failure builds." + Assert-Contains -Collection $wildFailureBuildIds -Value 6400002 -Message "wildcard-signature must include the real matching recurrence build." + Assert-NotContains -Collection $wildFailureBuildIds -Value 6400099 -Message "wildcard-signature must NOT include the decoy build that only shares the literal '[' character, proving ordinal (not -like) matching." + + # ------------------------------------------------------------------ + # Merge-AzdoBuildLists: pure unit coverage for the failed+partiallySucceeded merge/dedupe + # (item 2) independent of any network access. Dot-source the collector (satisfying its + # mandatory parameters with the already-validated missing-marker fixture, which exits after + # Step 1) purely to bring the function into scope. + # ------------------------------------------------------------------ + . $collector -IssueNumber 10 -OutputFile (Join-Path $tempRoot "merge-unit-dossier.json") -CandidateFile (Join-Path $tempRoot "merge-unit-candidate.json") -EvidenceRoot (Join-Path $tempRoot "merge-unit-evidence") -FixtureRoot $missingMarkerDir -RepositoryRoot $repositoryRoot -DossierSchemaFile $dossierSchema -CandidateSchemaFile $candidateSchema | Out-Null + + # Real Azure DevOps build objects deserialize as PSCustomObject (via Invoke-RestMethod / + # ConvertFrom-Json); Sort-Object -Property only resolves a plain hashtable's "properties" via + # PSCustomObject-style member resolution, not dictionary key lookup, so PSCustomObject here + # matches production shape and is required for the -Property startTime sort below to work. + $failedList = @( + [PSCustomObject]@{ id = 1; startTime = "2026-01-01T00:00:00Z" }, + [PSCustomObject]@{ id = 2; startTime = "2026-01-02T00:00:00Z" } + ) + $partiallySucceededList = @( + [PSCustomObject]@{ id = 2; startTime = "2026-01-02T00:00:00Z" }, + [PSCustomObject]@{ id = 3; startTime = "2026-01-03T00:00:00Z" } + ) + $merged = @(Merge-AzdoBuildLists -Lists @($failedList, $partiallySucceededList) -Cap 10) + Assert-Equal -Actual $merged.Count -Expected 3 -Message "Merge-AzdoBuildLists must dedupe the build shared by both resultFilter queries." + Assert-Equal -Actual $merged[0].id -Expected 3 -Message "Merge-AzdoBuildLists must sort by startTime descending (most recent first)." + Assert-Equal -Actual $merged[2].id -Expected 1 -Message "Merge-AzdoBuildLists must preserve the oldest build last." + + $cappedMerged = @(Merge-AzdoBuildLists -Lists @($failedList, $partiallySucceededList) -Cap 2) + Assert-Equal -Actual $cappedMerged.Count -Expected 2 -Message "Merge-AzdoBuildLists must honor the cap." + + $emptyMerged = @(Merge-AzdoBuildLists -Lists @(@(), @()) -Cap 5) + Assert-Equal -Actual $emptyMerged.Count -Expected 0 -Message "Merge-AzdoBuildLists must return a real empty array (not collapse to null) when both inputs are empty." + + $runNameCases = @( + [ordered]@{ Name = "Linux-Release-xunit"; Identity = "linux-release-xunit"; Platform = "Linux"; Configuration = "Release" }, + [ordered]@{ Name = "Linux-xunit_7"; Identity = "linux-xunit"; Platform = "Linux"; Configuration = "not-encoded" }, + [ordered]@{ Name = "Ubuntu.2404.Amd64.Open"; Identity = "ubuntu.2404.amd64.open"; Platform = "Linux"; Configuration = "not-encoded" }, + [ordered]@{ Name = "Ubuntu.2404.Amd64.Open_2"; Identity = "ubuntu.2404.amd64.open"; Platform = "Linux"; Configuration = "not-encoded" }, + [ordered]@{ Name = "OSX.26.Arm64.Open"; Identity = "osx.26.arm64.open"; Platform = "macOS"; Configuration = "not-encoded" }, + [ordered]@{ Name = "Windows.Amd64.VS2026.Open"; Identity = "windows.amd64.vs2026.open"; Platform = "Windows"; Configuration = "not-encoded" }, + [ordered]@{ Name = "Windows-Release-xunit"; Identity = "windows-release-xunit"; Platform = "Windows"; Configuration = "Release" }, + [ordered]@{ Name = "Quarantine-Mono-Linux-Release-xunit"; Identity = "quarantine-mono-linux-release-xunit"; Platform = "Linux"; Configuration = "Release" }, + [ordered]@{ Name = "ComponentsE2E-CoreCLR-Linux-Release-xunit"; Identity = "componentse2e-coreclr-linux-release-xunit"; Platform = "Linux"; Configuration = "Release" }, + [ordered]@{ Name = "ComponentsE2E-WebAssembly-Linux-Release-xunit"; Identity = "componentse2e-webassembly-linux-release-xunit"; Platform = "Linux"; Configuration = "Release" }, + [ordered]@{ Name = "Linux-Release-js"; Identity = "linux-release-js"; Platform = "Linux"; Configuration = "Release" } + ) + foreach ($runNameCase in $runNameCases) + { + $environment = Get-TestRunEnvironmentFromName -RunName $runNameCase.Name + Assert-Equal -Actual $environment.TestRunIdentity -Expected $runNameCase.Identity -Message "TestRun identity mismatch for '$($runNameCase.Name)'." + Assert-Equal -Actual $environment.Platform -Expected $runNameCase.Platform -Message "Platform mismatch for '$($runNameCase.Name)'." + Assert-Equal -Actual $environment.Configuration -Expected $runNameCase.Configuration -Message "Configuration mismatch for '$($runNameCase.Name)'." + } + + $windowedQueryUri = [System.Uri]::UnescapeDataString((Get-AzdoNegativeBuildQueryUri ` + -DefinitionId 83 ` + -MinimumStartTime ([System.DateTimeOffset]"2026-07-30T00:00:00Z") ` + -MaximumStartTime ([System.DateTimeOffset]"2026-08-01T00:00:00Z") ` + -Cap 20)) + foreach ($expectedQueryPart in @( + "queryOrder=startTimeDescending", + "minTime=2026-07-30T00:00:00.0000000+00:00", + "maxTime=2026-08-01T00:00:00.0000000+00:00", + "`$top=20")) + { + if (-not $windowedQueryUri.Contains($expectedQueryPart, [System.StringComparison]::Ordinal)) + { + throw "Windowed Passed-build query is missing '$expectedQueryPart': $windowedQueryUri" + } + } + + Write-Host "All test-quarantine-kbe-shadow collector tests passed." +} +finally +{ + $env:GITHUB_REF = $originalGitHubRef + $env:GITHUB_SHA = $originalGitHubSha + + if (Test-Path -LiteralPath $tempRoot) + { + Remove-Item -LiteralPath $tempRoot -Recurse -Force + } +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 new file mode 100644 index 000000000000..e46e9873dca7 --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 @@ -0,0 +1,869 @@ +#!/usr/bin/env pwsh + +[CmdletBinding()] +param() + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$evaluator = "$PSScriptRoot/Evaluate-TestQuarantineKbeCandidate.ps1" +$candidateSchema = "$PSScriptRoot/test-quarantine-kbe-shadow-candidate.schema.json" +$receiptSchema = "$PSScriptRoot/test-quarantine-kbe-shadow-receipt.schema.json" +$repositoryRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path +$repositoryHead = (& git -C $repositoryRoot rev-parse HEAD).Trim() +$tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) "aspnetcore-kbe-shadow-$([System.Guid]::NewGuid().ToString('N'))" +$outsideRoot = "$tempRoot-outside" +$symbolicLinkPath = Join-Path $tempRoot "linked-evidence" + +function Assert-Equal +{ + param( + [Parameter(Mandatory = $true)]$Actual, + [Parameter(Mandatory = $true)]$Expected, + [Parameter(Mandatory = $true)][string]$Message + ) + + if ($Actual -ne $Expected) + { + throw "$Message Expected '$Expected', actual '$Actual'." + } +} + +function Write-Candidate +{ + param( + [Parameter(Mandatory = $true)][string]$Path, + [Parameter(Mandatory = $true)][string[]]$Signature, + [Parameter(Mandatory = $true)][object[]]$Logs, + [string]$SignatureKind = "ErrorMessage", + [string]$DuplicateStatus = "none", + [bool]$CompleteCoverage = $true, + [string]$ProposedClassification = "new-kbe-candidate" + ) + + $candidate = [ordered]@{ + schema_version = 1 + repository = "dotnet/aspnetcore" + repository_ref = [ordered]@{ + branch = "main" + commit_sha = $repositoryHead + } + issue = [ordered]@{ + number = 12345 + url = "https://github.com/dotnet/aspnetcore/issues/12345" + } + test = [ordered]@{ + fully_qualified_name = "Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + } + signature = [ordered]@{ + kind = $SignatureKind + values = $Signature + build_retry = $false + exclude_console_log = $false + } + policy = [ordered]@{ + minimum_failure_logs = 2 + minimum_negative_logs = 1 + } + evidence = [ordered]@{ + raw_logs = $Logs + corroborating_context = @( + [ordered]@{ + source = "quarantine-issue" + url = "https://github.com/dotnet/aspnetcore/issues/12345" + } + ) + } + duplicate_check = [ordered]@{ + status = $DuplicateStatus + checked_utc = [System.DateTimeOffset]::UtcNow.ToString("O") + coverage = [ordered]@{ + open_kbes = $CompleteCoverage + recently_closed_kbes = $CompleteCoverage + open_fix_prs = $CompleteCoverage + recently_merged_fix_prs = $CompleteCoverage + } + references = @() + queries = @( + [ordered]@{ + category = "open-kbe" + query = "repo:dotnet/aspnetcore is:issue label:`"Known Build Error`" SampleTests" + complete = $CompleteCoverage + result_numbers = @() + }, + [ordered]@{ + category = "recently-closed-kbe" + query = "repo:dotnet/aspnetcore is:issue is:closed SampleTests" + complete = $CompleteCoverage + result_numbers = @() + }, + [ordered]@{ + category = "open-fix-pr" + query = "repo:dotnet/aspnetcore is:pr is:open SampleTests" + complete = $CompleteCoverage + result_numbers = @() + }, + [ordered]@{ + category = "recently-merged-fix-pr" + query = "repo:dotnet/aspnetcore is:pr is:merged SampleTests" + complete = $CompleteCoverage + result_numbers = @() + } + ) + } + proposed_classification = $ProposedClassification + } + + $candidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $Path +} + +function New-LogEntry +{ + param( + [Parameter(Mandatory = $true)][string]$Id, + [Parameter(Mandatory = $true)][string]$Role, + [Parameter(Mandatory = $true)][string]$Outcome, + [Parameter(Mandatory = $true)][string]$Path + ) + + return [ordered]@{ + id = $Id + role = $Role + outcome = $Outcome + path = $Path + source_url = "https://example.invalid/$Path" + sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot $Path) -Algorithm SHA256).Hash.ToLowerInvariant() + build = [ordered]@{ + id = switch ($Id) + { + "failure-1" { 1001; break } + "failure-2" { 1002; break } + default { 1003 } + } + pipeline_definition_id = 83 + source_branch = "refs/heads/main" + source_version = switch ($Id) + { + "failure-1" { "2222222222222222222222222222222222222222"; break } + "failure-2" { "3333333333333333333333333333333333333333"; break } + default { "4444444444444444444444444444444444444444" } + } + started_utc = switch ($Id) + { + "failure-1" { "2026-08-20T12:00:00Z"; break } + "failure-2" { "2026-08-21T12:00:00Z"; break } + default { "2026-08-20T18:00:00Z" } + } + status = "completed" + result = if ($Role -eq "failure") { "failed" } else { "succeeded" } + test_run_identity = "quarantine-mono-linux-release-xunit" + platform = "Linux" + configuration = "Release" + } + } +} + +try +{ + [System.IO.Directory]::CreateDirectory($tempRoot) | Out-Null + $signature = "Expected completion signal before deterministic deadline" + Set-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value @( + "2026-08-22T04:28:28.1901712Z [xUnit.net 00:48:54.05] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes [FAIL]" + "2026-08-22T04:28:28.1991498Z Xunit.Sdk.TrueException: $signature" + ) + 1..60 | ForEach-Object { Add-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value "Diagnostic line $_" } + Add-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value "Repeated diagnostic: $signature" + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Failed Microsoft.AspNetCore.Example.Tests.SampleTests.Completes [2 s]" + "Xunit.Sdk.TrueException: $signature" + ) + Set-Content -LiteralPath (Join-Path $tempRoot "negative.log") -Value @( + "[PASS] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + "Finished normally" + ) + + $logs = @( + (New-LogEntry -Id "failure-1" -Role "failure" -Outcome "failed" -Path "failure-1.log"), + (New-LogEntry -Id "failure-2" -Role "failure" -Outcome "failed" -Path "failure-2.log"), + (New-LogEntry -Id "negative" -Role "negative" -Outcome "passed" -Path "negative.log") + ) + + $candidatePath = Join-Path $tempRoot "candidate.json" + $receiptPath = Join-Path $tempRoot "receipt.json" + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "validated" -Message "Valid candidate status mismatch." + Assert-Equal -Actual $receipt.shadow_recommendation -Expected "new-kbe-candidate" -Message "Valid candidate recommendation mismatch." + Assert-Equal -Actual $receipt.eligible_for_kbe_enrichment -Expected $false -Message "Shadow evaluator must not authorize enrichment from unverified provenance." + Assert-Equal -Actual $receipt.evidence_provenance_verified -Expected $false -Message "Shadow evidence provenance must remain unverified." + + $logs[2].build.started_utc = "2026-08-19T12:00:00Z" + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Predating pass evidence status mismatch." + if (-not (($receipt.reasons -join "`n").Contains("strictly between"))) + { + throw "Predating pass evidence must report the interleaving gate." + } + + $logs[2].build.started_utc = "2026-08-22T12:00:00Z" + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Pass-after-last-failure status mismatch." + if (-not (($receipt.reasons -join "`n").Contains("strictly between"))) + { + throw "Pass-after-last evidence must report the interleaving gate." + } + + $logs[2].build.started_utc = "2026-08-20T18:00:00Z" + $logs[2].build.test_run_identity = "quarantine-coreclr-linux-release-xunit" + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Different TestRun identity pass evidence status mismatch." + if (-not (($receipt.reasons -join "`n").Contains("pipeline definition, canonical TestRun identity, platform, and configuration"))) + { + throw "Mono-failure/CoreCLR-pass evidence must report the environment gate." + } + $logs[2].build.test_run_identity = "quarantine-mono-linux-release-xunit" + + Set-Content -LiteralPath (Join-Path $tempRoot "negative.log") -Value @( + "[SKIP] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + "Skipped by test infrastructure" + ) + $logs[2].outcome = "skipped" + $logs[2].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "negative.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Skip-only negative evidence status mismatch." + Assert-Equal -Actual $receipt.evidence.distinct_negative_build_count -Expected 0 -Message "Skipped evidence must not satisfy the authoritative Passed build gate." + + Set-Content -LiteralPath (Join-Path $tempRoot "negative.log") -Value @( + "[PASS] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + "Finished normally" + ) + $logs[2].outcome = "passed" + $logs[2].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "negative.log") -Algorithm SHA256).Hash.ToLowerInvariant() + $logs[0].build.platform = "unknown" + $logs[0].build.configuration = "unknown" + $logs[0].build.test_run_identity = "unknown" + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Unknown environment evidence status mismatch." + if (-not (($receipt.reasons -join "`n").Contains("unknown platform")) -or + -not (($receipt.reasons -join "`n").Contains("unknown configuration")) -or + -not (($receipt.reasons -join "`n").Contains("unknown canonical TestRun identity"))) + { + throw "Unknown environment evidence must report every missing dimension." + } + $logs[0].build.platform = "Linux" + $logs[0].build.configuration = "Release" + $logs[0].build.test_run_identity = "quarantine-mono-linux-release-xunit" + + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Starting an unrelated test" + "Xunit.Sdk.TrueException: $signature" + ) + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Unassociated failure evidence status mismatch." + Assert-Equal -Actual $receipt.evidence.logs[1].failed_test_detected -Expected $false -Message "Unassociated failure evidence marker mismatch." + + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Failed Microsoft.AspNetCore.Example.Tests.SampleTests.Completes [2 s]" + "Xunit.Sdk.TrueException: $signature" + ) + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + + Add-Content -LiteralPath (Join-Path $tempRoot "negative.log") -Value "[SKIP] $signature" + $logs[2].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "negative.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Collision candidate status mismatch." + Assert-Equal -Actual $receipt.shadow_recommendation -Expected "human-review" -Message "Collision candidate recommendation mismatch." + Assert-Equal -Actual $receipt.evidence.pass_or_skip_collision_count -Expected 1 -Message "Pass/skip collision count mismatch." + + Set-Content -LiteralPath (Join-Path $tempRoot "negative.log") -Value "Passed Other.Tests.Completes [1 ms] $signature" + $logs[2].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "negative.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "VSTest pass collision status mismatch." + Assert-Equal -Actual $receipt.evidence.pass_or_skip_collision_count -Expected 1 -Message "VSTest pass collision count mismatch." + + Set-Content -LiteralPath (Join-Path $tempRoot "negative.log") -Value "[PASS] SampleTests.Completes" + $logs[2].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "negative.log") -Algorithm SHA256).Hash.ToLowerInvariant() + + $multiValueSignature = @( + "Expected completion signal", + "before deterministic deadline" + ) + Set-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value @( + "[FAIL] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + $multiValueSignature[0] + $multiValueSignature[1] + "[PASS] $($multiValueSignature[0])" + "[PASS] $($multiValueSignature[1])" + ) + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Failed Microsoft.AspNetCore.Example.Tests.SampleTests.Completes [2 s]" + $multiValueSignature[0] + $multiValueSignature[1] + ) + $logs[0].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-1.log") -Algorithm SHA256).Hash.ToLowerInvariant() + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate -Path $candidatePath -Signature $multiValueSignature -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Multi-value pass collision status mismatch." + Assert-Equal -Actual $receipt.evidence.pass_or_skip_collision_count -Expected 2 -Message "Multi-value pass collision count mismatch." + + Set-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value @( + "[FAIL] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + "Xunit.Sdk.TrueException: $signature" + ) + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Failed Microsoft.AspNetCore.Example.Tests.SampleTests.Completes [2 s]" + "Xunit.Sdk.TrueException: $signature" + ) + $logs[0].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-1.log") -Algorithm SHA256).Hash.ToLowerInvariant() + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs -CompleteCoverage $false + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Incomplete duplicate check status mismatch." + Assert-Equal -Actual $receipt.shadow_recommendation -Expected "insufficient-evidence" -Message "Incomplete duplicate check recommendation mismatch." + + Copy-Item -LiteralPath (Join-Path $tempRoot "failure-1.log") -Destination (Join-Path $tempRoot "failure-2.log") -Force + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Duplicate failure evidence status mismatch." + Assert-Equal -Actual $receipt.evidence.distinct_failure_log_count -Expected 1 -Message "Distinct failure evidence count mismatch." + + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Failed Microsoft.AspNetCore.Example.Tests.SampleTests.Completes [2 s]" + "Xunit.Sdk.TrueException: $signature" + ) + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + $logs[1].build.id = $logs[0].build.id + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Same-build failure evidence status mismatch." + Assert-Equal -Actual $receipt.evidence.distinct_failure_log_count -Expected 2 -Message "Same-build distinct log count mismatch." + Assert-Equal -Actual $receipt.evidence.distinct_failure_build_count -Expected 1 -Message "Same-build distinct build count mismatch." + $logs[1].build.id = 1002 + + Write-Candidate ` + -Path $candidatePath ` + -Signature @("Microsoft.AspNetCore.Example.Tests.SampleTests.Completes") ` + -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Bare test identifier status mismatch." + Assert-Equal -Actual $receipt.eligible_for_kbe_enrichment -Expected $false -Message "Bare test identifier eligibility mismatch." + + Write-Candidate ` + -Path $candidatePath ` + -Signature @("SampleTests.Completes") ` + -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Test identifier fragment status mismatch." + + Write-Candidate ` + -Path $candidatePath ` + -Signature @("SampleTests\.Completes") ` + -SignatureKind "ErrorPattern" ` + -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Regex test identifier fragment status mismatch." + + Set-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value @( + "[FAIL] Microsoft.AspNetCore.Example.Tests.SampleTests.CompletesAsync" + "Xunit.Sdk.TrueException: $signature" + ) + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Failed Microsoft.AspNetCore.Example.Tests.SampleTests.CompletesAsync [2 s]" + "Xunit.Sdk.TrueException: $signature" + ) + $logs[0].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-1.log") -Algorithm SHA256).Hash.ToLowerInvariant() + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Prefixed test name association status mismatch." + Assert-Equal -Actual $receipt.evidence.logs[0].failed_test_detected -Expected $false -Message "Prefixed test name was treated as the declared test." + + Set-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value @( + "Exception message: `"[FAIL] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes`"" + "Xunit.Sdk.TrueException: $signature" + ) + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Diagnostic text mentions [FAILED] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + "Xunit.Sdk.TrueException: $signature" + ) + $logs[0].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-1.log") -Algorithm SHA256).Hash.ToLowerInvariant() + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Embedded failure marker status mismatch." + Assert-Equal -Actual $receipt.evidence.logs[0].failed_test_detected -Expected $false -Message "Embedded failure text was treated as a failed-test record." + + Set-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value @( + "[FAIL] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + "Xunit.Sdk.TrueException: $signature" + ) + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Failed Microsoft.AspNetCore.Example.Tests.SampleTests.Completes [2 s]" + "Xunit.Sdk.TrueException: $signature" + ) + $logs[0].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-1.log") -Algorithm SHA256).Hash.ToLowerInvariant() + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + + Write-Candidate ` + -Path $candidatePath ` + -Signature @("^Xunit\.Sdk\.TrueException: Expected completion signal before deterministic deadline$") ` + -SignatureKind "ErrorPattern" ` + -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "validated" -Message "Regex candidate status mismatch." + Assert-Equal -Actual $receipt.shadow_recommendation -Expected "new-kbe-candidate" -Message "Regex candidate recommendation mismatch." + + Write-Candidate ` + -Path $candidatePath ` + -Signature @($signature) ` + -Logs $logs ` + -DuplicateStatus "existing-kbe" ` + -ProposedClassification "reuse-existing-kbe" + + $existingCandidate = Get-Content -LiteralPath $candidatePath -Raw | ConvertFrom-Json -Depth 32 + $existingCandidate.duplicate_check.references = @("issue:54321") + $existingCandidate.duplicate_check.queries[0].result_numbers = @(54321) + $existingCandidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $candidatePath + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "validated" -Message "Existing KBE status mismatch." + Assert-Equal -Actual $receipt.shadow_recommendation -Expected "reuse-existing-kbe" -Message "Existing KBE recommendation mismatch." + + Write-Candidate ` + -Path $candidatePath ` + -Signature @($signature) ` + -Logs $logs ` + -DuplicateStatus "existing-fix-pr" ` + -ProposedClassification "quarantine-only" + $existingFixCandidate = Get-Content -LiteralPath $candidatePath -Raw | ConvertFrom-Json -Depth 32 + $existingFixCandidate.duplicate_check.references = @("pull-request:54322") + $existingFixCandidate.duplicate_check.queries[2].result_numbers = @(54322) + $existingFixCandidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $candidatePath + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Unsupported existing fix PR status mismatch." + if (-not (($receipt.reasons -join "`n").Contains("closing-link and changed-file relevance"))) + { + throw "Unsupported existing fix PR must report the missing proof." + } + + $existingCandidate.duplicate_check.queries[0].result_numbers = @() + $existingCandidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $candidatePath + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Unsubstantiated existing KBE reference status mismatch." + Assert-Equal -Actual $receipt.shadow_recommendation -Expected "human-review" -Message "Unsubstantiated existing KBE reference recommendation mismatch." + + Write-Candidate ` + -Path $candidatePath ` + -Signature @($signature) ` + -Logs $logs ` + -ProposedClassification "reuse-existing-kbe" + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Inconsistent duplicate classification status mismatch." + Assert-Equal -Actual $receipt.shadow_recommendation -Expected "human-review" -Message "Inconsistent duplicate classification recommendation mismatch." + + Write-Candidate ` + -Path $candidatePath ` + -Signature @($signature) ` + -Logs $logs + $contradictoryDuplicateCandidate = Get-Content -LiteralPath $candidatePath -Raw | ConvertFrom-Json -Depth 32 + $contradictoryDuplicateCandidate.duplicate_check.queries[0].result_numbers = @(54321) + $contradictoryDuplicateCandidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $candidatePath + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Contradictory no-duplicate status mismatch." + Assert-Equal -Actual $receipt.shadow_recommendation -Expected "human-review" -Message "Contradictory no-duplicate recommendation mismatch." + + $testName = "Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + Set-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value @( + "[FAIL] $testName" + "First unrelated failure" + ) + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "[FAIL] $testName" + "Second unrelated failure" + ) + $logs[0].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-1.log") -Algorithm SHA256).Hash.ToLowerInvariant() + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate ` + -Path $candidatePath ` + -Signature @("Microsoft\.AspNetCore\.Example\.Tests\.SampleTests\.Completes") ` + -SignatureKind "ErrorPattern" ` + -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Regex-escaped test identifier status mismatch." + Assert-Equal -Actual $receipt.eligible_for_kbe_enrichment -Expected $false -Message "Regex-escaped test identifier eligibility mismatch." + + Set-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value @( + "[FAIL] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + "Xunit.Sdk.TrueException: $signature" + ) + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Failed Microsoft.AspNetCore.Example.Tests.SampleTests.Completes [2 s]" + "Xunit.Sdk.TrueException: $signature" + ) + $logs[0].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-1.log") -Algorithm SHA256).Hash.ToLowerInvariant() + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + + Write-Candidate ` + -Path $candidatePath ` + -Signature @($signature) ` + -Logs $logs + $staleCandidate = Get-Content -LiteralPath $candidatePath -Raw | ConvertFrom-Json -Depth 32 + $staleCandidate.duplicate_check.checked_utc = [System.DateTimeOffset]::UtcNow.AddHours(-30).ToString("o") + $staleCandidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $candidatePath + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Stale duplicate search status mismatch." + Assert-Equal -Actual $receipt.shadow_recommendation -Expected "insufficient-evidence" -Message "Stale duplicate search recommendation mismatch." + + Write-Candidate ` + -Path $candidatePath ` + -Signature @($signature) ` + -Logs $logs + $wrongRepositoryCandidate = Get-Content -LiteralPath $candidatePath -Raw | ConvertFrom-Json -Depth 32 + $wrongRepositoryCandidate.repository_ref.commit_sha = "1111111111111111111111111111111111111111" + $wrongRepositoryCandidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $candidatePath + $repositoryMismatchRejected = $false + try + { + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + } + catch + { + $repositoryMismatchRejected = $_.Exception.Message -match "repository commit does not match" + } + + Assert-Equal -Actual $repositoryMismatchRejected -Expected $true -Message "Repository commit mismatch was not rejected." + Assert-Equal -Actual (Test-Path -LiteralPath $receiptPath) -Expected $false -Message "A stale receipt survived repository rejection." + + [System.IO.Directory]::CreateDirectory($outsideRoot) | Out-Null + $outsideEvidencePath = Join-Path $outsideRoot "failure.log" + Copy-Item -LiteralPath (Join-Path $tempRoot "failure-1.log") -Destination $outsideEvidencePath + $symbolicLinkCreated = $false + try + { + [System.IO.Directory]::CreateSymbolicLink($symbolicLinkPath, $outsideRoot) | Out-Null + $symbolicLinkCreated = $true + } + catch + { + Write-Warning "Skipping symbolic-link regression because the platform denied link creation." + } + + if ($symbolicLinkCreated) + { + Write-Candidate ` + -Path $candidatePath ` + -Signature @($signature) ` + -Logs $logs + $linkedCandidate = Get-Content -LiteralPath $candidatePath -Raw | ConvertFrom-Json -Depth 32 + $linkedCandidate.evidence.raw_logs[0].path = "linked-evidence/failure.log" + $linkedCandidate.evidence.raw_logs[0].sha256 = (Get-FileHash -LiteralPath $outsideEvidencePath -Algorithm SHA256).Hash.ToLowerInvariant() + $linkedCandidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $candidatePath + $symbolicLinkRejected = $false + try + { + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + } + catch + { + $symbolicLinkRejected = $_.Exception.Message -match "must not traverse a symbolic link" + } + + Assert-Equal -Actual $symbolicLinkRejected -Expected $true -Message "A symlinked evidence parent was not rejected." + Assert-Equal -Actual (Test-Path -LiteralPath $receiptPath) -Expected $false -Message "A stale receipt survived symbolic-link rejection." + } + + Write-Candidate ` + -Path $candidatePath ` + -Signature @($signature) ` + -Logs $logs + $tamperedCandidate = Get-Content -LiteralPath $candidatePath -Raw | ConvertFrom-Json -Depth 32 + $tamperedCandidate.evidence.raw_logs[0].sha256 = "0" * 64 + $tamperedCandidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $candidatePath + $hashMismatchRejected = $false + try + { + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + } + catch + { + $hashMismatchRejected = $_.Exception.Message -match "Evidence hash mismatch" + } + + Assert-Equal -Actual $hashMismatchRejected -Expected $true -Message "Tampered evidence was not rejected." + Assert-Equal -Actual (Test-Path -LiteralPath $receiptPath) -Expected $false -Message "A stale receipt survived evidence rejection." + + Write-Host "All test-quarantine KBE shadow evaluator tests passed." +} +finally +{ + if (Test-Path -LiteralPath $symbolicLinkPath) + { + Remove-Item -LiteralPath $symbolicLinkPath -Force + } + + if (Test-Path -LiteralPath $tempRoot) + { + Remove-Item -LiteralPath $tempRoot -Recurse -Force + } + + if (Test-Path -LiteralPath $outsideRoot) + { + Remove-Item -LiteralPath $outsideRoot -Recurse -Force + } +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-WorkflowScriptInjectionSafety.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-WorkflowScriptInjectionSafety.ps1 new file mode 100644 index 000000000000..84c4ba83002f --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-WorkflowScriptInjectionSafety.ps1 @@ -0,0 +1,132 @@ +#!/usr/bin/env pwsh +<# +.SYNOPSIS + Static regression test: no GitHub Actions `${{ ... }}` expression may appear inside a `run:` + script body in the test-quarantine-kbe-shadow workflows. + +.DESCRIPTION + Directly interpolating a `${{ ... }}` expression -- especially a workflow_dispatch input such + as `${{ inputs.signature }}` -- into a `run:` script body is a script-injection vector: a value + containing a quote, backtick, or newline can execute arbitrary commands on the runner. Every + value this workflow needs inside a script must instead flow through a step (or job) `env:` + binding and be read back as an opaque environment variable (e.g. `$env:SIGNATURE_INPUT`), + which the shell/PowerShell parser never re-parses as script text regardless of its content. + + This test parses each target workflow file's YAML block-scalar `run:` bodies (both `run: |` + and unquoted single-line `run: ...` forms) using a minimal, indentation-based reader -- no + external YAML/GitHub Actions parser dependency -- and fails if any of them contains a `${{` + token. It intentionally does not inspect `if:`, `env:`, `with:`, or `concurrency:` values: a + `${{ }}` expression there is evaluated by the Actions engine itself, not by a shell, and is not + a script-injection vector. +#> + +[CmdletBinding()] +param() + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$workflowsRoot = (Resolve-Path "$PSScriptRoot/../..").Path +$targetFiles = @( + (Join-Path $workflowsRoot "test-quarantine-kbe-shadow.yml"), + (Join-Path $workflowsRoot "test-quarantine-kbe-shadow-tests.yml") +) + +function Get-IndentWidth +{ + param([Parameter(Mandatory = $true)][string]$Line) + + $trimmed = $Line.TrimStart(" ") + return $Line.Length - $trimmed.Length +} + +function Get-RunBlockBodies +{ + param([Parameter(Mandatory = $true)][string]$Path) + + $lines = [System.IO.File]::ReadAllLines($Path) + $blocks = [System.Collections.Generic.List[object]]::new() + $i = 0 + while ($i -lt $lines.Count) + { + $line = $lines[$i] + $singleLineMatch = [regex]::Match($line, '^(\s*)run:\s*(?!\||>)(\S.*)$') + $blockMatch = [regex]::Match($line, '^(\s*)run:\s*[|>][+-]?\s*$') + + if ($singleLineMatch.Success) + { + $null = $blocks.Add([ordered]@{ StartLine = $i + 1; Text = $singleLineMatch.Groups[2].Value }) + $i++ + continue + } + + if ($blockMatch.Success) + { + $keyIndent = $blockMatch.Groups[1].Value.Length + $bodyLines = [System.Collections.Generic.List[string]]::new() + $j = $i + 1 + while ($j -lt $lines.Count) + { + $candidate = $lines[$j] + if ($candidate.Trim().Length -eq 0) + { + $bodyLines.Add($candidate) + $j++ + continue + } + if ((Get-IndentWidth -Line $candidate) -le $keyIndent) + { + break + } + $bodyLines.Add($candidate) + $j++ + } + $null = $blocks.Add([ordered]@{ StartLine = $i + 1; Text = ($bodyLines -join "`n") }) + $i = $j + continue + } + + $i++ + } + + return $blocks +} + +try +{ + $failures = [System.Collections.Generic.List[string]]::new() + + foreach ($file in $targetFiles) + { + if (-not (Test-Path -LiteralPath $file)) + { + throw "Target workflow file does not exist: $file" + } + + $blocks = Get-RunBlockBodies -Path $file + if ($blocks.Count -eq 0) + { + throw "No 'run:' blocks were found in $file; the parser may be broken (this test expects at least one)." + } + + foreach ($block in $blocks) + { + if ($block.Text.Contains('${{')) + { + $null = $failures.Add("$($file):$($block.StartLine): 'run:' block contains a '`${{ ... }}' expression.") + } + } + } + + if ($failures.Count -gt 0) + { + throw "Script-injection guard failed:`n" + ($failures -join "`n") + } + + Write-Host "No 'run:' block in the test-quarantine-kbe-shadow workflows contains a GitHub Actions expression." +} +catch +{ + Write-Error $_ + exit 1 +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json new file mode 100644 index 000000000000..141a41c22087 --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json @@ -0,0 +1,119 @@ +{ + "schema_version": 1, + "repository": "dotnet/aspnetcore", + "collector": { + "name": "Collect-TestQuarantineKbeEvidence.ps1", + "version": 1, + "generated_utc": "", + "fixture_mode": true, + "manual_signature_provided": false + }, + "issue": { + "number": 68724, + "url": "https://github.com/dotnet/aspnetcore/issues/68724", + "state": "open", + "labels": [ + "test-failure", + "area-blazor" + ], + "actor": "github-actions[bot]", + "has_workflow_marker": true, + "has_workflow_metadata": true, + "workflow_run_id": 32632851798 + }, + "outcome": "incomplete", + "provenance": { + "repository_ref_verification": { + "event_ref": "refs/heads/main", + "event_sha": "", + "checkout_sha": "", + "current_main_sha": "", + "checkout_matches_event_sha": true, + "event_ref_is_main": true, + "dispatch_sha_on_main": true, + "matches_main": true + }, + "azdo_builds": [ + { + "id": 1563420, + "found": true, + "retrieved_utc": "", + "source": "issue-body-reference", + "definition_id": 87, + "source_branch": "refs/heads/main", + "source_version": "4bb91afc6d034cfcd78cbf15cdb21e0f6f419d68", + "started_utc": "2026-08-22T03:28:01.2925985Z", + "finished_utc": "2026-08-22T05:06:06.219227Z", + "status": "completed", + "result": "failed" + } + ], + "build_insights_snapshots": [ + { + "source_version": "4bb91afc6d034cfcd78cbf15cdb21e0f6f419d68", + "found": false, + "retrieved_utc": "", + "exact_test_referenced": false, + "short_name_referenced": false, + "known_issue_referenced": false, + "known_issue_numbers": [] + } + ], + "raw_evidence_sources": [], + "duplicate_search": { + "status": "none", + "checked_utc": "", + "coverage": { + "open_kbes": true, + "recently_closed_kbes": true, + "open_fix_prs": true, + "recently_merged_fix_prs": true + }, + "references": [], + "queries": [ + { + "category": "open-kbe", + "query": "repo:dotnet/aspnetcore is:issue is:open label:\"Known Build Error\" 68724", + "complete": true, + "result_numbers": [], + "total_count": 0 + }, + { + "category": "recently-closed-kbe", + "query": "repo:dotnet/aspnetcore is:issue is:closed closed:>=2026-06-05 label:\"Known Build Error\" 68724", + "complete": true, + "result_numbers": [], + "total_count": 0 + }, + { + "category": "open-fix-pr", + "query": "repo:dotnet/aspnetcore is:pr is:open 68724", + "complete": true, + "result_numbers": [], + "total_count": 0 + }, + { + "category": "recently-merged-fix-pr", + "query": "repo:dotnet/aspnetcore is:pr is:merged merged:>=2026-06-05 68724", + "complete": true, + "result_numbers": [], + "total_count": 0 + } + ], + "unvalidated_candidates": [] + } + }, + "candidate": null, + "incomplete": { + "reason_codes": [ + "multiple-test-identities-unresolved" + ], + "message": "Collector could not produce a validated candidate for issue #68724 : multiple-test-identities-unresolved.", + "missing_evidence": [ + { + "kind": "test-name", + "detail": "'## Failing Test(s)' names 2 distinct test identities (Microsoft.AspNetCore.Components.E2ETest.Tests.VirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll; Microsoft.AspNetCore.Components.E2ETest.ServerExecutionTests.ServerVirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll); this collector requires exactly one unambiguous identity per run rather than guessing which one actually failed." + } + ] + } +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json new file mode 100644 index 000000000000..50efeec46638 --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json @@ -0,0 +1,56 @@ +{ + "issue": { + "number": 68724, + "state": "open", + "labels": [ + "test-failure", + "area-blazor" + ], + "body": "## Failing Test(s)\n`Microsoft.AspNetCore.Components.E2ETest.Tests.VirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll` (and its server-execution subclass override, `Microsoft.AspNetCore.Components.E2ETest.ServerExecutionTests.ServerVirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll`)\n\n## Failure Frequency\nFailed 4 times over the past 30 days.\n\n## Error Message\n```text\nOpenQA.Selenium.BrowserAssertFailedException : Xunit.Sdk.TrueException: Item 950 should remain aligned with the viewport top once the last item has loaded, but top rendered index was -1, scrollTop=0.\n```\n\n## Build\nhttps://dev.azure.com/dnceng-public/public/_build/results?buildId=1563420\n\n\n\n\n", + "user": { + "login": "github-actions[bot]" + } + }, + "azdo_builds": { + "1563420": { + "definition": { + "id": 87 + }, + "sourceVersion": "4bb91afc6d034cfcd78cbf15cdb21e0f6f419d68", + "startTime": "2026-08-22T03:28:01.2925985Z", + "finishTime": "2026-08-22T05:06:06.219227Z", + "result": "failed", + "sourceBranch": "refs/heads/main", + "status": "completed" + } + }, + "recurrence_scan": {}, + "negative_scan": {}, + "vstmr_summary": {}, + "vstmr_detail": {}, + "vstmr_runs": {}, + "check_runs": {}, + "duplicate_search": { + "open-kbe": { + "complete": true, + "result_numbers": [], + "total_count": 0 + }, + "recently-closed-kbe": { + "complete": true, + "result_numbers": [], + "total_count": 0 + }, + "open-fix-pr": { + "complete": true, + "result_numbers": [], + "total_count": 0 + }, + "recently-merged-fix-pr": { + "complete": true, + "result_numbers": [], + "total_count": 0 + } + }, + "duplicate_candidate_text": {} +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json new file mode 100644 index 000000000000..209fe0e328ca --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json @@ -0,0 +1,201 @@ +{ + "schema_version": 1, + "repository": "dotnet/aspnetcore", + "collector": { + "name": "Collect-TestQuarantineKbeEvidence.ps1", + "version": 1, + "generated_utc": "", + "fixture_mode": true, + "manual_signature_provided": true + }, + "issue": { + "number": 68945, + "url": "https://github.com/dotnet/aspnetcore/issues/68945", + "state": "open", + "labels": [ + "test-failure", + "area-networking" + ], + "actor": "app/github-actions", + "has_workflow_marker": true, + "has_workflow_metadata": true, + "workflow_run_id": 33496438442 + }, + "outcome": "incomplete", + "provenance": { + "repository_ref_verification": { + "event_ref": "refs/heads/main", + "event_sha": "", + "checkout_sha": "", + "current_main_sha": "", + "checkout_matches_event_sha": true, + "event_ref_is_main": true, + "dispatch_sha_on_main": true, + "matches_main": true + }, + "azdo_builds": [ + { + "id": 1569737, + "found": true, + "retrieved_utc": "", + "source": "issue-body-reference", + "definition_id": 83, + "source_branch": "refs/heads/main", + "source_version": "b5666daed660cf1862a197784eee65b42a74a64a", + "started_utc": "2026-08-27T08:00:00Z", + "finished_utc": "2026-08-27T09:45:00Z", + "status": "completed", + "result": "failed" + }, + { + "id": 1538879, + "found": true, + "retrieved_utc": "", + "source": "issue-body-reference", + "definition_id": 83, + "source_branch": "refs/heads/main", + "source_version": "7773bea1d63c05d7b7043d2ecd2a3fcdd0ff18dc", + "started_utc": "2026-08-04T08:00:00Z", + "finished_utc": "2026-08-04T09:45:00Z", + "status": "completed", + "result": "failed" + }, + { + "id": 1540500, + "found": true, + "retrieved_utc": "", + "source": "negative-scan", + "definition_id": 83, + "source_branch": "refs/heads/main", + "source_version": "d0bb51a3cabe3bd24dac952bcf8a183c91b54baa", + "started_utc": "2026-08-15T08:00:00Z", + "finished_utc": "2026-08-15T09:45:00Z", + "status": "completed", + "result": "succeeded" + } + ], + "build_insights_snapshots": [ + { + "source_version": "b5666daed660cf1862a197784eee65b42a74a64a", + "found": false, + "retrieved_utc": "", + "exact_test_referenced": false, + "short_name_referenced": false, + "known_issue_referenced": false, + "known_issue_numbers": [] + }, + { + "source_version": "7773bea1d63c05d7b7043d2ecd2a3fcdd0ff18dc", + "found": false, + "retrieved_utc": "", + "exact_test_referenced": false, + "short_name_referenced": false, + "known_issue_referenced": false, + "known_issue_numbers": [] + } + ], + "raw_evidence_sources": [ + { + "build_id": 1569737, + "role": "failure", + "kind": "vstmr-detail", + "run_id": 72708311, + "result_id": 400010, + "helix_unavailable": true, + "test_run_identity": "windows.amd64.vs2026.open", + "platform": "Windows", + "configuration": "not-encoded", + "found": true, + "captured_utc": "", + "sha256": "9ce4ad96859470b4e8756421f9aa4f89079cbb60af728e516ac5b2eb095157f1", + "evidence_path": "issue-68945-build-1569737-failure.log" + }, + { + "build_id": 1538879, + "role": "failure", + "found": false, + "captured_utc": "", + "note": "No VSTMR result for build 1538879 matched the expected outcome/signature for this test." + }, + { + "build_id": 1540500, + "role": "negative", + "kind": "vstmr-detail", + "run_id": 72708313, + "result_id": 400030, + "helix_unavailable": true, + "test_run_identity": "windows.amd64.vs2026.open", + "platform": "Windows", + "configuration": "not-encoded", + "found": true, + "captured_utc": "", + "sha256": "856486ff4bd7fa7052a9fd195f0ae82f267d0c184cda825072625f6545a1ae89", + "evidence_path": "issue-68945-build-1540500-negative.log" + } + ], + "duplicate_search": { + "status": "none", + "checked_utc": "", + "coverage": { + "open_kbes": true, + "recently_closed_kbes": true, + "open_fix_prs": true, + "recently_merged_fix_prs": true + }, + "references": [], + "queries": [ + { + "category": "open-kbe", + "query": "repo:dotnet/aspnetcore is:issue is:open label:\"Known Build Error\" POST_ClientCancellationUpload_RequestAbortRaised", + "complete": true, + "result_numbers": [], + "total_count": 0 + }, + { + "category": "recently-closed-kbe", + "query": "repo:dotnet/aspnetcore is:issue is:closed closed:>=2026-06-05 label:\"Known Build Error\" POST_ClientCancellationUpload_RequestAbortRaised", + "complete": true, + "result_numbers": [], + "total_count": 0 + }, + { + "category": "open-fix-pr", + "query": "repo:dotnet/aspnetcore is:pr is:open POST_ClientCancellationUpload_RequestAbortRaised", + "complete": true, + "result_numbers": [], + "total_count": 0 + }, + { + "category": "recently-merged-fix-pr", + "query": "repo:dotnet/aspnetcore is:pr is:merged merged:>=2026-06-05 POST_ClientCancellationUpload_RequestAbortRaised", + "complete": true, + "result_numbers": [], + "total_count": 0 + } + ], + "unvalidated_candidates": [] + } + }, + "candidate": null, + "incomplete": { + "reason_codes": [ + "raw-evidence-insufficient", + "passed-evidence-not-interleaved" + ], + "message": "Collector could not produce a validated candidate for issue #68945 : raw-evidence-insufficient, passed-evidence-not-interleaved.", + "missing_evidence": [ + { + "kind": "vstmr-evidence", + "detail": "Build 1538879: no matching, retrievable VSTMR result detail." + }, + { + "kind": "raw-evidence", + "detail": "Only 1 distinct build(s) produced retrievable failure evidence; at least 2 are required." + }, + { + "kind": "pass-evidence", + "detail": "No environment-matched Passed occurrence was strictly between an earlier and a later authoritative failure." + } + ] + } +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json new file mode 100644 index 000000000000..5c596fa7b590 --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json @@ -0,0 +1,122 @@ +{ + "issue": { + "number": 68945, + "state": "open", + "labels": [ + "test-failure", + "area-networking" + ], + "body": "## Failing Test(s)\n- `Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised`\n\n## Details\n\nThis test has failed intermittently on `main` in the past 30 days:\n- Build [1569737](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1569737&view=results) (2026-08-27)\n- Build [1538879](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1538879&view=results) (2026-08-04)\n\nThe failures are separated by many passing runs in between (not back-to-back), so this is flakiness rather than a consistent regression. The test has never previously carried a `[QuarantinedTest]` attribute.\n\n\n\n\n", + "user": { + "login": "app/github-actions" + } + }, + "azdo_builds": { + "1569737": { + "definition": { + "id": 83 + }, + "sourceVersion": "b5666daed660cf1862a197784eee65b42a74a64a", + "startTime": "2026-08-27T08:00:00Z", + "finishTime": "2026-08-27T09:45:00Z", + "result": "failed", + "sourceBranch": "refs/heads/main", + "status": "completed" + }, + "1538879": { + "definition": { + "id": 83 + }, + "sourceVersion": "7773bea1d63c05d7b7043d2ecd2a3fcdd0ff18dc", + "startTime": "2026-08-04T08:00:00Z", + "finishTime": "2026-08-04T09:45:00Z", + "result": "failed", + "sourceBranch": "refs/heads/main", + "status": "completed" + } + }, + "recurrence_scan": {}, + "negative_scan": { + "83": [ + { + "id": 1540500, + "sourceVersion": "d0bb51a3cabe3bd24dac952bcf8a183c91b54baa", + "startTime": "2026-08-15T08:00:00Z", + "finishTime": "2026-08-15T09:45:00Z", + "result": "succeeded", + "sourceBranch": "refs/heads/main", + "status": "completed" + } + ] + }, + "vstmr_summary": { + "1569737": [ + { + "id": 400010, + "runId": 72708311, + "outcome": "Failed", + "automatedTestName": "Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised", + "testCaseTitle": "Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised" + } + ], + "1540500": [ + { + "id": 400030, + "runId": 72708313, + "outcome": "Passed", + "automatedTestName": "Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised", + "testCaseTitle": "Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised" + } + ] + }, + "vstmr_detail": { + "72708311:400010": { + "outcome": "Failed", + "errorMessage": "System.Threading.Tasks.TaskCanceledException: The operation was canceled.", + "stackTrace": "at System.Net.Http.Http3RequestStream.SendDataAsync(...)", + "testCase": { + "name": "Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised" + } + }, + "72708313:400030": { + "outcome": "Passed", + "errorMessage": null, + "stackTrace": null, + "testCase": { + "name": "Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised" + } + } + }, + "vstmr_runs": { + "72708311": { + "name": "Windows.Amd64.VS2026.Open" + }, + "72708313": { + "name": "Windows.Amd64.VS2026.Open" + } + }, + "check_runs": {}, + "duplicate_search": { + "open-kbe": { + "complete": true, + "result_numbers": [], + "total_count": 0 + }, + "recently-closed-kbe": { + "complete": true, + "result_numbers": [], + "total_count": 0 + }, + "open-fix-pr": { + "complete": true, + "result_numbers": [], + "total_count": 0 + }, + "recently-merged-fix-pr": { + "complete": true, + "result_numbers": [], + "total_count": 0 + } + }, + "duplicate_candidate_text": {} +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json new file mode 100644 index 000000000000..a81ed8dbc4ab --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json @@ -0,0 +1,212 @@ +{ + "schema_version": 1, + "repository": "dotnet/aspnetcore", + "collector": { + "name": "Collect-TestQuarantineKbeEvidence.ps1", + "version": 1, + "generated_utc": "", + "fixture_mode": true, + "manual_signature_provided": true + }, + "issue": { + "number": 68947, + "url": "https://github.com/dotnet/aspnetcore/issues/68947", + "state": "open", + "labels": [ + "test-failure", + "area-blazor" + ], + "actor": "app/github-actions", + "has_workflow_marker": true, + "has_workflow_metadata": true, + "workflow_run_id": 33496438442 + }, + "outcome": "incomplete", + "provenance": { + "repository_ref_verification": { + "event_ref": "refs/heads/main", + "event_sha": "", + "checkout_sha": "", + "current_main_sha": "", + "checkout_matches_event_sha": true, + "event_ref_is_main": true, + "dispatch_sha_on_main": true, + "matches_main": true + }, + "azdo_builds": [ + { + "id": 1551326, + "found": true, + "retrieved_utc": "", + "source": "issue-body-reference", + "definition_id": 87, + "source_branch": "refs/heads/main", + "source_version": "2a0388b463b2b80c4de4b6de4409857432ae9c1d", + "started_utc": "2026-08-13T04:18:53.903Z", + "finished_utc": "2026-08-13T04:31:51.187Z", + "status": "completed", + "result": "partiallySucceeded" + }, + { + "id": 1537561, + "found": false, + "retrieved_utc": "", + "source": "issue-body-reference", + "note": "Build metadata was not retrievable; it may have aged out of Azure DevOps retention." + }, + { + "id": 1549000, + "found": true, + "retrieved_utc": "", + "source": "recurrence-scan", + "definition_id": 87, + "source_branch": "refs/heads/main", + "source_version": "ef86306faaa4b31e962f06b93c2ce21e4a18bf17", + "started_utc": "2026-08-10T09:00:00Z", + "finished_utc": "2026-08-10T10:30:00Z", + "status": "completed", + "result": "partiallySucceeded" + }, + { + "id": 1545000, + "found": true, + "retrieved_utc": "", + "source": "negative-scan", + "definition_id": 87, + "source_branch": "refs/heads/main", + "source_version": "113a606f96ebd832970a1f377748746ff6526abc", + "started_utc": "2026-08-08T09:00:00Z", + "finished_utc": "2026-08-08T10:30:00Z", + "status": "completed", + "result": "succeeded" + } + ], + "build_insights_snapshots": [ + { + "source_version": "2a0388b463b2b80c4de4b6de4409857432ae9c1d", + "found": false, + "retrieved_utc": "", + "exact_test_referenced": false, + "short_name_referenced": false, + "known_issue_referenced": false, + "known_issue_numbers": [] + }, + { + "source_version": "ef86306faaa4b31e962f06b93c2ce21e4a18bf17", + "found": false, + "retrieved_utc": "", + "exact_test_referenced": false, + "short_name_referenced": false, + "known_issue_referenced": false, + "known_issue_numbers": [] + } + ], + "raw_evidence_sources": [ + { + "build_id": 1551326, + "role": "failure", + "kind": "vstmr-detail", + "run_id": 42708308, + "result_id": 100014, + "helix_unavailable": true, + "test_run_identity": "quarantine-mono-linux-release-xunit", + "platform": "Linux", + "configuration": "Release", + "found": true, + "captured_utc": "", + "sha256": "34cb3881593f491fba3598a0ee26c2e172c0b8e46c40b8e9ac3abde64af85529", + "evidence_path": "issue-68947-build-1551326-failure.log" + }, + { + "build_id": 1549000, + "role": "failure", + "kind": "vstmr-detail", + "run_id": 52708309, + "result_id": 200055, + "helix_unavailable": true, + "test_run_identity": "quarantine-mono-linux-release-xunit", + "platform": "Linux", + "configuration": "Release", + "found": true, + "captured_utc": "", + "sha256": "3772fbff9518a95cfe45f97d7a2f6250cc4af5b43c8d3c370f6d3a8fab8435d0", + "evidence_path": "issue-68947-build-1549000-failure.log" + }, + { + "build_id": 1545000, + "role": "negative", + "kind": "vstmr-detail", + "run_id": 62708310, + "result_id": 300099, + "helix_unavailable": true, + "test_run_identity": "quarantine-mono-linux-release-xunit", + "platform": "Linux", + "configuration": "Release", + "found": true, + "captured_utc": "", + "sha256": "17e453e2b6e8c8b2e2b16e3dcc2ea16e91bfd37a4c54b2077c52f71e21c54920", + "evidence_path": "issue-68947-build-1545000-negative.log" + } + ], + "duplicate_search": { + "status": "none", + "checked_utc": "", + "coverage": { + "open_kbes": true, + "recently_closed_kbes": true, + "open_fix_prs": true, + "recently_merged_fix_prs": true + }, + "references": [], + "queries": [ + { + "category": "open-kbe", + "query": "repo:dotnet/aspnetcore is:issue is:open label:\"Known Build Error\" RedirectEnhancedNonBlazorGetToExternal", + "complete": true, + "result_numbers": [], + "total_count": 0 + }, + { + "category": "recently-closed-kbe", + "query": "repo:dotnet/aspnetcore is:issue is:closed closed:>=2026-06-05 label:\"Known Build Error\" RedirectEnhancedNonBlazorGetToExternal", + "complete": true, + "result_numbers": [], + "total_count": 0 + }, + { + "category": "open-fix-pr", + "query": "repo:dotnet/aspnetcore is:pr is:open RedirectEnhancedNonBlazorGetToExternal", + "complete": true, + "result_numbers": [], + "total_count": 0 + }, + { + "category": "recently-merged-fix-pr", + "query": "repo:dotnet/aspnetcore is:pr is:merged merged:>=2026-06-05 RedirectEnhancedNonBlazorGetToExternal", + "complete": true, + "result_numbers": [], + "total_count": 0 + } + ], + "unvalidated_candidates": [] + } + }, + "candidate": null, + "incomplete": { + "reason_codes": [ + "raw-evidence-insufficient", + "passed-evidence-not-interleaved" + ], + "message": "Collector could not produce a validated candidate for issue #68947 : raw-evidence-insufficient, passed-evidence-not-interleaved.", + "missing_evidence": [ + { + "kind": "azdo-build", + "detail": "Build 1537561 metadata could not be retrieved." + }, + { + "kind": "pass-evidence", + "detail": "No environment-matched Passed occurrence was strictly between an earlier and a later authoritative failure." + } + ] + } +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json new file mode 100644 index 000000000000..69102ccb3a93 --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json @@ -0,0 +1,143 @@ +{ + "issue": { + "number": 68947, + "state": "open", + "labels": [ + "test-failure", + "area-blazor" + ], + "body": "## Failing Test(s)\n- `Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal`\n\n## Details\n\nThis test has failed intermittently on `main` in the past 30 days with a WebDriver/browser-startup timeout signature (`OpenQA.Selenium.WebDriverException` -> `TaskCanceledException`), the same signature that led to the related-but-distinct quarantine of `RedirectStreamingPostToExternal`/`RedirectEnhancedPostToExternal` in #68849:\n- Build [1551326](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1551326&view=results) (2026-08-13)\n- Build [1537561](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1537561&view=results) (2026-08-03)\n\nThe failures are separated by many passing runs, so this is browser/WebDriver flakiness rather than a consistent regression. The test has never previously carried a `[QuarantinedTest]` attribute.\n\n\n\n\n", + "user": { + "login": "app/github-actions" + } + }, + "azdo_builds": { + "1551326": { + "definition": { + "id": 87 + }, + "sourceVersion": "2a0388b463b2b80c4de4b6de4409857432ae9c1d", + "startTime": "2026-08-13T04:18:53.903Z", + "finishTime": "2026-08-13T04:31:51.187Z", + "result": "partiallySucceeded", + "sourceBranch": "refs/heads/main", + "status": "completed" + } + }, + "recurrence_scan": { + "87": [ + { + "id": 1549000, + "sourceVersion": "ef86306faaa4b31e962f06b93c2ce21e4a18bf17", + "startTime": "2026-08-10T09:00:00Z", + "finishTime": "2026-08-10T10:30:00Z", + "result": "partiallySucceeded", + "sourceBranch": "refs/heads/main", + "status": "completed" + } + ] + }, + "negative_scan": { + "87": [ + { + "id": 1545000, + "sourceVersion": "113a606f96ebd832970a1f377748746ff6526abc", + "startTime": "2026-08-08T09:00:00Z", + "finishTime": "2026-08-08T10:30:00Z", + "result": "succeeded", + "sourceBranch": "refs/heads/main", + "status": "completed" + } + ] + }, + "vstmr_summary": { + "1551326": [ + { + "id": 100014, + "runId": 42708308, + "outcome": "Failed", + "automatedTestName": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal", + "testCaseTitle": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal(disableThrowNavigationException: False)" + } + ], + "1549000": [ + { + "id": 200055, + "runId": 52708309, + "outcome": "Failed", + "automatedTestName": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal", + "testCaseTitle": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal(disableThrowNavigationException: False)" + } + ], + "1545000": [ + { + "id": 300099, + "runId": 62708310, + "outcome": "Passed", + "automatedTestName": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal", + "testCaseTitle": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal(disableThrowNavigationException: False)" + } + ] + }, + "vstmr_detail": { + "42708308:100014": { + "outcome": "Failed", + "errorMessage": "OpenQA.Selenium.WebDriverException : The HTTP request to the remote WebDriver server for URL http://localhost:43119/session/f57178cbb4abf6238d976e8cc5f5357b/window/rect timed out after 60 seconds.\n---- System.Threading.Tasks.TaskCanceledException : The request was canceled due to the configured HttpClient.Timeout of 60 seconds elapsing.", + "stackTrace": " at OpenQA.Selenium.Remote.HttpCommandExecutor.ExecuteAsync(Command commandToExecute)\n at Microsoft.AspNetCore.E2ETesting.BrowserTestBase.InitializeBrowser(String isolationContext) in /mnt/vss/_work/1/s/src/Shared/E2ETesting/BrowserTestBase.cs:line 97\n at Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.InitializeAsync() in /mnt/vss/_work/1/s/src/Components/test/E2ETest/ServerRenderingTests/RedirectionTest.cs:line 29", + "testCase": { + "name": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal(disableThrowNavigationException: False)" + } + }, + "52708309:200055": { + "outcome": "Failed", + "errorMessage": "OpenQA.Selenium.WebDriverException : The HTTP request to the remote WebDriver server for URL http://localhost:43119/session/f57178cbb4abf6238d976e8cc5f5357b/window/rect timed out after 60 seconds.\n---- System.Threading.Tasks.TaskCanceledException : The request was canceled due to the configured HttpClient.Timeout of 60 seconds elapsing.", + "stackTrace": " at OpenQA.Selenium.Remote.HttpCommandExecutor.ExecuteAsync(Command commandToExecute)\n at Microsoft.AspNetCore.E2ETesting.BrowserTestBase.InitializeBrowser(String isolationContext) in /mnt/vss/_work/1/s/src/Shared/E2ETesting/BrowserTestBase.cs:line 97\n at Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.InitializeAsync() in /mnt/vss/_work/1/s/src/Components/test/E2ETest/ServerRenderingTests/RedirectionTest.cs:line 29\n(observed in a separate run)", + "testCase": { + "name": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal(disableThrowNavigationException: False)" + } + }, + "62708310:300099": { + "outcome": "Passed", + "errorMessage": null, + "stackTrace": null, + "testCase": { + "name": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal(disableThrowNavigationException: False)" + } + } + }, + "vstmr_runs": { + "42708308": { + "name": "Quarantine-Mono-Linux-Release-xunit" + }, + "52708309": { + "name": "Quarantine-Mono-Linux-Release-xunit" + }, + "62708310": { + "name": "Quarantine-Mono-Linux-Release-xunit" + } + }, + "check_runs": {}, + "duplicate_search": { + "open-kbe": { + "complete": true, + "result_numbers": [], + "total_count": 0 + }, + "recently-closed-kbe": { + "complete": true, + "result_numbers": [], + "total_count": 0 + }, + "open-fix-pr": { + "complete": true, + "result_numbers": [], + "total_count": 0 + }, + "recently-merged-fix-pr": { + "complete": true, + "result_numbers": [], + "total_count": 0 + } + }, + "duplicate_candidate_text": {} +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json new file mode 100644 index 000000000000..b50c1cccff10 --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json @@ -0,0 +1,508 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/dotnet/aspnetcore/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json", + "title": "ASP.NET Core test quarantine KBE shadow candidate", + "description": "Untrusted candidate input proposed for deterministic Known Build Error signature evaluation. Source metadata remains unverified until a trusted collector is added.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "repository", + "repository_ref", + "issue", + "test", + "signature", + "policy", + "evidence", + "duplicate_check", + "proposed_classification" + ], + "properties": { + "schema_version": { + "const": 1 + }, + "repository": { + "const": "dotnet/aspnetcore" + }, + "repository_ref": { + "type": "object", + "additionalProperties": false, + "required": [ + "branch", + "commit_sha" + ], + "properties": { + "branch": { + "const": "main" + }, + "commit_sha": { + "$ref": "#/$defs/gitSha" + } + } + }, + "issue": { + "type": "object", + "additionalProperties": false, + "required": [ + "number", + "url" + ], + "properties": { + "number": { + "type": "integer", + "minimum": 1 + }, + "url": { + "type": "string", + "pattern": "^https://github\\.com/dotnet/aspnetcore/issues/[1-9][0-9]*$" + } + } + }, + "test": { + "type": "object", + "additionalProperties": false, + "required": [ + "fully_qualified_name" + ], + "properties": { + "fully_qualified_name": { + "type": "string", + "minLength": 3, + "maxLength": 1024, + "pattern": "^[^\\r\\n]+$" + } + } + }, + "signature": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "values", + "build_retry", + "exclude_console_log" + ], + "properties": { + "kind": { + "enum": [ + "ErrorMessage", + "ErrorPattern" + ] + }, + "values": { + "type": "array", + "minItems": 1, + "maxItems": 5, + "items": { + "type": "string", + "minLength": 8, + "maxLength": 2048, + "pattern": "^[^\\r\\n]+$" + } + }, + "build_retry": { + "type": "boolean" + }, + "exclude_console_log": { + "type": "boolean" + } + } + }, + "policy": { + "type": "object", + "additionalProperties": false, + "required": [ + "minimum_failure_logs", + "minimum_negative_logs" + ], + "properties": { + "minimum_failure_logs": { + "type": "integer", + "minimum": 2, + "maximum": 10 + }, + "minimum_negative_logs": { + "type": "integer", + "minimum": 1, + "maximum": 20 + } + } + }, + "evidence": { + "type": "object", + "additionalProperties": false, + "required": [ + "raw_logs", + "corroborating_context" + ], + "properties": { + "raw_logs": { + "type": "array", + "minItems": 1, + "maxItems": 30, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "role", + "outcome", + "path", + "source_url", + "sha256", + "build" + ], + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$" + }, + "role": { + "enum": [ + "failure", + "negative" + ] + }, + "outcome": { + "enum": [ + "failed", + "passed", + "skipped", + "unrelated" + ] + }, + "path": { + "type": "string", + "pattern": "^(?!.*(?:^|[\\\\/])\\.\\.(?:[\\\\/]|$))[A-Za-z0-9][A-Za-z0-9_.\\/-]{0,255}$" + }, + "source_url": { + "type": "string", + "format": "uri", + "maxLength": 2048, + "pattern": "^https://" + }, + "sha256": { + "$ref": "#/$defs/sha256" + }, + "build": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "pipeline_definition_id", + "source_branch", + "source_version", + "started_utc", + "status", + "result", + "test_run_identity", + "platform", + "configuration" + ], + "properties": { + "id": { + "type": "integer", + "minimum": 1 + }, + "pipeline_definition_id": { + "type": "integer", + "enum": [ + 83, + 87 + ] + }, + "source_branch": { + "const": "refs/heads/main" + }, + "source_version": { + "$ref": "#/$defs/gitSha" + }, + "started_utc": { + "type": "string", + "format": "date-time" + }, + "status": { + "const": "completed" + }, + "result": { + "enum": [ + "failed", + "partiallySucceeded", + "succeeded" + ] + }, + "test_run_identity": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\\r\\n]+$" + }, + "platform": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\\r\\n]+$" + }, + "configuration": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\\r\\n]+$" + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "role": { + "const": "failure" + } + } + }, + "then": { + "properties": { + "outcome": { + "const": "failed" + }, + "build": { + "properties": { + "result": { + "enum": [ + "failed", + "partiallySucceeded" + ] + } + } + } + } + } + }, + { + "if": { + "properties": { + "role": { + "const": "negative" + } + } + }, + "then": { + "properties": { + "outcome": { + "enum": [ + "passed", + "skipped", + "unrelated" + ] + }, + "build": { + "properties": { + "result": { + "const": "succeeded" + } + } + } + } + } + } + ] + } + }, + "corroborating_context": { + "type": "array", + "maxItems": 20, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "source", + "url" + ], + "properties": { + "source": { + "enum": [ + "build-insights", + "github-check", + "quarantine-issue" + ] + }, + "url": { + "type": "string", + "format": "uri", + "maxLength": 2048, + "pattern": "^https://" + } + } + } + } + } + }, + "duplicate_check": { + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "checked_utc", + "coverage", + "references", + "queries" + ], + "properties": { + "status": { + "enum": [ + "none", + "existing-kbe", + "existing-fix-pr", + "ambiguous", + "integrity-filtered", + "not-evaluated" + ] + }, + "checked_utc": { + "type": "string", + "format": "date-time" + }, + "coverage": { + "type": "object", + "additionalProperties": false, + "required": [ + "open_kbes", + "recently_closed_kbes", + "open_fix_prs", + "recently_merged_fix_prs" + ], + "properties": { + "open_kbes": { + "type": "boolean" + }, + "recently_closed_kbes": { + "type": "boolean" + }, + "open_fix_prs": { + "type": "boolean" + }, + "recently_merged_fix_prs": { + "type": "boolean" + } + } + }, + "references": { + "type": "array", + "maxItems": 20, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^(issue|pull-request):[1-9][0-9]*$" + } + }, + "queries": { + "type": "array", + "minItems": 4, + "maxItems": 20, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "category", + "query", + "complete", + "result_numbers" + ], + "properties": { + "category": { + "enum": [ + "open-kbe", + "recently-closed-kbe", + "open-fix-pr", + "recently-merged-fix-pr" + ] + }, + "query": { + "type": "string", + "minLength": 1, + "maxLength": 1024, + "pattern": "^[^\\r\\n]+$" + }, + "complete": { + "type": "boolean" + }, + "result_numbers": { + "type": "array", + "maxItems": 50, + "uniqueItems": true, + "items": { + "type": "integer", + "minimum": 1 + } + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "existing-kbe" + } + } + }, + "then": { + "properties": { + "references": { + "contains": { + "type": "string", + "pattern": "^issue:[1-9][0-9]*$" + }, + "minContains": 1 + } + } + } + }, + { + "if": { + "properties": { + "status": { + "const": "existing-fix-pr" + } + } + }, + "then": { + "properties": { + "references": { + "contains": { + "type": "string", + "pattern": "^pull-request:[1-9][0-9]*$" + }, + "minContains": 1 + } + } + } + } + ] + }, + "proposed_classification": { + "enum": [ + "reuse-existing-kbe", + "new-kbe-candidate", + "quarantine-only", + "infrastructure", + "timeout-needs-classification", + "insufficient-evidence", + "human-review" + ] + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "gitSha": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + } + } +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json new file mode 100644 index 000000000000..8ae09a48db76 --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json @@ -0,0 +1,793 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/dotnet/aspnetcore/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json", + "title": "ASP.NET Core test quarantine KBE shadow collector dossier", + "description": "Deterministic, read-only collector output for exactly one open dotnet/aspnetcore test-quarantine issue. Records provenance for public Azure DevOps/VSTMR test-result evidence and GitHub Build Insights check snapshots (advisory/corroborating only, never authoritative), then either emits a candidate object satisfying test-quarantine-kbe-shadow-candidate.schema.json for Evaluate-TestQuarantineKbeCandidate.ps1, or a structured incomplete outcome.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "repository", + "collector", + "issue", + "outcome", + "provenance", + "candidate", + "incomplete" + ], + "properties": { + "schema_version": { + "const": 1 + }, + "repository": { + "const": "dotnet/aspnetcore" + }, + "collector": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "version", + "generated_utc", + "fixture_mode", + "manual_signature_provided" + ], + "properties": { + "name": { + "const": "Collect-TestQuarantineKbeEvidence.ps1" + }, + "version": { + "const": 1 + }, + "generated_utc": { + "type": "string", + "format": "date-time" + }, + "fixture_mode": { + "type": "boolean", + "description": "true when the collector read pre-fetched offline fixture data instead of making live network calls (tests and workflow self-test mode)." + }, + "manual_signature_provided": { + "type": "boolean", + "description": "true when the operator supplied -Signature because deterministic extraction from the issue body could not be proven unambiguous." + } + } + }, + "issue": { + "type": "object", + "additionalProperties": false, + "required": [ + "number", + "url", + "state", + "labels", + "actor", + "has_workflow_marker", + "has_workflow_metadata", + "workflow_run_id" + ], + "properties": { + "number": { + "type": "integer", + "minimum": 1 + }, + "url": { + "type": "string", + "pattern": "^https://github\\.com/dotnet/aspnetcore/issues/[1-9][0-9]*$" + }, + "state": { + "enum": [ + "open", + "closed" + ] + }, + "labels": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + }, + "actor": { + "type": [ + "string", + "null" + ], + "maxLength": 128 + }, + "has_workflow_marker": { + "type": "boolean", + "description": "true only when the issue body contains both expected test-quarantine workflow markers." + }, + "has_workflow_metadata": { + "type": "boolean", + "description": "true only when structured gh-aw-agentic-workflow metadata names workflow_id test-quarantine and has matching id/run URL values for a dotnet/aspnetcore Actions run." + }, + "workflow_run_id": { + "type": [ + "integer", + "null" + ], + "minimum": 1 + } + } + }, + "outcome": { + "enum": [ + "candidate", + "incomplete" + ] + }, + "provenance": { + "type": "object", + "additionalProperties": false, + "required": [ + "repository_ref_verification", + "azdo_builds", + "build_insights_snapshots", + "raw_evidence_sources", + "duplicate_search" + ], + "properties": { + "repository_ref_verification": { + "type": "object", + "additionalProperties": false, + "required": [ + "event_ref", + "event_sha", + "checkout_sha", + "current_main_sha", + "checkout_matches_event_sha", + "event_ref_is_main", + "dispatch_sha_on_main", + "matches_main" + ], + "properties": { + "event_ref": { + "type": "string", + "maxLength": 256 + }, + "event_sha": { + "type": [ + "string", + "null" + ], + "pattern": "^[0-9a-f]{40}$" + }, + "checkout_sha": { + "$ref": "#/$defs/gitSha" + }, + "current_main_sha": { + "type": [ + "string", + "null" + ] + }, + "checkout_matches_event_sha": { + "type": "boolean" + }, + "event_ref_is_main": { + "type": "boolean" + }, + "dispatch_sha_on_main": { + "type": "boolean" + }, + "matches_main": { + "type": "boolean", + "description": "true only when the workflow dispatch ref is exactly refs/heads/main, checkout_sha equals event_sha, and event_sha is identical to or an ancestor/member of current main. A candidate is only ever emitted when this is true." + } + } + }, + "azdo_builds": { + "type": "array", + "maxItems": 64, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "found", + "retrieved_utc", + "source" + ], + "properties": { + "id": { + "type": "integer", + "minimum": 1 + }, + "found": { + "type": "boolean" + }, + "retrieved_utc": { + "type": "string", + "format": "date-time" + }, + "source": { + "enum": [ + "issue-body-reference", + "recurrence-scan", + "negative-scan" + ] + }, + "definition_id": { + "type": "integer", + "minimum": 1 + }, + "source_branch": { + "type": "string", + "maxLength": 256 + }, + "source_version": { + "$ref": "#/$defs/gitSha" + }, + "started_utc": { + "type": "string", + "format": "date-time" + }, + "finished_utc": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "result": { + "type": "string", + "maxLength": 64 + }, + "status": { + "type": "string", + "maxLength": 64 + }, + "note": { + "type": "string", + "maxLength": 512 + } + }, + "allOf": [ + { + "if": { + "properties": { + "found": { + "const": true + } + } + }, + "then": { + "required": [ + "definition_id", + "source_branch", + "source_version", + "started_utc", + "status", + "result" + ] + } + }, + { + "if": { + "properties": { + "found": { + "const": false + } + } + }, + "then": { + "required": [ + "note" + ] + } + } + ] + } + }, + "build_insights_snapshots": { + "type": "array", + "maxItems": 32, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "source_version", + "found", + "retrieved_utc", + "exact_test_referenced", + "short_name_referenced", + "known_issue_referenced", + "known_issue_numbers" + ], + "properties": { + "source_version": { + "$ref": "#/$defs/gitSha" + }, + "found": { + "type": "boolean" + }, + "retrieved_utc": { + "type": "string", + "format": "date-time" + }, + "check_id": { + "type": "integer", + "minimum": 1 + }, + "app_slug": { + "type": [ + "string", + "null" + ], + "maxLength": 128 + }, + "conclusion": { + "type": [ + "string", + "null" + ], + "maxLength": 32 + }, + "title": { + "type": "string", + "maxLength": 512 + }, + "text_sha256": { + "$ref": "#/$defs/sha256" + }, + "text_excerpt": { + "type": "string", + "maxLength": 2000 + }, + "html_url": { + "type": "string", + "format": "uri", + "pattern": "^https://" + }, + "details_url": { + "type": [ + "string", + "null" + ], + "format": "uri", + "pattern": "^https://" + }, + "snapshot_id": { + "type": [ + "string", + "null" + ], + "maxLength": 128 + }, + "exact_test_referenced": { + "type": "boolean", + "description": "Conservative match requiring the quarantined test's full fully-qualified name to appear verbatim in the check-run text. False when no snapshot was found or only a short method name matched." + }, + "short_name_referenced": { + "type": "boolean", + "description": "The test's bare method name (last '.'-separated segment) appeared in the check-run text, but the full fully-qualified name did not. Recorded for transparency only; never treated as an exact-test match because a short method name commonly collides with unrelated tests." + }, + "known_issue_referenced": { + "type": "boolean", + "description": "true only when known_issue_numbers is non-empty -- i.e. the text names a concrete dotnet/aspnetcore issue number/URL near a 'Known Issue' style label. The bare phrase 'Known Issue' with no associated issue reference (e.g. a heading or table column label) does not set this true." + }, + "known_issue_numbers": { + "type": "array", + "maxItems": 20, + "uniqueItems": true, + "items": { + "type": "integer", + "minimum": 1 + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "found": { + "const": true + } + } + }, + "then": { + "required": [ + "check_id", + "app_slug", + "conclusion", + "title", + "text_sha256", + "text_excerpt", + "details_url", + "html_url", + "snapshot_id" + ] + } + } + ] + } + }, + "raw_evidence_sources": { + "type": "array", + "maxItems": 64, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "build_id", + "role", + "found", + "captured_utc" + ], + "properties": { + "build_id": { + "type": "integer", + "minimum": 1 + }, + "role": { + "enum": [ + "failure", + "negative" + ] + }, + "kind": { + "enum": [ + "vstmr-detail" + ] + }, + "run_id": { + "type": "integer", + "minimum": 1 + }, + "result_id": { + "type": "integer", + "minimum": 1 + }, + "helix_unavailable": { + "type": "boolean", + "description": "true when no Helix job/work-item coordinates could be discovered for this result (the common case for ordinary xUnit test results -- Helix coordinates are only present on the Helix work-item's own crash/'.WorkItemExecution' pseudo-test row). The VSTMR detail result text is still treated as authoritative evidence in this case." + }, + "helix_job": { + "type": "string", + "maxLength": 128 + }, + "helix_workitem": { + "type": "string", + "maxLength": 256 + }, + "test_run_identity": { + "type": "string", + "maxLength": 128 + }, + "platform": { + "type": "string", + "maxLength": 128 + }, + "configuration": { + "type": "string", + "maxLength": 128 + }, + "found": { + "type": "boolean" + }, + "captured_utc": { + "type": "string", + "format": "date-time" + }, + "sha256": { + "$ref": "#/$defs/sha256" + }, + "evidence_path": { + "type": "string", + "maxLength": 512 + }, + "note": { + "type": "string", + "maxLength": 512 + } + }, + "allOf": [ + { + "if": { + "properties": { + "found": { + "const": true + } + } + }, + "then": { + "required": [ + "kind", + "run_id", + "result_id", + "helix_unavailable", + "test_run_identity", + "platform", + "configuration", + "sha256", + "evidence_path" + ] + } + } + ] + } + }, + "duplicate_search": { + "$ref": "#/$defs/duplicateCheck" + } + } + }, + "candidate": { + "type": [ + "object", + "null" + ], + "description": "Present only when outcome is 'candidate'. Must independently satisfy test-quarantine-kbe-shadow-candidate.schema.json; validated separately by the collector and by Evaluate-TestQuarantineKbeCandidate.ps1." + }, + "incomplete": { + "type": [ + "object", + "null" + ], + "additionalProperties": false, + "required": [ + "reason_codes", + "message", + "missing_evidence" + ], + "properties": { + "reason_codes": { + "type": "array", + "minItems": 1, + "maxItems": 16, + "uniqueItems": true, + "items": { + "enum": [ + "issue-not-canonical-quarantine", + "issue-not-open", + "test-name-unresolvable", + "multiple-test-identities-unresolved", + "repository-ref-not-main", + "workflow-dispatch-ref-not-main", + "checkout-sha-not-dispatch-sha", + "build-reference-unresolvable", + "build-metadata-expired", + "azdo-build-definition-not-allowed", + "azdo-build-source-branch-not-main", + "azdo-build-not-completed", + "azdo-build-result-incompatible", + "raw-evidence-expired", + "raw-evidence-insufficient", + "passed-evidence-not-interleaved", + "passed-evidence-environment-mismatch", + "evidence-test-run-identity-unknown", + "evidence-platform-unknown", + "evidence-configuration-unknown", + "recurrence-single-build-only", + "signature-extraction-ambiguous", + "duplicate-search-incomplete", + "duplicate-detail-fetch-incomplete", + "github-api-error" + ] + } + }, + "message": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "missing_evidence": { + "type": "array", + "maxItems": 64, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "detail" + ], + "properties": { + "kind": { + "type": "string", + "maxLength": 64 + }, + "detail": { + "type": "string", + "maxLength": 512 + } + } + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "outcome": { + "const": "candidate" + } + } + }, + "then": { + "properties": { + "candidate": { + "type": "object" + }, + "incomplete": { + "type": "null" + } + } + } + }, + { + "if": { + "properties": { + "outcome": { + "const": "incomplete" + } + } + }, + "then": { + "properties": { + "candidate": { + "type": "null" + }, + "incomplete": { + "type": "object" + } + } + } + } + ], + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "gitSha": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "duplicateCheck": { + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "checked_utc", + "coverage", + "references", + "queries", + "unvalidated_candidates" + ], + "properties": { + "status": { + "enum": [ + "none", + "existing-kbe", + "existing-fix-pr", + "ambiguous", + "integrity-filtered", + "not-evaluated" + ] + }, + "checked_utc": { + "type": "string", + "format": "date-time" + }, + "coverage": { + "type": "object", + "additionalProperties": false, + "required": [ + "open_kbes", + "recently_closed_kbes", + "open_fix_prs", + "recently_merged_fix_prs" + ], + "properties": { + "open_kbes": { + "type": "boolean" + }, + "recently_closed_kbes": { + "type": "boolean" + }, + "open_fix_prs": { + "type": "boolean" + }, + "recently_merged_fix_prs": { + "type": "boolean" + } + } + }, + "references": { + "type": "array", + "maxItems": 20, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^(issue|pull-request):[1-9][0-9]*$" + } + }, + "queries": { + "type": "array", + "minItems": 4, + "maxItems": 20, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "category", + "query", + "complete", + "result_numbers", + "total_count" + ], + "properties": { + "category": { + "enum": [ + "open-kbe", + "recently-closed-kbe", + "open-fix-pr", + "recently-merged-fix-pr" + ] + }, + "query": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "complete": { + "type": "boolean", + "description": "true only when GitHub reported incomplete_results=false AND every matching item (total_count) was actually retrieved -- a truncated page count no longer counts as complete." + }, + "result_numbers": { + "type": "array", + "maxItems": 200, + "uniqueItems": true, + "items": { + "type": "integer", + "minimum": 1 + } + }, + "total_count": { + "type": "integer", + "minimum": 0 + } + } + } + }, + "unvalidated_candidates": { + "type": "array", + "maxItems": 200, + "description": "Search hits that were fetched but could not establish the exact FQN plus a compatible KBE signature. Fix PRs are always recorded here until closing-link and changed-file relevance proof is implemented. A detail-fetch failure separately makes the affected query incomplete. Never contributes to 'references' or an existing-kbe/existing-fix-pr status.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "category", + "number", + "reason" + ], + "properties": { + "category": { + "enum": [ + "open-kbe", + "recently-closed-kbe", + "open-fix-pr", + "recently-merged-fix-pr" + ] + }, + "number": { + "type": "integer", + "minimum": 1 + }, + "reason": { + "type": "string", + "maxLength": 256 + } + } + } + } + } + } + } +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json new file mode 100644 index 000000000000..6d86312b53b7 --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json @@ -0,0 +1,800 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/dotnet/aspnetcore/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json", + "title": "ASP.NET Core test quarantine KBE shadow receipt", + "description": "Read-only deterministic evaluation receipt for one proposed Known Build Error signature.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "repository", + "repository_ref", + "generated_utc", + "evaluator", + "issue", + "test", + "signature", + "policy", + "evidence", + "duplicate_check", + "agent_proposed_classification", + "deterministic_status", + "shadow_recommendation", + "eligible_for_kbe_enrichment", + "evidence_provenance_verified", + "human_review_required", + "zero_remote_writes", + "reasons" + ], + "properties": { + "schema_version": { + "const": 1 + }, + "repository": { + "const": "dotnet/aspnetcore" + }, + "repository_ref": { + "$ref": "#/$defs/repositoryRef" + }, + "generated_utc": { + "type": "string", + "format": "date-time" + }, + "evaluator": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "version", + "matcher", + "failure_association_window_lines", + "candidate_sha256", + "candidate_schema_sha256", + "receipt_schema_sha256" + ], + "properties": { + "name": { + "const": "Evaluate-TestQuarantineKbeCandidate.ps1" + }, + "version": { + "const": 1 + }, + "matcher": { + "const": "Build Insights KBE ErrorMessage/ErrorPattern semantics with failed-test association" + }, + "failure_association_window_lines": { + "const": 50 + }, + "candidate_sha256": { + "$ref": "#/$defs/sha256" + }, + "candidate_schema_sha256": { + "$ref": "#/$defs/sha256" + }, + "receipt_schema_sha256": { + "$ref": "#/$defs/sha256" + } + } + }, + "issue": { + "$ref": "#/$defs/issue" + }, + "test": { + "$ref": "#/$defs/test" + }, + "signature": { + "$ref": "#/$defs/signature" + }, + "policy": { + "$ref": "#/$defs/policy" + }, + "evidence": { + "type": "object", + "additionalProperties": false, + "required": [ + "failure_log_count", + "negative_log_count", + "distinct_failure_log_count", + "distinct_negative_log_count", + "distinct_failure_build_count", + "distinct_negative_build_count", + "all_failure_logs_matched", + "negative_collision_count", + "pass_or_skip_collision_count", + "logs", + "corroborating_context" + ], + "properties": { + "failure_log_count": { + "type": "integer", + "minimum": 0 + }, + "negative_log_count": { + "type": "integer", + "minimum": 0 + }, + "distinct_failure_log_count": { + "type": "integer", + "minimum": 0 + }, + "distinct_negative_log_count": { + "type": "integer", + "minimum": 0 + }, + "distinct_failure_build_count": { + "type": "integer", + "minimum": 0 + }, + "distinct_negative_build_count": { + "type": "integer", + "minimum": 0 + }, + "all_failure_logs_matched": { + "type": "boolean" + }, + "negative_collision_count": { + "type": "integer", + "minimum": 0 + }, + "pass_or_skip_collision_count": { + "type": "integer", + "minimum": 0 + }, + "logs": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "role", + "outcome", + "path", + "source_url", + "sha256", + "build", + "line_count", + "matched", + "match_count", + "pass_or_skip_match_count", + "regex_timeout_count", + "failed_test_detected", + "failed_test_line_numbers", + "signature_associated_with_failed_test", + "matched_lines" + ], + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$" + }, + "role": { + "enum": [ + "failure", + "negative" + ] + }, + "outcome": { + "enum": [ + "failed", + "passed", + "skipped", + "unrelated" + ] + }, + "path": { + "type": "string", + "pattern": "^(?!.*(?:^|[\\\\/])\\.\\.(?:[\\\\/]|$))[A-Za-z0-9][A-Za-z0-9_.\\/-]{0,255}$" + }, + "source_url": { + "type": "string", + "format": "uri", + "maxLength": 2048, + "pattern": "^https://" + }, + "sha256": { + "$ref": "#/$defs/sha256" + }, + "build": { + "$ref": "#/$defs/build" + }, + "line_count": { + "type": "integer", + "minimum": 0 + }, + "matched": { + "type": "boolean" + }, + "match_count": { + "type": "integer", + "minimum": 0 + }, + "pass_or_skip_match_count": { + "type": "integer", + "minimum": 0 + }, + "regex_timeout_count": { + "type": "integer", + "minimum": 0 + }, + "failed_test_detected": { + "type": "boolean" + }, + "failed_test_line_numbers": { + "type": "array", + "maxItems": 20, + "items": { + "type": "integer", + "minimum": 1 + } + }, + "signature_associated_with_failed_test": { + "type": "boolean" + }, + "matched_lines": { + "type": "array", + "maxItems": 20, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "line_number", + "pattern_index", + "line_sha256", + "excerpt" + ], + "properties": { + "line_number": { + "type": "integer", + "minimum": 1 + }, + "pattern_index": { + "type": "integer", + "minimum": 0 + }, + "line_sha256": { + "$ref": "#/$defs/sha256" + }, + "excerpt": { + "type": "string", + "maxLength": 300 + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "role": { + "const": "failure" + } + } + }, + "then": { + "properties": { + "outcome": { + "const": "failed" + }, + "build": { + "properties": { + "result": { + "enum": [ + "failed", + "partiallySucceeded" + ] + } + } + } + } + } + }, + { + "if": { + "properties": { + "role": { + "const": "negative" + } + } + }, + "then": { + "properties": { + "outcome": { + "enum": [ + "passed", + "skipped", + "unrelated" + ] + }, + "build": { + "properties": { + "result": { + "const": "succeeded" + } + } + } + } + } + } + ] + } + }, + "corroborating_context": { + "type": "array", + "maxItems": 20, + "items": { + "$ref": "#/$defs/corroboratingContext" + } + } + } + }, + "duplicate_check": { + "$ref": "#/$defs/duplicateCheck" + }, + "agent_proposed_classification": { + "enum": [ + "reuse-existing-kbe", + "new-kbe-candidate", + "quarantine-only", + "infrastructure", + "timeout-needs-classification", + "insufficient-evidence", + "human-review" + ] + }, + "deterministic_status": { + "enum": [ + "validated", + "rejected", + "incomplete" + ] + }, + "shadow_recommendation": { + "enum": [ + "reuse-existing-kbe", + "new-kbe-candidate", + "existing-fix-pr", + "quarantine-only", + "infrastructure", + "timeout-needs-classification", + "insufficient-evidence", + "human-review" + ] + }, + "eligible_for_kbe_enrichment": { + "description": "Version 1 never authorizes repository mutation because evidence provenance is not independently authenticated.", + "const": false + }, + "evidence_provenance_verified": { + "description": "False until a trusted deterministic collector binds authoritative remote artifact metadata to the local evidence bundle.", + "const": false + }, + "human_review_required": { + "const": true + }, + "zero_remote_writes": { + "const": true + }, + "reasons": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1, + "maxLength": 512 + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "eligible_for_kbe_enrichment": { + "const": true + } + } + }, + "then": { + "properties": { + "deterministic_status": { + "const": "validated" + }, + "shadow_recommendation": { + "enum": [ + "reuse-existing-kbe", + "new-kbe-candidate" + ] + } + } + } + }, + { + "if": { + "properties": { + "deterministic_status": { + "const": "incomplete" + } + } + }, + "then": { + "properties": { + "eligible_for_kbe_enrichment": { + "const": false + }, + "shadow_recommendation": { + "const": "insufficient-evidence" + } + } + } + }, + { + "if": { + "properties": { + "deterministic_status": { + "const": "rejected" + } + } + }, + "then": { + "properties": { + "eligible_for_kbe_enrichment": { + "const": false + }, + "shadow_recommendation": { + "const": "human-review" + } + } + } + } + ], + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "gitSha": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "repositoryRef": { + "type": "object", + "additionalProperties": false, + "required": [ + "branch", + "commit_sha" + ], + "properties": { + "branch": { + "const": "main" + }, + "commit_sha": { + "$ref": "#/$defs/gitSha" + } + } + }, + "issue": { + "type": "object", + "additionalProperties": false, + "required": [ + "number", + "url" + ], + "properties": { + "number": { + "type": "integer", + "minimum": 1 + }, + "url": { + "type": "string", + "pattern": "^https://github\\.com/dotnet/aspnetcore/issues/[1-9][0-9]*$" + } + } + }, + "test": { + "type": "object", + "additionalProperties": false, + "required": [ + "fully_qualified_name" + ], + "properties": { + "fully_qualified_name": { + "type": "string", + "minLength": 3, + "maxLength": 1024, + "pattern": "^[^\\r\\n]+$" + } + } + }, + "signature": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "values", + "build_retry", + "exclude_console_log" + ], + "properties": { + "kind": { + "enum": [ + "ErrorMessage", + "ErrorPattern" + ] + }, + "values": { + "type": "array", + "minItems": 1, + "maxItems": 5, + "items": { + "type": "string", + "minLength": 8, + "maxLength": 2048, + "pattern": "^[^\\r\\n]+$" + } + }, + "build_retry": { + "type": "boolean" + }, + "exclude_console_log": { + "type": "boolean" + } + } + }, + "policy": { + "type": "object", + "additionalProperties": false, + "required": [ + "minimum_failure_logs", + "minimum_negative_logs" + ], + "properties": { + "minimum_failure_logs": { + "type": "integer", + "minimum": 2, + "maximum": 10 + }, + "minimum_negative_logs": { + "type": "integer", + "minimum": 1, + "maximum": 20 + } + } + }, + "build": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "pipeline_definition_id", + "source_branch", + "source_version", + "started_utc", + "status", + "result", + "test_run_identity", + "platform", + "configuration" + ], + "properties": { + "id": { + "type": "integer", + "minimum": 1 + }, + "pipeline_definition_id": { + "type": "integer", + "enum": [ + 83, + 87 + ] + }, + "source_branch": { + "const": "refs/heads/main" + }, + "source_version": { + "$ref": "#/$defs/gitSha" + }, + "started_utc": { + "type": "string", + "format": "date-time" + }, + "status": { + "const": "completed" + }, + "result": { + "enum": [ + "failed", + "partiallySucceeded", + "succeeded" + ] + }, + "test_run_identity": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\\r\\n]+$" + }, + "platform": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\\r\\n]+$" + }, + "configuration": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\\r\\n]+$" + } + } + }, + "corroboratingContext": { + "type": "object", + "additionalProperties": false, + "required": [ + "source", + "url" + ], + "properties": { + "source": { + "enum": [ + "build-insights", + "github-check", + "quarantine-issue" + ] + }, + "url": { + "type": "string", + "format": "uri", + "maxLength": 2048, + "pattern": "^https://" + } + } + }, + "duplicateCheck": { + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "checked_utc", + "coverage", + "references", + "queries" + ], + "properties": { + "status": { + "enum": [ + "none", + "existing-kbe", + "existing-fix-pr", + "ambiguous", + "integrity-filtered", + "not-evaluated" + ] + }, + "checked_utc": { + "type": "string", + "format": "date-time" + }, + "coverage": { + "type": "object", + "additionalProperties": false, + "required": [ + "open_kbes", + "recently_closed_kbes", + "open_fix_prs", + "recently_merged_fix_prs" + ], + "properties": { + "open_kbes": { + "type": "boolean" + }, + "recently_closed_kbes": { + "type": "boolean" + }, + "open_fix_prs": { + "type": "boolean" + }, + "recently_merged_fix_prs": { + "type": "boolean" + } + } + }, + "references": { + "type": "array", + "maxItems": 20, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^(issue|pull-request):[1-9][0-9]*$" + } + }, + "queries": { + "type": "array", + "minItems": 4, + "maxItems": 20, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "category", + "query", + "complete", + "result_numbers" + ], + "properties": { + "category": { + "enum": [ + "open-kbe", + "recently-closed-kbe", + "open-fix-pr", + "recently-merged-fix-pr" + ] + }, + "query": { + "type": "string", + "minLength": 1, + "maxLength": 1024, + "pattern": "^[^\\r\\n]+$" + }, + "complete": { + "type": "boolean" + }, + "result_numbers": { + "type": "array", + "maxItems": 50, + "uniqueItems": true, + "items": { + "type": "integer", + "minimum": 1 + } + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "existing-kbe" + } + } + }, + "then": { + "properties": { + "references": { + "contains": { + "type": "string", + "pattern": "^issue:[1-9][0-9]*$" + }, + "minContains": 1 + } + } + } + }, + { + "if": { + "properties": { + "status": { + "const": "existing-fix-pr" + } + } + }, + "then": { + "properties": { + "references": { + "contains": { + "type": "string", + "pattern": "^pull-request:[1-9][0-9]*$" + }, + "minContains": 1 + } + } + } + } + ] + } + } +} diff --git a/.github/workflows/test-quarantine-kbe-shadow-tests.yml b/.github/workflows/test-quarantine-kbe-shadow-tests.yml new file mode 100644 index 000000000000..83afbba487aa --- /dev/null +++ b/.github/workflows/test-quarantine-kbe-shadow-tests.yml @@ -0,0 +1,44 @@ +name: Test quarantine KBE shadow (unit tests) + +# Repository-convention CI coverage for the read-only test-quarantine-kbe-shadow evaluator and +# collector: runs their deterministic, offline PowerShell test suites whenever this workflow file, +# the scripts directory, or its schemas/fixtures change. Every fixture used here is a small, +# sanitized, offline JSON document; no network access or repository secrets are required. + +on: + pull_request: + paths: + - ".github/workflows/test-quarantine-kbe-shadow.yml" + - ".github/workflows/test-quarantine-kbe-shadow-tests.yml" + - ".github/workflows/scripts/test-quarantine-kbe-shadow/**" + workflow_dispatch: + +permissions: + contents: read + +jobs: + run-tests: + name: Run evaluator and collector tests + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + submodules: false + + - name: Run Evaluate-TestQuarantineKbeCandidate.ps1 tests + shell: pwsh + run: | + & "$env:GITHUB_WORKSPACE/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1" + + - name: Run Collect-TestQuarantineKbeEvidence.ps1 tests + shell: pwsh + run: | + & "$env:GITHUB_WORKSPACE/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1" + + - name: Run workflow script-injection safety guard + shell: pwsh + run: | + & "$env:GITHUB_WORKSPACE/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-WorkflowScriptInjectionSafety.ps1" diff --git a/.github/workflows/test-quarantine-kbe-shadow.yml b/.github/workflows/test-quarantine-kbe-shadow.yml new file mode 100644 index 000000000000..6973c690e565 --- /dev/null +++ b/.github/workflows/test-quarantine-kbe-shadow.yml @@ -0,0 +1,133 @@ +name: Test quarantine KBE shadow (single issue) + +# Maintainer-triggered, read-only shadow evaluation for exactly one existing dotnet/aspnetcore +# test-quarantine issue. Build Insights is the only GitHub check consumed, as corroborating +# evidence only. See .github/workflows/scripts/test-quarantine-kbe-shadow/README.md for the full +# trust boundary, evidence model, and promotion gates. +# +# This workflow NEVER labels, comments on, or otherwise mutates any issue, pull request, branch, +# or repository file. It only reads public GitHub/Azure DevOps/Helix data and uploads short- +# retention artifacts (candidate/dossier, receipt, and a human-readable summary). + +on: + workflow_dispatch: + inputs: + issue_number: + description: "The dotnet/aspnetcore issue number to evaluate. Must be the canonical, currently open test-failure quarantine issue for the test(s) it names." + required: true + signature: + description: "Optional manual failure-signature override. Only needed when the issue body has no fenced '## Error Message' code block that the collector can deterministically extract." + required: false + +# Fail closed on forks: this workflow only ever runs via an explicit, permissioned +# workflow_dispatch, but a fork's copy of this file would otherwise execute in the fork's own +# `github.repository` context. Gating on the canonical repository keeps a forked copy inert. +concurrency: + group: test-quarantine-kbe-shadow-${{ inputs.issue_number }} + cancel-in-progress: true + +permissions: + contents: read + issues: read + pull-requests: read + # Used only for the Build Insights check; raw Azure DevOps/VSTMR evidence remains authoritative. + checks: read + +jobs: + collect-and-evaluate: + name: Collect evidence and evaluate + if: ${{ github.repository == 'dotnet/aspnetcore' }} + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + # Every workflow_dispatch input is passed through a step `env:` binding and read back via + # $env: inside the script -- never interpolated directly into `run:` script text via + # `${{ inputs.* }}`. A signature value containing a quote, backtick, or newline embedded + # directly into the script body could otherwise execute arbitrary runner commands; an + # environment-variable value is opaque data to the shell/PowerShell parser regardless of + # its content. + - name: Validate and normalize inputs + id: normalize + shell: pwsh + env: + ISSUE_NUMBER_INPUT: ${{ inputs.issue_number }} + run: | + if ($env:ISSUE_NUMBER_INPUT -notmatch '^[1-9][0-9]*$') + { + Write-Error "issue_number must be a positive integer." + exit 1 + } + "ISSUE_NUMBER=$($env:ISSUE_NUMBER_INPUT)" >> $env:GITHUB_ENV + + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + submodules: false + persist-credentials: false + + - name: Collect quarantine evidence + id: collect + shell: pwsh + env: + GITHUB_TOKEN: ${{ github.token }} + ISSUE_NUMBER: ${{ env.ISSUE_NUMBER }} + SIGNATURE_INPUT: ${{ inputs.signature }} + EVENT_REF: ${{ github.ref }} + EVENT_SHA: ${{ github.sha }} + run: | + $params = @{ + IssueNumber = [int]$env:ISSUE_NUMBER + OutputFile = "$env:RUNNER_TEMP/dossier.json" + CandidateFile = "$env:RUNNER_TEMP/candidate.json" + EvidenceRoot = "$env:RUNNER_TEMP/evidence" + EventRef = $env:EVENT_REF + EventSha = $env:EVENT_SHA + } + if (-not [string]::IsNullOrWhiteSpace($env:SIGNATURE_INPUT)) + { + $params["Signature"] = $env:SIGNATURE_INPUT + } + & "$env:GITHUB_WORKSPACE/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1" @params + + $dossier = Get-Content -LiteralPath "$env:RUNNER_TEMP/dossier.json" -Raw | ConvertFrom-Json -Depth 32 + "outcome=$($dossier.outcome)" >> $env:GITHUB_OUTPUT + + - name: Evaluate candidate + id: evaluate + if: ${{ steps.collect.outputs.outcome == 'candidate' }} + shell: pwsh + run: | + & "$env:GITHUB_WORKSPACE/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1" ` + -CandidateFile "$env:RUNNER_TEMP/candidate.json" ` + -EvidenceRoot "$env:RUNNER_TEMP/evidence" ` + -OutputFile "$env:RUNNER_TEMP/receipt.json" + + - name: Render human-readable summary + if: ${{ always() && steps.collect.outcome == 'success' }} + shell: pwsh + run: | + $params = @{ + DossierFile = "$env:RUNNER_TEMP/dossier.json" + OutputFile = "$env:RUNNER_TEMP/summary.md" + } + if (Test-Path -LiteralPath "$env:RUNNER_TEMP/receipt.json") + { + $params["ReceiptFile"] = "$env:RUNNER_TEMP/receipt.json" + } + & "$env:GITHUB_WORKSPACE/.github/workflows/scripts/test-quarantine-kbe-shadow/New-TestQuarantineKbeSummary.ps1" @params + Get-Content -LiteralPath "$env:RUNNER_TEMP/summary.md" -Raw >> $env:GITHUB_STEP_SUMMARY + + - name: Upload dossier and receipt + if: ${{ always() && steps.collect.outcome == 'success' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-quarantine-kbe-shadow-issue-${{ env.ISSUE_NUMBER }} + path: | + ${{ runner.temp }}/dossier.json + ${{ runner.temp }}/candidate.json + ${{ runner.temp }}/receipt.json + ${{ runner.temp }}/summary.md + ${{ runner.temp }}/evidence/** + if-no-files-found: ignore + retention-days: 7